minio cache

pull/1329/head
withchao 2 years ago
parent ce9883cf26
commit c9ab3c63f8

@ -209,6 +209,9 @@ func batchGetCache2[T any, K comparable](
return fns(ctx, key)
})
if err != nil {
if errs.ErrRecordNotFound.Is(err) {
continue
}
return nil, err
}
res = append(res, val)

@ -2,10 +2,12 @@ package cache
import (
"context"
"github.com/OpenIMSDK/tools/errs"
"github.com/dtm-labs/rockscache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/redis/go-redis/v9"
"image"
"time"
)
@ -113,39 +115,40 @@ func (g *s3CacheRedis) GetKey(ctx context.Context, engine string, name string) (
})
}
//type MinioCache interface {
// metaCache
// GetThumbnailKey(ctx context.Context, key string) (string, error)
// //DelS3Key(engine string, keys ...string) S3Cache
//}
//
//func NewMinioCache(rdb redis.UniversalClient, s3 s3.Interface) MinioCache {
// rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions())
// return &minioCacheRedis{
// rcClient: rcClient,
// expireTime: time.Hour * 12,
// s3: s3,
// metaCache: NewMetaCacheRedis(rcClient),
// }
//}
//
//type minioCacheRedis struct {
// metaCache
// s3 s3.Interface
// rcClient *rockscache.Client
// expireTime time.Duration
//}
//
//func (g *minioCacheRedis) NewCache() MinioCache {
// return &minioCacheRedis{
// rcClient: g.rcClient,
// expireTime: g.expireTime,
// s3: g.s3,
// metaCache: NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...),
// }
//}
//
//func (g *minioCacheRedis) DelS3Key(engine string, keys ...string) S3Cache {
type MinioCache interface {
metaCache
GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context, key string) (*MinioImageInfo, image.Image, error)) (*MinioImageInfo, image.Image, error)
GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error)
//DelS3Key(engine string, keys ...string) S3Cache
}
func NewMinioCache(rdb redis.UniversalClient, s3 s3.Interface) MinioCache {
rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions())
return &minioCacheRedis{
rcClient: rcClient,
expireTime: time.Hour * 12,
s3: s3,
metaCache: NewMetaCacheRedis(rcClient),
}
}
type minioCacheRedis struct {
metaCache
s3 s3.Interface
rcClient *rockscache.Client
expireTime time.Duration
}
func (g *minioCacheRedis) NewCache() MinioCache {
return &minioCacheRedis{
rcClient: g.rcClient,
expireTime: g.expireTime,
s3: g.s3,
metaCache: NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...),
}
}
//func (g *minioCacheRedis) DelS3Key(engine string, keys ...string) MinioCache {
// s3cache := g.NewCache()
// ks := make([]string, 0, len(keys))
// for _, key := range keys {
@ -154,13 +157,44 @@ func (g *s3CacheRedis) GetKey(ctx context.Context, engine string, name string) (
// s3cache.AddKeys(ks...)
// return s3cache
//}
//
//func (g *minioCacheRedis) getMinioImageInfoKey(name string) string {
// return "MINIO:" + ":" + name
//}
//
//func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, name string) (string, error) {
// return getCache(ctx, g.rcClient, g.getS3Key(engine, name), g.expireTime, func(ctx context.Context) (*s3.ObjectInfo, error) {
// return g.s3.StatObject(ctx, name)
// })
//}
func (g *minioCacheRedis) getMinioImageInfoKey(name string) string {
return "MINIO:IMAGE:" + name
}
func (g *minioCacheRedis) getMinioImageThumbnailKey(name string) string {
return "MINIO:THUMBNAIL:" + name
}
func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context, key string) (*MinioImageInfo, image.Image, error)) (*MinioImageInfo, image.Image, error) {
var img image.Image
info, err := getCache(ctx, g.rcClient, g.getMinioImageInfoKey(key), g.expireTime, func(ctx context.Context) (info *MinioImageInfo, err error) {
info, img, err = fn(ctx, key)
return
})
if err != nil {
return nil, nil, err
}
if !info.IsImg {
return nil, nil, errs.ErrData.Wrap("object not image")
}
return info, img, nil
}
func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context, key string, format string, width int, height int) (string, error)) (string, error) {
return getCache(ctx, g.rcClient, g.getMinioImageThumbnailKey(key), g.expireTime, func(ctx context.Context) (string, error) {
info, img, err := g.GetImageObjectKeyInfo(ctx, key, getInfo)
if err != nil {
return "", err
}
return minioCache(ctx, key, format, width, height, info, img)
})
}
type MinioImageInfo struct {
IsImg bool `json:"i,omitempty"`
Width int `json:"w,omitempty"`
Height int `json:"h,omitempty"`
Format string `json:"f,omitempty"`
Etag string `json:"e,omitempty"`
}

@ -15,21 +15,14 @@
package minio
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"github.com/dtm-labs/rockscache"
"image"
"image/gif"
"image/jpeg"
"image/png"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"io"
"net/http"
"net/url"
"path"
"path/filepath"
"reflect"
"strconv"
"strings"
@ -57,10 +50,11 @@ const (
)
const (
maxImageWidth = 1024
maxImageHeight = 1024
maxImageSize = 1024 * 1024 * 50
pathInfo = "openim/thumbnail"
maxImageWidth = 1024
maxImageHeight = 1024
maxImageSize = 1024 * 1024 * 50
pathInfo = "openim/thumbnail"
maxImageInfoSize = 1024
)
func NewMinio() (s3.Interface, error) {
@ -125,7 +119,7 @@ type Minio struct {
lock sync.Locker
init bool
prefix string
rcClient *rockscache.Client
cache cache.MinioCache
}
func (m *Minio) initMinio(ctx context.Context) error {
@ -431,107 +425,108 @@ func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration
if opt.Image == nil || (opt.Image.Width < 0 && opt.Image.Height < 0 && opt.Image.Format == "") || (opt.Image.Width > maxImageWidth || opt.Image.Height > maxImageHeight) {
return m.presignedGetObject(ctx, name, expire, reqParams)
}
fileInfo, err := m.StatObject(ctx, name)
if err != nil {
return "", err
}
if fileInfo.Size > maxImageSize {
return "", errors.New("file size too large")
}
objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json")
var (
img image.Image
info minioImageInfo
)
data, err := m.getObjectData(ctx, objectInfoPath, 1024)
if err == nil {
if err := json.Unmarshal(data, &info); err != nil {
return "", fmt.Errorf("unmarshal minio image info.json error: %w", err)
}
if info.NotImage {
return "", errors.New("not image")
}
} else if m.IsNotFound(err) {
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
}
defer reader.Close()
imageInfo, format, err := ImageStat(reader)
if err == nil {
info.NotImage = false
info.Format = format
info.Width, info.Height = ImageWidthHeight(imageInfo)
img = imageInfo
} else {
info.NotImage = true
}
data, err := json.Marshal(&info)
if err != nil {
return "", err
}
if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil {
return "", err
}
} else {
return "", err
}
if opt.Image.Width > info.Width || opt.Image.Width <= 0 {
opt.Image.Width = info.Width
}
if opt.Image.Height > info.Height || opt.Image.Height <= 0 {
opt.Image.Height = info.Height
}
opt.Image.Format = strings.ToLower(opt.Image.Format)
if opt.Image.Format == formatJpg {
opt.Image.Format = formatJpeg
}
switch opt.Image.Format {
case formatPng:
case formatJpeg:
case formatGif:
default:
if info.Format == formatGif {
opt.Image.Format = formatGif
} else {
opt.Image.Format = formatJpeg
}
}
reqParams.Set("response-content-type", "image/"+opt.Image.Format)
if opt.Image.Width == info.Width && opt.Image.Height == info.Height && opt.Image.Format == info.Format {
return m.presignedGetObject(ctx, name, expire, reqParams)
}
cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Image.Width, opt.Image.Height, opt.Image.Format))
if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil {
return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
} else if !m.IsNotFound(err) {
return "", err
}
if img == nil {
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
}
defer reader.Close()
img, _, err = ImageStat(reader)
if err != nil {
return "", err
}
}
thumbnail := resizeImage(img, opt.Image.Width, opt.Image.Height)
buf := bytes.NewBuffer(nil)
switch opt.Image.Format {
case formatPng:
err = png.Encode(buf, thumbnail)
case formatJpeg:
err = jpeg.Encode(buf, thumbnail, nil)
case formatGif:
err = gif.Encode(buf, thumbnail, nil)
}
if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
return "", err
}
return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
return m.GetImageThumbnail(ctx, name, expire, opt.Image)
//fileInfo, err := m.StatObject(ctx, name)
//if err != nil {
// return "", err
//}
//if fileInfo.Size > maxImageSize {
// return "", errors.New("file size too large")
//}
//objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json")
//var (
// img image.Image
// info minioImageInfo
//)
//data, err := m.getObjectData(ctx, objectInfoPath, 1024)
//if err == nil {
// if err := json.Unmarshal(data, &info); err != nil {
// return "", fmt.Errorf("unmarshal minio image info.json error: %w", err)
// }
// if info.NotImage {
// return "", errors.New("not image")
// }
//} else if m.IsNotFound(err) {
// reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
// if err != nil {
// return "", err
// }
// defer reader.Close()
// imageInfo, format, err := ImageStat(reader)
// if err == nil {
// info.NotImage = false
// info.Format = format
// info.Width, info.Height = ImageWidthHeight(imageInfo)
// img = imageInfo
// } else {
// info.NotImage = true
// }
// data, err := json.Marshal(&info)
// if err != nil {
// return "", err
// }
// if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil {
// return "", err
// }
//} else {
// return "", err
//}
//if opt.Image.Width > info.Width || opt.Image.Width <= 0 {
// opt.Image.Width = info.Width
//}
//if opt.Image.Height > info.Height || opt.Image.Height <= 0 {
// opt.Image.Height = info.Height
//}
//opt.Image.Format = strings.ToLower(opt.Image.Format)
//if opt.Image.Format == formatJpg {
// opt.Image.Format = formatJpeg
//}
//switch opt.Image.Format {
//case formatPng:
//case formatJpeg:
//case formatGif:
//default:
// if info.Format == formatGif {
// opt.Image.Format = formatGif
// } else {
// opt.Image.Format = formatJpeg
// }
//}
//reqParams.Set("response-content-type", "image/"+opt.Image.Format)
//if opt.Image.Width == info.Width && opt.Image.Height == info.Height && opt.Image.Format == info.Format {
// return m.presignedGetObject(ctx, name, expire, reqParams)
//}
//cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Image.Width, opt.Image.Height, opt.Image.Format))
//if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil {
// return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
//} else if !m.IsNotFound(err) {
// return "", err
//}
//if img == nil {
// reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
// if err != nil {
// return "", err
// }
// defer reader.Close()
// img, _, err = ImageStat(reader)
// if err != nil {
// return "", err
// }
//}
//thumbnail := resizeImage(img, opt.Image.Width, opt.Image.Height)
//buf := bytes.NewBuffer(nil)
//switch opt.Image.Format {
//case formatPng:
// err = png.Encode(buf, thumbnail)
//case formatJpeg:
// err = jpeg.Encode(buf, thumbnail, nil)
//case formatGif:
// err = gif.Encode(buf, thumbnail, nil)
//}
//if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
// return "", err
//}
//return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
}
func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]byte, error) {
@ -543,7 +538,7 @@ func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]
if limit < 0 {
return io.ReadAll(object)
}
return io.ReadAll(io.LimitReader(object, 1024))
return io.ReadAll(io.LimitReader(object, limit))
}
func (m *Minio) GetThumbnailKey(ctx context.Context, name string) (string, error) {

@ -0,0 +1,235 @@
package minio
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"github.com/minio/minio-go/v7"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
"image"
"image/gif"
"image/jpeg"
"image/png"
"net/url"
"path"
"path/filepath"
"strings"
"time"
)
//func (m *Minio) getHashImageInfo1(ctx context.Context, key string) (*cache.MinioImageInfo, image.Image, error) {
//
// return nil, nil, nil
//}
func (m *Minio) get1(ctx context.Context, key string, format string, width int, height int, info *cache.MinioImageInfo, img image.Image) (string, error) {
return "", nil
}
func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
info, img, err := m.cache.GetImageObjectKeyInfo(ctx, name, m.getObjectImageInfo)
if err != nil {
return "", err
}
if opt.Width > info.Width || opt.Width <= 0 {
opt.Width = info.Width
}
if opt.Height > info.Height || opt.Height <= 0 {
opt.Height = info.Height
}
opt.Format = strings.ToLower(opt.Format)
if opt.Format == formatJpg {
opt.Format = formatJpeg
}
switch opt.Format {
case formatPng, formatJpeg, formatGif:
default:
opt.Format = ""
//if info.Format == formatGif {
// opt.Format = formatGif
//} else {
// opt.Format = formatJpeg
//}
}
if opt.Width == info.Width && opt.Height == info.Height && (opt.Format == info.Format || opt.Format == "") {
return "", nil
}
if opt.Format == "" {
switch opt.Format {
case formatGif:
opt.Format = formatGif
case formatJpeg:
opt.Format = formatJpeg
case formatPng:
opt.Format = formatPng
default:
opt.Format = formatPng
}
}
key, err := m.cache.GetThumbnailKey(ctx, name, opt.Format, opt.Width, opt.Height, func(ctx context.Context) (string, error) {
if img == nil {
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
}
defer reader.Close()
img, _, err = ImageStat(reader)
if err != nil {
return "", err
}
}
thumbnail := resizeImage(img, opt.Width, opt.Height)
buf := bytes.NewBuffer(nil)
switch opt.Format {
case formatPng:
err = png.Encode(buf, thumbnail)
case formatJpeg:
err = jpeg.Encode(buf, thumbnail, nil)
case formatGif:
err = gif.Encode(buf, thumbnail, nil)
}
cacheKey := filepath.Join(pathInfo, info.Etag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
return "", err
}
return cacheKey, nil
})
if err != nil {
return "", err
}
return m.presignedGetObject(ctx, key, expire, reqParams)
}
func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.MinioImageInfo, image.Image, error) {
fileInfo, err := m.StatObject(ctx, name)
if err != nil {
return nil, nil, err
}
if fileInfo.Size > maxImageSize {
return nil, nil, errors.New("file size too large")
}
imageData, err := m.getObjectData(ctx, name, fileInfo.Size)
if err != nil {
return nil, nil, err
}
var info cache.MinioImageInfo
imageInfo, format, err := ImageStat(bytes.NewReader(imageData))
if err == nil {
info.IsImg = true
info.Format = format
info.Width, info.Height = ImageWidthHeight(imageInfo)
} else {
info.IsImg = false
}
info.Etag = fileInfo.ETag
return &info, imageInfo, nil
}
func (m *Minio) GetImageThumbnail(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
fileInfo, err := m.StatObject(ctx, name)
if err != nil {
return "", err
}
if fileInfo.Size > maxImageSize {
return "", errors.New("file size too large")
}
objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json")
var (
img image.Image
info minioImageInfo
)
data, err := m.getObjectData(ctx, objectInfoPath, maxImageInfoSize)
if err == nil {
if err := json.Unmarshal(data, &info); err != nil {
return "", fmt.Errorf("unmarshal minio image info.json error: %w", err)
}
if info.NotImage {
return "", errors.New("not image")
}
} else if m.IsNotFound(err) {
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
}
defer reader.Close()
imageInfo, format, err := ImageStat(reader)
if err == nil {
info.NotImage = false
info.Format = format
info.Width, info.Height = ImageWidthHeight(imageInfo)
img = imageInfo
} else {
info.NotImage = true
}
data, err := json.Marshal(&info)
if err != nil {
return "", err
}
if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil {
return "", err
}
} else {
return "", err
}
if opt.Width > info.Width || opt.Width <= 0 {
opt.Width = info.Width
}
if opt.Height > info.Height || opt.Height <= 0 {
opt.Height = info.Height
}
opt.Format = strings.ToLower(opt.Format)
if opt.Format == formatJpg {
opt.Format = formatJpeg
}
switch opt.Format {
case formatPng:
case formatJpeg:
case formatGif:
default:
if info.Format == formatGif {
opt.Format = formatGif
} else {
opt.Format = formatJpeg
}
}
reqParams := make(url.Values)
reqParams.Set("response-content-type", "image/"+opt.Format)
if opt.Width == info.Width && opt.Height == info.Height && opt.Format == info.Format {
return m.presignedGetObject(ctx, name, expire, reqParams)
}
cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil {
return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
} else if !m.IsNotFound(err) {
return "", err
}
if img == nil {
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
}
defer reader.Close()
img, _, err = ImageStat(reader)
if err != nil {
return "", err
}
}
thumbnail := resizeImage(img, opt.Width, opt.Height)
buf := bytes.NewBuffer(nil)
switch opt.Format {
case formatPng:
err = png.Encode(buf, thumbnail)
case formatJpeg:
err = jpeg.Encode(buf, thumbnail, nil)
case formatGif:
err = gif.Encode(buf, thumbnail, nil)
}
if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
return "", err
}
return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
}
Loading…
Cancel
Save