From fd928b746dd70be174bef8b8c84f6dea15adf009 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 2 Nov 2023 15:14:42 +0800 Subject: [PATCH] feat: minio cache --- internal/rpc/third/third.go | 2 +- pkg/common/db/cache/s3.go | 72 ++++++-------- pkg/common/db/s3/minio/minio.go | 126 ++---------------------- pkg/common/db/s3/minio/struct.go | 22 ----- pkg/common/db/s3/minio/thumbnail.go | 145 +++++----------------------- 5 files changed, 64 insertions(+), 303 deletions(-) delete mode 100644 pkg/common/db/s3/minio/struct.go diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index c91c7a1c6..a350c6915 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -67,7 +67,7 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e var o s3.Interface switch config.Config.Object.Enable { case "minio": - o, err = minio.NewMinio() + o, err = minio.NewMinio(cache.NewMinioCache(rdb)) case "cos": o, err = cos.NewCos() case "oss": diff --git a/pkg/common/db/cache/s3.go b/pkg/common/db/cache/s3.go index cfa8ec9ba..f8f8894b6 100644 --- a/pkg/common/db/cache/s3.go +++ b/pkg/common/db/cache/s3.go @@ -2,12 +2,11 @@ package cache import ( "context" - "github.com/OpenIMSDK/tools/errs" "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "github.com/redis/go-redis/v9" - "image" + "strconv" "time" ) @@ -117,24 +116,23 @@ func (g *s3CacheRedis) GetKey(ctx context.Context, engine string, name string) ( type MinioCache interface { metaCache - GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context, key string) (*MinioImageInfo, image.Image, error)) (*MinioImageInfo, image.Image, error) + GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*MinioImageInfo, error)) (*MinioImageInfo, error) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error) - //DelS3Key(engine string, keys ...string) S3Cache + DelObjectImageInfoKey(keys ...string) MinioCache + DelImageThumbnailKey(key string, format string, width int, height int) MinioCache } -func NewMinioCache(rdb redis.UniversalClient, s3 s3.Interface) MinioCache { +func NewMinioCache(rdb redis.UniversalClient) MinioCache { rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions()) return &minioCacheRedis{ rcClient: rcClient, - expireTime: time.Hour * 12, - s3: s3, + expireTime: time.Hour * 24 * 7, metaCache: NewMetaCacheRedis(rcClient), } } type minioCacheRedis struct { metaCache - s3 s3.Interface rcClient *rockscache.Client expireTime time.Duration } @@ -143,52 +141,44 @@ func (g *minioCacheRedis) NewCache() MinioCache { return &minioCacheRedis{ rcClient: g.rcClient, expireTime: g.expireTime, - s3: g.s3, metaCache: NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...), } } -//func (g *minioCacheRedis) DelS3Key(engine string, keys ...string) MinioCache { -// s3cache := g.NewCache() -// ks := make([]string, 0, len(keys)) -// for _, key := range keys { -// ks = append(ks, g.getS3Key(engine, key)) -// } -// s3cache.AddKeys(ks...) -// return s3cache -//} +func (g *minioCacheRedis) DelObjectImageInfoKey(keys ...string) MinioCache { + s3cache := g.NewCache() + ks := make([]string, 0, len(keys)) + for _, key := range keys { + ks = append(ks, g.getObjectImageInfoKey(key)) + } + s3cache.AddKeys(ks...) + return s3cache +} + +func (g *minioCacheRedis) DelImageThumbnailKey(key string, format string, width int, height int) MinioCache { + s3cache := g.NewCache() + s3cache.AddKeys(g.getMinioImageThumbnailKey(key, format, width, height)) + return s3cache +} -func (g *minioCacheRedis) getMinioImageInfoKey(name string) string { - return "MINIO:IMAGE:" + name +func (g *minioCacheRedis) getObjectImageInfoKey(key string) string { + return "MINIO:IMAGE:" + key } -func (g *minioCacheRedis) getMinioImageThumbnailKey(name string) string { - return "MINIO:THUMBNAIL:" + name +func (g *minioCacheRedis) getMinioImageThumbnailKey(key string, format string, width int, height int) string { + return "MINIO:THUMBNAIL:" + format + ":w" + strconv.Itoa(width) + ":h" + strconv.Itoa(height) + ":" + key } -func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context, key string) (*MinioImageInfo, image.Image, error)) (*MinioImageInfo, image.Image, error) { - var img image.Image - info, err := getCache(ctx, g.rcClient, g.getMinioImageInfoKey(key), g.expireTime, func(ctx context.Context) (info *MinioImageInfo, err error) { - info, img, err = fn(ctx, key) - return - }) +func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*MinioImageInfo, error)) (*MinioImageInfo, error) { + info, err := getCache(ctx, g.rcClient, g.getObjectImageInfoKey(key), g.expireTime, fn) if err != nil { - return nil, nil, err + return nil, err } - if !info.IsImg { - return nil, nil, errs.ErrData.Wrap("object not image") - } - return info, img, nil + return info, nil } -func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context, key string, format string, width int, height int) (string, error)) (string, error) { - return getCache(ctx, g.rcClient, g.getMinioImageThumbnailKey(key), g.expireTime, func(ctx context.Context) (string, error) { - info, img, err := g.GetImageObjectKeyInfo(ctx, key, getInfo) - if err != nil { - return "", err - } - return minioCache(ctx, key, format, width, height, info, img) - }) +func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error) { + return getCache(ctx, g.rcClient, g.getMinioImageThumbnailKey(key, format, width, height), g.expireTime, minioCache) } type MinioImageInfo struct { diff --git a/pkg/common/db/s3/minio/minio.go b/pkg/common/db/s3/minio/minio.go index 4f018d7e4..ab9f55372 100644 --- a/pkg/common/db/s3/minio/minio.go +++ b/pkg/common/db/s3/minio/minio.go @@ -50,14 +50,13 @@ const ( ) const ( - maxImageWidth = 1024 - maxImageHeight = 1024 - maxImageSize = 1024 * 1024 * 50 - pathInfo = "openim/thumbnail" - maxImageInfoSize = 1024 + maxImageWidth = 1024 + maxImageHeight = 1024 + maxImageSize = 1024 * 1024 * 50 + imageThumbnailPath = "openim/thumbnail" ) -func NewMinio() (s3.Interface, error) { +func NewMinio(cache cache.MinioCache) (s3.Interface, error) { u, err := url.Parse(config.Config.Object.Minio.Endpoint) if err != nil { return nil, err @@ -223,6 +222,7 @@ func (m *Minio) CompleteMultipartUpload(ctx context.Context, uploadID string, na if err != nil { return nil, err } + m.delObjectImageInfoKey(ctx, name, upload.Size) return &s3.CompleteMultipartUploadResult{ Location: upload.Location, Bucket: upload.Bucket, @@ -385,7 +385,7 @@ func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name str return res, nil } -func (m *Minio) presignedGetObject(ctx context.Context, name string, expire time.Duration, query url.Values) (string, error) { +func (m *Minio) PresignedGetObject(ctx context.Context, name string, expire time.Duration, query url.Values) (string, error) { if expire <= 0 { expire = time.Hour * 24 * 365 * 99 // 99 years } else if expire < time.Second { @@ -423,110 +423,9 @@ func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration } } if opt.Image == nil || (opt.Image.Width < 0 && opt.Image.Height < 0 && opt.Image.Format == "") || (opt.Image.Width > maxImageWidth || opt.Image.Height > maxImageHeight) { - return m.presignedGetObject(ctx, name, expire, reqParams) - } - return m.GetImageThumbnail(ctx, name, expire, opt.Image) - //fileInfo, err := m.StatObject(ctx, name) - //if err != nil { - // return "", err - //} - //if fileInfo.Size > maxImageSize { - // return "", errors.New("file size too large") - //} - //objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json") - //var ( - // img image.Image - // info minioImageInfo - //) - //data, err := m.getObjectData(ctx, objectInfoPath, 1024) - //if err == nil { - // if err := json.Unmarshal(data, &info); err != nil { - // return "", fmt.Errorf("unmarshal minio image info.json error: %w", err) - // } - // if info.NotImage { - // return "", errors.New("not image") - // } - //} else if m.IsNotFound(err) { - // reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{}) - // if err != nil { - // return "", err - // } - // defer reader.Close() - // imageInfo, format, err := ImageStat(reader) - // if err == nil { - // info.NotImage = false - // info.Format = format - // info.Width, info.Height = ImageWidthHeight(imageInfo) - // img = imageInfo - // } else { - // info.NotImage = true - // } - // data, err := json.Marshal(&info) - // if err != nil { - // return "", err - // } - // if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil { - // return "", err - // } - //} else { - // return "", err - //} - //if opt.Image.Width > info.Width || opt.Image.Width <= 0 { - // opt.Image.Width = info.Width - //} - //if opt.Image.Height > info.Height || opt.Image.Height <= 0 { - // opt.Image.Height = info.Height - //} - //opt.Image.Format = strings.ToLower(opt.Image.Format) - //if opt.Image.Format == formatJpg { - // opt.Image.Format = formatJpeg - //} - //switch opt.Image.Format { - //case formatPng: - //case formatJpeg: - //case formatGif: - //default: - // if info.Format == formatGif { - // opt.Image.Format = formatGif - // } else { - // opt.Image.Format = formatJpeg - // } - //} - //reqParams.Set("response-content-type", "image/"+opt.Image.Format) - //if opt.Image.Width == info.Width && opt.Image.Height == info.Height && opt.Image.Format == info.Format { - // return m.presignedGetObject(ctx, name, expire, reqParams) - //} - //cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Image.Width, opt.Image.Height, opt.Image.Format)) - //if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil { - // return m.presignedGetObject(ctx, cacheKey, expire, reqParams) - //} else if !m.IsNotFound(err) { - // return "", err - //} - //if img == nil { - // reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{}) - // if err != nil { - // return "", err - // } - // defer reader.Close() - // img, _, err = ImageStat(reader) - // if err != nil { - // return "", err - // } - //} - //thumbnail := resizeImage(img, opt.Image.Width, opt.Image.Height) - //buf := bytes.NewBuffer(nil) - //switch opt.Image.Format { - //case formatPng: - // err = png.Encode(buf, thumbnail) - //case formatJpeg: - // err = jpeg.Encode(buf, thumbnail, nil) - //case formatGif: - // err = gif.Encode(buf, thumbnail, nil) - //} - //if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil { - // return "", err - //} - //return m.presignedGetObject(ctx, cacheKey, expire, reqParams) + return m.PresignedGetObject(ctx, name, expire, reqParams) + } + return m.getImageThumbnailURL(ctx, name, expire, opt.Image) } func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]byte, error) { @@ -540,8 +439,3 @@ func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([] } return io.ReadAll(io.LimitReader(object, limit)) } - -func (m *Minio) GetThumbnailKey(ctx context.Context, name string) (string, error) { - - return "", nil -} diff --git a/pkg/common/db/s3/minio/struct.go b/pkg/common/db/s3/minio/struct.go deleted file mode 100644 index 28b8bfdc3..000000000 --- a/pkg/common/db/s3/minio/struct.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package minio - -type minioImageInfo struct { - NotImage bool `json:"notImage,omitempty"` - Width int `json:"width,omitempty"` - Height int `json:"height,omitempty"` - Format string `json:"format,omitempty"` -} diff --git a/pkg/common/db/s3/minio/thumbnail.go b/pkg/common/db/s3/minio/thumbnail.go index 6a683ac61..fa3581572 100644 --- a/pkg/common/db/s3/minio/thumbnail.go +++ b/pkg/common/db/s3/minio/thumbnail.go @@ -3,9 +3,10 @@ package minio import ( "bytes" "context" - "encoding/json" "errors" "fmt" + "github.com/OpenIMSDK/tools/errs" + "github.com/OpenIMSDK/tools/log" "github.com/minio/minio-go/v7" "github.com/openimsdk/open-im-server/v3/pkg/common/db/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3" @@ -14,27 +15,23 @@ import ( "image/jpeg" "image/png" "net/url" - "path" "path/filepath" "strings" "time" ) -//func (m *Minio) getHashImageInfo1(ctx context.Context, key string) (*cache.MinioImageInfo, image.Image, error) { -// -// return nil, nil, nil -//} - -func (m *Minio) get1(ctx context.Context, key string, format string, width int, height int, info *cache.MinioImageInfo, img image.Image) (string, error) { - - return "", nil -} - -func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) { - info, img, err := m.cache.GetImageObjectKeyInfo(ctx, name, m.getObjectImageInfo) +func (m *Minio) getImageThumbnailURL(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) { + var img image.Image + info, err := m.cache.GetImageObjectKeyInfo(ctx, name, func(ctx context.Context) (info *cache.MinioImageInfo, err error) { + info, img, err = m.getObjectImageInfo(ctx, name) + return + }) if err != nil { return "", err } + if !info.IsImg { + return "", errs.ErrData.Wrap("object not image") + } if opt.Width > info.Width || opt.Width <= 0 { opt.Width = info.Width } @@ -49,14 +46,11 @@ func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.D case formatPng, formatJpeg, formatGif: default: opt.Format = "" - //if info.Format == formatGif { - // opt.Format = formatGif - //} else { - // opt.Format = formatJpeg - //} } + reqParams := make(url.Values) if opt.Width == info.Width && opt.Height == info.Height && (opt.Format == info.Format || opt.Format == "") { - return "", nil + reqParams.Set("response-content-type", "image/"+info.Format) + return m.PresignedGetObject(ctx, name, expire, reqParams) } if opt.Format == "" { switch opt.Format { @@ -92,7 +86,7 @@ func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.D case formatGif: err = gif.Encode(buf, thumbnail, nil) } - cacheKey := filepath.Join(pathInfo, info.Etag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format)) + cacheKey := filepath.Join(imageThumbnailPath, info.Etag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format)) if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil { return "", err } @@ -101,7 +95,8 @@ func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.D if err != nil { return "", err } - return m.presignedGetObject(ctx, key, expire, reqParams) + reqParams.Set("response-content-type", "image/"+opt.Format) + return m.PresignedGetObject(ctx, key, expire, reqParams) } func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.MinioImageInfo, image.Image, error) { @@ -129,107 +124,11 @@ func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.Min return &info, imageInfo, nil } -func (m *Minio) GetImageThumbnail(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) { - fileInfo, err := m.StatObject(ctx, name) - if err != nil { - return "", err - } - if fileInfo.Size > maxImageSize { - return "", errors.New("file size too large") - } - objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json") - var ( - img image.Image - info minioImageInfo - ) - data, err := m.getObjectData(ctx, objectInfoPath, maxImageInfoSize) - if err == nil { - if err := json.Unmarshal(data, &info); err != nil { - return "", fmt.Errorf("unmarshal minio image info.json error: %w", err) - } - if info.NotImage { - return "", errors.New("not image") - } - } else if m.IsNotFound(err) { - reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{}) - if err != nil { - return "", err - } - defer reader.Close() - imageInfo, format, err := ImageStat(reader) - if err == nil { - info.NotImage = false - info.Format = format - info.Width, info.Height = ImageWidthHeight(imageInfo) - img = imageInfo - } else { - info.NotImage = true - } - data, err := json.Marshal(&info) - if err != nil { - return "", err - } - if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil { - return "", err - } - } else { - return "", err +func (m *Minio) delObjectImageInfoKey(ctx context.Context, key string, size int64) { + if size > 0 && size > maxImageSize { + return } - if opt.Width > info.Width || opt.Width <= 0 { - opt.Width = info.Width - } - if opt.Height > info.Height || opt.Height <= 0 { - opt.Height = info.Height - } - opt.Format = strings.ToLower(opt.Format) - if opt.Format == formatJpg { - opt.Format = formatJpeg - } - switch opt.Format { - case formatPng: - case formatJpeg: - case formatGif: - default: - if info.Format == formatGif { - opt.Format = formatGif - } else { - opt.Format = formatJpeg - } - } - reqParams := make(url.Values) - reqParams.Set("response-content-type", "image/"+opt.Format) - if opt.Width == info.Width && opt.Height == info.Height && opt.Format == info.Format { - return m.presignedGetObject(ctx, name, expire, reqParams) - } - cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format)) - if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil { - return m.presignedGetObject(ctx, cacheKey, expire, reqParams) - } else if !m.IsNotFound(err) { - return "", err - } - if img == nil { - reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{}) - if err != nil { - return "", err - } - defer reader.Close() - img, _, err = ImageStat(reader) - if err != nil { - return "", err - } - } - thumbnail := resizeImage(img, opt.Width, opt.Height) - buf := bytes.NewBuffer(nil) - switch opt.Format { - case formatPng: - err = png.Encode(buf, thumbnail) - case formatJpeg: - err = jpeg.Encode(buf, thumbnail, nil) - case formatGif: - err = gif.Encode(buf, thumbnail, nil) - } - if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil { - return "", err + if err := m.cache.DelObjectImageInfoKey(key).ExecDel(ctx); err != nil { + log.ZError(ctx, "DelObjectImageInfoKey failed", err, "key", key) } - return m.presignedGetObject(ctx, cacheKey, expire, reqParams) }