msb-public
/
Open-IM-Server
mirror of https://github.com/OpenIMSDK/Open-IM-Server
2
0
Code Issues Projects Releases Wiki Activity

feat: minio cache

Browse Source
pull/1329/head
withchao 2 years ago
parent c9ab3c63f8
commit fd928b746d
5 changed files with 64 additions and 303 deletions
Show Stats Download Patch File Download Diff File

2
internal/rpc/third/third.go
Unescape View File

@ -67,7 +67,7 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
var o s3.Interface
switch config.Config.Object.Enable {
case "minio":
o, err = minio.NewMinio()
o, err = minio.NewMinio(cache.NewMinioCache(rdb))
case "cos":
o, err = cos.NewCos()
case "oss":

72
pkg/common/db/cache/s3.go vendored
Unescape View File

@ -2,12 +2,11 @@ package cache
import (
"context"
"github.com/OpenIMSDK/tools/errs"
"github.com/dtm-labs/rockscache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/redis/go-redis/v9"
"image"
"strconv"
"time"
)
@ -117,24 +116,23 @@ func (g *s3CacheRedis) GetKey(ctx context.Context, engine string, name string) (
type MinioCache interface {
metaCache
GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context, key string) (*MinioImageInfo, image.Image, error)) (*MinioImageInfo, image.Image, error)
GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*MinioImageInfo, error)) (*MinioImageInfo, error)
GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error)
//DelS3Key(engine string, keys ...string) S3Cache
DelObjectImageInfoKey(keys ...string) MinioCache
DelImageThumbnailKey(key string, format string, width int, height int) MinioCache
}
func NewMinioCache(rdb redis.UniversalClient, s3 s3.Interface) MinioCache {
func NewMinioCache(rdb redis.UniversalClient) MinioCache {
rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions())
return &minioCacheRedis{
rcClient: rcClient,
expireTime: time.Hour * 12,
s3: s3,
expireTime: time.Hour * 24 * 7,
metaCache: NewMetaCacheRedis(rcClient),
}
}
type minioCacheRedis struct {
metaCache
s3 s3.Interface
rcClient *rockscache.Client
expireTime time.Duration
}
@ -143,52 +141,44 @@ func (g *minioCacheRedis) NewCache() MinioCache {
return &minioCacheRedis{
rcClient: g.rcClient,
expireTime: g.expireTime,
s3: g.s3,
metaCache: NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...),
}
}
//func (g *minioCacheRedis) DelS3Key(engine string, keys ...string) MinioCache {
// s3cache := g.NewCache()
// ks := make([]string, 0, len(keys))
// for _, key := range keys {
// ks = append(ks, g.getS3Key(engine, key))
// }
// s3cache.AddKeys(ks...)
// return s3cache
//}
func (g *minioCacheRedis) DelObjectImageInfoKey(keys ...string) MinioCache {
s3cache := g.NewCache()
ks := make([]string, 0, len(keys))
for _, key := range keys {
ks = append(ks, g.getObjectImageInfoKey(key))
}
s3cache.AddKeys(ks...)
return s3cache
}
func (g *minioCacheRedis) DelImageThumbnailKey(key string, format string, width int, height int) MinioCache {
s3cache := g.NewCache()
s3cache.AddKeys(g.getMinioImageThumbnailKey(key, format, width, height))
return s3cache
}
func (g *minioCacheRedis) getMinioImageInfoKey(name string) string {
return "MINIO:IMAGE:" + name
func (g *minioCacheRedis) getObjectImageInfoKey(key string) string {
return "MINIO:IMAGE:" + key
}
func (g *minioCacheRedis) getMinioImageThumbnailKey(name string) string {
return "MINIO:THUMBNAIL:" + name
func (g *minioCacheRedis) getMinioImageThumbnailKey(key string, format string, width int, height int) string {
return "MINIO:THUMBNAIL:" + format + ":w" + strconv.Itoa(width) + ":h" + strconv.Itoa(height) + ":" + key
}
func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context, key string) (*MinioImageInfo, image.Image, error)) (*MinioImageInfo, image.Image, error) {
var img image.Image
info, err := getCache(ctx, g.rcClient, g.getMinioImageInfoKey(key), g.expireTime, func(ctx context.Context) (info *MinioImageInfo, err error) {
info, img, err = fn(ctx, key)
return
})
func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*MinioImageInfo, error)) (*MinioImageInfo, error) {
info, err := getCache(ctx, g.rcClient, g.getObjectImageInfoKey(key), g.expireTime, fn)
if err != nil {
return nil, nil, err
return nil, err
}
if !info.IsImg {
return nil, nil, errs.ErrData.Wrap("object not image")
}
return info, img, nil
return info, nil
}
func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context, key string, format string, width int, height int) (string, error)) (string, error) {
return getCache(ctx, g.rcClient, g.getMinioImageThumbnailKey(key), g.expireTime, func(ctx context.Context) (string, error) {
info, img, err := g.GetImageObjectKeyInfo(ctx, key, getInfo)
if err != nil {
return "", err
}
return minioCache(ctx, key, format, width, height, info, img)
})
func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error) {
return getCache(ctx, g.rcClient, g.getMinioImageThumbnailKey(key, format, width, height), g.expireTime, minioCache)
}
type MinioImageInfo struct {

126
pkg/common/db/s3/minio/minio.go
Unescape View File

@ -50,14 +50,13 @@ const (
)
const (
maxImageWidth = 1024
maxImageHeight = 1024
maxImageSize = 1024 * 1024 * 50
pathInfo = "openim/thumbnail"
maxImageInfoSize = 1024
maxImageWidth = 1024
maxImageHeight = 1024
maxImageSize = 1024 * 1024 * 50
imageThumbnailPath = "openim/thumbnail"
)
func NewMinio() (s3.Interface, error) {
func NewMinio(cache cache.MinioCache) (s3.Interface, error) {
u, err := url.Parse(config.Config.Object.Minio.Endpoint)
if err != nil {
return nil, err
@ -223,6 +222,7 @@ func (m *Minio) CompleteMultipartUpload(ctx context.Context, uploadID string, na
if err != nil {
return nil, err
}
m.delObjectImageInfoKey(ctx, name, upload.Size)
return &s3.CompleteMultipartUploadResult{
Location: upload.Location,
Bucket: upload.Bucket,
@ -385,7 +385,7 @@ func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name str
return res, nil
}
func (m *Minio) presignedGetObject(ctx context.Context, name string, expire time.Duration, query url.Values) (string, error) {
func (m *Minio) PresignedGetObject(ctx context.Context, name string, expire time.Duration, query url.Values) (string, error) {
if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second {
@ -423,110 +423,9 @@ func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration
}
}
if opt.Image == nil || (opt.Image.Width < 0 && opt.Image.Height < 0 && opt.Image.Format == "") || (opt.Image.Width > maxImageWidth || opt.Image.Height > maxImageHeight) {
return m.presignedGetObject(ctx, name, expire, reqParams)
}
return m.GetImageThumbnail(ctx, name, expire, opt.Image)
//fileInfo, err := m.StatObject(ctx, name)
//if err != nil {
// return "", err
//}
//if fileInfo.Size > maxImageSize {
// return "", errors.New("file size too large")
//}
//objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json")
//var (
// img image.Image
// info minioImageInfo
//)
//data, err := m.getObjectData(ctx, objectInfoPath, 1024)
//if err == nil {
// if err := json.Unmarshal(data, &info); err != nil {
// return "", fmt.Errorf("unmarshal minio image info.json error: %w", err)
// }
// if info.NotImage {
// return "", errors.New("not image")
// }
//} else if m.IsNotFound(err) {
// reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
// if err != nil {
// return "", err
// }
// defer reader.Close()
// imageInfo, format, err := ImageStat(reader)
// if err == nil {
// info.NotImage = false
// info.Format = format
// info.Width, info.Height = ImageWidthHeight(imageInfo)
// img = imageInfo
// } else {
// info.NotImage = true
// }
// data, err := json.Marshal(&info)
// if err != nil {
// return "", err
// }
// if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil {
// return "", err
// }
//} else {
// return "", err
//}
//if opt.Image.Width > info.Width || opt.Image.Width <= 0 {
// opt.Image.Width = info.Width
//}
//if opt.Image.Height > info.Height || opt.Image.Height <= 0 {
// opt.Image.Height = info.Height
//}
//opt.Image.Format = strings.ToLower(opt.Image.Format)
//if opt.Image.Format == formatJpg {
// opt.Image.Format = formatJpeg
//}
//switch opt.Image.Format {
//case formatPng:
//case formatJpeg:
//case formatGif:
//default:
// if info.Format == formatGif {
// opt.Image.Format = formatGif
// } else {
// opt.Image.Format = formatJpeg
// }
//}
//reqParams.Set("response-content-type", "image/"+opt.Image.Format)
//if opt.Image.Width == info.Width && opt.Image.Height == info.Height && opt.Image.Format == info.Format {
// return m.presignedGetObject(ctx, name, expire, reqParams)
//}
//cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Image.Width, opt.Image.Height, opt.Image.Format))
//if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil {
// return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
//} else if !m.IsNotFound(err) {
// return "", err
//}
//if img == nil {
// reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
// if err != nil {
// return "", err
// }
// defer reader.Close()
// img, _, err = ImageStat(reader)
// if err != nil {
// return "", err
// }
//}
//thumbnail := resizeImage(img, opt.Image.Width, opt.Image.Height)
//buf := bytes.NewBuffer(nil)
//switch opt.Image.Format {
//case formatPng:
// err = png.Encode(buf, thumbnail)
//case formatJpeg:
// err = jpeg.Encode(buf, thumbnail, nil)
//case formatGif:
// err = gif.Encode(buf, thumbnail, nil)
//}
//if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
// return "", err
//}
//return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
return m.PresignedGetObject(ctx, name, expire, reqParams)
}
return m.getImageThumbnailURL(ctx, name, expire, opt.Image)
}
func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]byte, error) {
@ -540,8 +439,3 @@ func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]
}
return io.ReadAll(io.LimitReader(object, limit))
}
func (m *Minio) GetThumbnailKey(ctx context.Context, name string) (string, error) {
return "", nil
}

22
pkg/common/db/s3/minio/struct.go
Unescape View File

@ -1,22 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package minio
type minioImageInfo struct {
NotImage bool `json:"notImage,omitempty"`
Width int `json:"width,omitempty"`
Height int `json:"height,omitempty"`
Format string `json:"format,omitempty"`
}

145
pkg/common/db/s3/minio/thumbnail.go
Unescape View File

@ -3,9 +3,10 @@ package minio
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/minio/minio-go/v7"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
@ -14,27 +15,23 @@ import (
"image/jpeg"
"image/png"
"net/url"
"path"
"path/filepath"
"strings"
"time"
)
//func (m *Minio) getHashImageInfo1(ctx context.Context, key string) (*cache.MinioImageInfo, image.Image, error) {
//
// return nil, nil, nil
//}
func (m *Minio) get1(ctx context.Context, key string, format string, width int, height int, info *cache.MinioImageInfo, img image.Image) (string, error) {
return "", nil
}
func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
info, img, err := m.cache.GetImageObjectKeyInfo(ctx, name, m.getObjectImageInfo)
func (m *Minio) getImageThumbnailURL(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
var img image.Image
info, err := m.cache.GetImageObjectKeyInfo(ctx, name, func(ctx context.Context) (info *cache.MinioImageInfo, err error) {
info, img, err = m.getObjectImageInfo(ctx, name)
return
})
if err != nil {
return "", err
}
if !info.IsImg {
return "", errs.ErrData.Wrap("object not image")
}
if opt.Width > info.Width || opt.Width <= 0 {
opt.Width = info.Width
}
@ -49,14 +46,11 @@ func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.D
case formatPng, formatJpeg, formatGif:
default:
opt.Format = ""
//if info.Format == formatGif {
// opt.Format = formatGif
//} else {
// opt.Format = formatJpeg
//}
}
reqParams := make(url.Values)
if opt.Width == info.Width && opt.Height == info.Height && (opt.Format == info.Format || opt.Format == "") {
return "", nil
reqParams.Set("response-content-type", "image/"+info.Format)
return m.PresignedGetObject(ctx, name, expire, reqParams)
}
if opt.Format == "" {
switch opt.Format {
@ -92,7 +86,7 @@ func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.D
case formatGif:
err = gif.Encode(buf, thumbnail, nil)
}
cacheKey := filepath.Join(pathInfo, info.Etag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
cacheKey := filepath.Join(imageThumbnailPath, info.Etag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
return "", err
}
@ -101,7 +95,8 @@ func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.D
if err != nil {
return "", err
}
return m.presignedGetObject(ctx, key, expire, reqParams)
reqParams.Set("response-content-type", "image/"+opt.Format)
return m.PresignedGetObject(ctx, key, expire, reqParams)
}
func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.MinioImageInfo, image.Image, error) {
@ -129,107 +124,11 @@ func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.Min
return &info, imageInfo, nil
}
func (m *Minio) GetImageThumbnail(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
fileInfo, err := m.StatObject(ctx, name)
if err != nil {
return "", err
}
if fileInfo.Size > maxImageSize {
return "", errors.New("file size too large")
}
objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json")
var (
img image.Image
info minioImageInfo
)
data, err := m.getObjectData(ctx, objectInfoPath, maxImageInfoSize)
if err == nil {
if err := json.Unmarshal(data, &info); err != nil {
return "", fmt.Errorf("unmarshal minio image info.json error: %w", err)
}
if info.NotImage {
return "", errors.New("not image")
}
} else if m.IsNotFound(err) {
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
}
defer reader.Close()
imageInfo, format, err := ImageStat(reader)
if err == nil {
info.NotImage = false
info.Format = format
info.Width, info.Height = ImageWidthHeight(imageInfo)
img = imageInfo
} else {
info.NotImage = true
}
data, err := json.Marshal(&info)
if err != nil {
return "", err
}
if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil {
return "", err
}
} else {
return "", err
func (m *Minio) delObjectImageInfoKey(ctx context.Context, key string, size int64) {
if size > 0 && size > maxImageSize {
return
}
if opt.Width > info.Width || opt.Width <= 0 {
opt.Width = info.Width
}
if opt.Height > info.Height || opt.Height <= 0 {
opt.Height = info.Height
}
opt.Format = strings.ToLower(opt.Format)
if opt.Format == formatJpg {
opt.Format = formatJpeg
}
switch opt.Format {
case formatPng:
case formatJpeg:
case formatGif:
default:
if info.Format == formatGif {
opt.Format = formatGif
} else {
opt.Format = formatJpeg
}
}
reqParams := make(url.Values)
reqParams.Set("response-content-type", "image/"+opt.Format)
if opt.Width == info.Width && opt.Height == info.Height && opt.Format == info.Format {
return m.presignedGetObject(ctx, name, expire, reqParams)
}
cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil {
return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
} else if !m.IsNotFound(err) {
return "", err
}
if img == nil {
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
}
defer reader.Close()
img, _, err = ImageStat(reader)
if err != nil {
return "", err
}
}
thumbnail := resizeImage(img, opt.Width, opt.Height)
buf := bytes.NewBuffer(nil)
switch opt.Format {
case formatPng:
err = png.Encode(buf, thumbnail)
case formatJpeg:
err = jpeg.Encode(buf, thumbnail, nil)
case formatGif:
err = gif.Encode(buf, thumbnail, nil)
}
if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
return "", err
if err := m.cache.DelObjectImageInfoKey(key).ExecDel(ctx); err != nil {
log.ZError(ctx, "DelObjectImageInfoKey failed", err, "key", key)
}
return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
}

Write Preview
Loading…
Cancel
Save