msb-public
/
Open-IM-Server
mirror of https://github.com/OpenIMSDK/Open-IM-Server
2
0
Code Issues Projects Releases Wiki Activity

feat: minio cache

Browse Source
pull/1329/head
withchao 2 years ago
parent c9ab3c63f8
commit fd928b746d
5 changed files with 64 additions and 303 deletions
Show Stats Download Patch File Download Diff File

2
internal/rpc/third/third.go
Unescape View File

@ -67,7 +67,7 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
var o s3.Interface var o s3.Interface
switch config.Config.Object.Enable { switch config.Config.Object.Enable {
case "minio": case "minio":
o, err = minio.NewMinio() o, err = minio.NewMinio(cache.NewMinioCache(rdb))
case "cos": case "cos":
o, err = cos.NewCos() o, err = cos.NewCos()
case "oss": case "oss":

72
pkg/common/db/cache/s3.go vendored
Unescape View File

@ -2,12 +2,11 @@ package cache
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/errs"
"github.com/dtm-labs/rockscache" "github.com/dtm-labs/rockscache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
"image" "strconv"
"time" "time"
) )
@ -117,24 +116,23 @@ func (g *s3CacheRedis) GetKey(ctx context.Context, engine string, name string) (
type MinioCache interface { type MinioCache interface {
metaCache metaCache
GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context, key string) (*MinioImageInfo, image.Image, error)) (*MinioImageInfo, image.Image, error) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*MinioImageInfo, error)) (*MinioImageInfo, error)
GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error)
//DelS3Key(engine string, keys ...string) S3Cache DelObjectImageInfoKey(keys ...string) MinioCache
DelImageThumbnailKey(key string, format string, width int, height int) MinioCache
} }
func NewMinioCache(rdb redis.UniversalClient, s3 s3.Interface) MinioCache { func NewMinioCache(rdb redis.UniversalClient) MinioCache {
rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions()) rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions())
return &minioCacheRedis{ return &minioCacheRedis{
rcClient: rcClient, rcClient: rcClient,
expireTime: time.Hour * 12, expireTime: time.Hour * 24 * 7,
s3: s3,
metaCache: NewMetaCacheRedis(rcClient), metaCache: NewMetaCacheRedis(rcClient),
} }
} }
type minioCacheRedis struct { type minioCacheRedis struct {
metaCache metaCache
s3 s3.Interface
rcClient *rockscache.Client rcClient *rockscache.Client
expireTime time.Duration expireTime time.Duration
} }
@ -143,52 +141,44 @@ func (g *minioCacheRedis) NewCache() MinioCache {
return &minioCacheRedis{ return &minioCacheRedis{
rcClient: g.rcClient, rcClient: g.rcClient,
expireTime: g.expireTime, expireTime: g.expireTime,
s3: g.s3,
metaCache: NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...), metaCache: NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...),
} }
} }
//func (g *minioCacheRedis) DelS3Key(engine string, keys ...string) MinioCache { func (g *minioCacheRedis) DelObjectImageInfoKey(keys ...string) MinioCache {
// s3cache := g.NewCache() s3cache := g.NewCache()
// ks := make([]string, 0, len(keys)) ks := make([]string, 0, len(keys))
// for _, key := range keys { for _, key := range keys {
// ks = append(ks, g.getS3Key(engine, key)) ks = append(ks, g.getObjectImageInfoKey(key))
// }
// s3cache.AddKeys(ks...)
// return s3cache
//}
func (g *minioCacheRedis) getMinioImageInfoKey(name string) string {
return "MINIO:IMAGE:" + name
} }
s3cache.AddKeys(ks...)
func (g *minioCacheRedis) getMinioImageThumbnailKey(name string) string { return s3cache
return "MINIO:THUMBNAIL:" + name
} }
func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context, key string) (*MinioImageInfo, image.Image, error)) (*MinioImageInfo, image.Image, error) { func (g *minioCacheRedis) DelImageThumbnailKey(key string, format string, width int, height int) MinioCache {
var img image.Image s3cache := g.NewCache()
info, err := getCache(ctx, g.rcClient, g.getMinioImageInfoKey(key), g.expireTime, func(ctx context.Context) (info *MinioImageInfo, err error) { s3cache.AddKeys(g.getMinioImageThumbnailKey(key, format, width, height))
info, img, err = fn(ctx, key) return s3cache
return
})
if err != nil {
return nil, nil, err
} }
if !info.IsImg {
return nil, nil, errs.ErrData.Wrap("object not image") func (g *minioCacheRedis) getObjectImageInfoKey(key string) string {
return "MINIO:IMAGE:" + key
} }
return info, img, nil
func (g *minioCacheRedis) getMinioImageThumbnailKey(key string, format string, width int, height int) string {
return "MINIO:THUMBNAIL:" + format + ":w" + strconv.Itoa(width) + ":h" + strconv.Itoa(height) + ":" + key
} }
func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context, key string, format string, width int, height int) (string, error)) (string, error) { func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*MinioImageInfo, error)) (*MinioImageInfo, error) {
return getCache(ctx, g.rcClient, g.getMinioImageThumbnailKey(key), g.expireTime, func(ctx context.Context) (string, error) { info, err := getCache(ctx, g.rcClient, g.getObjectImageInfoKey(key), g.expireTime, fn)
info, img, err := g.GetImageObjectKeyInfo(ctx, key, getInfo)
if err != nil { if err != nil {
return "", err return nil, err
} }
return minioCache(ctx, key, format, width, height, info, img) return info, nil
}) }
func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error) {
return getCache(ctx, g.rcClient, g.getMinioImageThumbnailKey(key, format, width, height), g.expireTime, minioCache)
} }
type MinioImageInfo struct { type MinioImageInfo struct {

120
pkg/common/db/s3/minio/minio.go
Unescape View File

@ -53,11 +53,10 @@ const (
maxImageWidth = 1024 maxImageWidth = 1024
maxImageHeight = 1024 maxImageHeight = 1024
maxImageSize = 1024 * 1024 * 50 maxImageSize = 1024 * 1024 * 50
pathInfo = "openim/thumbnail" imageThumbnailPath = "openim/thumbnail"
maxImageInfoSize = 1024
) )
func NewMinio() (s3.Interface, error) { func NewMinio(cache cache.MinioCache) (s3.Interface, error) {
u, err := url.Parse(config.Config.Object.Minio.Endpoint) u, err := url.Parse(config.Config.Object.Minio.Endpoint)
if err != nil { if err != nil {
return nil, err return nil, err
@ -223,6 +222,7 @@ func (m *Minio) CompleteMultipartUpload(ctx context.Context, uploadID string, na
if err != nil { if err != nil {
return nil, err return nil, err
} }
m.delObjectImageInfoKey(ctx, name, upload.Size)
return &s3.CompleteMultipartUploadResult{ return &s3.CompleteMultipartUploadResult{
Location: upload.Location, Location: upload.Location,
Bucket: upload.Bucket, Bucket: upload.Bucket,
@ -385,7 +385,7 @@ func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name str
return res, nil return res, nil
} }
func (m *Minio) presignedGetObject(ctx context.Context, name string, expire time.Duration, query url.Values) (string, error) { func (m *Minio) PresignedGetObject(ctx context.Context, name string, expire time.Duration, query url.Values) (string, error) {
if expire <= 0 { if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second { } else if expire < time.Second {
@ -423,110 +423,9 @@ func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration
} }
} }
if opt.Image == nil || (opt.Image.Width < 0 && opt.Image.Height < 0 && opt.Image.Format == "") || (opt.Image.Width > maxImageWidth || opt.Image.Height > maxImageHeight) { if opt.Image == nil || (opt.Image.Width < 0 && opt.Image.Height < 0 && opt.Image.Format == "") || (opt.Image.Width > maxImageWidth || opt.Image.Height > maxImageHeight) {
return m.presignedGetObject(ctx, name, expire, reqParams) return m.PresignedGetObject(ctx, name, expire, reqParams)
} }
return m.GetImageThumbnail(ctx, name, expire, opt.Image) return m.getImageThumbnailURL(ctx, name, expire, opt.Image)
//fileInfo, err := m.StatObject(ctx, name)
//if err != nil {
// return "", err
//}
//if fileInfo.Size > maxImageSize {
// return "", errors.New("file size too large")
//}
//objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json")
//var (
// img image.Image
// info minioImageInfo
//)
//data, err := m.getObjectData(ctx, objectInfoPath, 1024)
//if err == nil {
// if err := json.Unmarshal(data, &info); err != nil {
// return "", fmt.Errorf("unmarshal minio image info.json error: %w", err)
// }
// if info.NotImage {
// return "", errors.New("not image")
// }
//} else if m.IsNotFound(err) {
// reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
// if err != nil {
// return "", err
// }
// defer reader.Close()
// imageInfo, format, err := ImageStat(reader)
// if err == nil {
// info.NotImage = false
// info.Format = format
// info.Width, info.Height = ImageWidthHeight(imageInfo)
// img = imageInfo
// } else {
// info.NotImage = true
// }
// data, err := json.Marshal(&info)
// if err != nil {
// return "", err
// }
// if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil {
// return "", err
// }
//} else {
// return "", err
//}
//if opt.Image.Width > info.Width || opt.Image.Width <= 0 {
// opt.Image.Width = info.Width
//}
//if opt.Image.Height > info.Height || opt.Image.Height <= 0 {
// opt.Image.Height = info.Height
//}
//opt.Image.Format = strings.ToLower(opt.Image.Format)
//if opt.Image.Format == formatJpg {
// opt.Image.Format = formatJpeg
//}
//switch opt.Image.Format {
//case formatPng:
//case formatJpeg:
//case formatGif:
//default:
// if info.Format == formatGif {
// opt.Image.Format = formatGif
// } else {
// opt.Image.Format = formatJpeg
// }
//}
//reqParams.Set("response-content-type", "image/"+opt.Image.Format)
//if opt.Image.Width == info.Width && opt.Image.Height == info.Height && opt.Image.Format == info.Format {
// return m.presignedGetObject(ctx, name, expire, reqParams)
//}
//cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Image.Width, opt.Image.Height, opt.Image.Format))
//if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil {
// return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
//} else if !m.IsNotFound(err) {
// return "", err
//}
//if img == nil {
// reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
// if err != nil {
// return "", err
// }
// defer reader.Close()
// img, _, err = ImageStat(reader)
// if err != nil {
// return "", err
// }
//}
//thumbnail := resizeImage(img, opt.Image.Width, opt.Image.Height)
//buf := bytes.NewBuffer(nil)
//switch opt.Image.Format {
//case formatPng:
// err = png.Encode(buf, thumbnail)
//case formatJpeg:
// err = jpeg.Encode(buf, thumbnail, nil)
//case formatGif:
// err = gif.Encode(buf, thumbnail, nil)
//}
//if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
// return "", err
//}
//return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
} }
func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]byte, error) { func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]byte, error) {
@ -540,8 +439,3 @@ func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]
} }
return io.ReadAll(io.LimitReader(object, limit)) return io.ReadAll(io.LimitReader(object, limit))
} }
func (m *Minio) GetThumbnailKey(ctx context.Context, name string) (string, error) {
return "", nil
}

22
pkg/common/db/s3/minio/struct.go
Unescape View File

@ -1,22 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package minio
type minioImageInfo struct {
NotImage bool `json:"notImage,omitempty"`
Width int `json:"width,omitempty"`
Height int `json:"height,omitempty"`
Format string `json:"format,omitempty"`
}

145
pkg/common/db/s3/minio/thumbnail.go
Unescape View File

@ -3,9 +3,10 @@ package minio
import ( import (
"bytes" "bytes"
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
@ -14,27 +15,23 @@ import (
"image/jpeg" "image/jpeg"
"image/png" "image/png"
"net/url" "net/url"
"path"
"path/filepath" "path/filepath"
"strings" "strings"
"time" "time"
) )
//func (m *Minio) getHashImageInfo1(ctx context.Context, key string) (*cache.MinioImageInfo, image.Image, error) { func (m *Minio) getImageThumbnailURL(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
// var img image.Image
// return nil, nil, nil info, err := m.cache.GetImageObjectKeyInfo(ctx, name, func(ctx context.Context) (info *cache.MinioImageInfo, err error) {
//} info, img, err = m.getObjectImageInfo(ctx, name)
return
func (m *Minio) get1(ctx context.Context, key string, format string, width int, height int, info *cache.MinioImageInfo, img image.Image) (string, error) { })
return "", nil
}
func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
info, img, err := m.cache.GetImageObjectKeyInfo(ctx, name, m.getObjectImageInfo)
if err != nil { if err != nil {
return "", err return "", err
} }
if !info.IsImg {
return "", errs.ErrData.Wrap("object not image")
}
if opt.Width > info.Width || opt.Width <= 0 { if opt.Width > info.Width || opt.Width <= 0 {
opt.Width = info.Width opt.Width = info.Width
} }
@ -49,14 +46,11 @@ func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.D
case formatPng, formatJpeg, formatGif: case formatPng, formatJpeg, formatGif:
default: default:
opt.Format = "" opt.Format = ""
//if info.Format == formatGif {
// opt.Format = formatGif
//} else {
// opt.Format = formatJpeg
//}
} }
reqParams := make(url.Values)
if opt.Width == info.Width && opt.Height == info.Height && (opt.Format == info.Format || opt.Format == "") { if opt.Width == info.Width && opt.Height == info.Height && (opt.Format == info.Format || opt.Format == "") {
return "", nil reqParams.Set("response-content-type", "image/"+info.Format)
return m.PresignedGetObject(ctx, name, expire, reqParams)
} }
if opt.Format == "" { if opt.Format == "" {
switch opt.Format { switch opt.Format {
@ -92,7 +86,7 @@ func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.D
case formatGif: case formatGif:
err = gif.Encode(buf, thumbnail, nil) err = gif.Encode(buf, thumbnail, nil)
} }
cacheKey := filepath.Join(pathInfo, info.Etag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format)) cacheKey := filepath.Join(imageThumbnailPath, info.Etag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil { if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
return "", err return "", err
} }
@ -101,7 +95,8 @@ func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.D
if err != nil { if err != nil {
return "", err return "", err
} }
return m.presignedGetObject(ctx, key, expire, reqParams) reqParams.Set("response-content-type", "image/"+opt.Format)
return m.PresignedGetObject(ctx, key, expire, reqParams)
} }
func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.MinioImageInfo, image.Image, error) { func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.MinioImageInfo, image.Image, error) {
@ -129,107 +124,11 @@ func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.Min
return &info, imageInfo, nil return &info, imageInfo, nil
} }
func (m *Minio) GetImageThumbnail(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) { func (m *Minio) delObjectImageInfoKey(ctx context.Context, key string, size int64) {
fileInfo, err := m.StatObject(ctx, name) if size > 0 && size > maxImageSize {
if err != nil { return
return "", err
}
if fileInfo.Size > maxImageSize {
return "", errors.New("file size too large")
}
objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json")
var (
img image.Image
info minioImageInfo
)
data, err := m.getObjectData(ctx, objectInfoPath, maxImageInfoSize)
if err == nil {
if err := json.Unmarshal(data, &info); err != nil {
return "", fmt.Errorf("unmarshal minio image info.json error: %w", err)
}
if info.NotImage {
return "", errors.New("not image")
}
} else if m.IsNotFound(err) {
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
} }
defer reader.Close() if err := m.cache.DelObjectImageInfoKey(key).ExecDel(ctx); err != nil {
imageInfo, format, err := ImageStat(reader) log.ZError(ctx, "DelObjectImageInfoKey failed", err, "key", key)
if err == nil {
info.NotImage = false
info.Format = format
info.Width, info.Height = ImageWidthHeight(imageInfo)
img = imageInfo
} else {
info.NotImage = true
}
data, err := json.Marshal(&info)
if err != nil {
return "", err
}
if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil {
return "", err
}
} else {
return "", err
}
if opt.Width > info.Width || opt.Width <= 0 {
opt.Width = info.Width
}
if opt.Height > info.Height || opt.Height <= 0 {
opt.Height = info.Height
}
opt.Format = strings.ToLower(opt.Format)
if opt.Format == formatJpg {
opt.Format = formatJpeg
}
switch opt.Format {
case formatPng:
case formatJpeg:
case formatGif:
default:
if info.Format == formatGif {
opt.Format = formatGif
} else {
opt.Format = formatJpeg
}
}
reqParams := make(url.Values)
reqParams.Set("response-content-type", "image/"+opt.Format)
if opt.Width == info.Width && opt.Height == info.Height && opt.Format == info.Format {
return m.presignedGetObject(ctx, name, expire, reqParams)
}
cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil {
return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
} else if !m.IsNotFound(err) {
return "", err
}
if img == nil {
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
}
defer reader.Close()
img, _, err = ImageStat(reader)
if err != nil {
return "", err
}
}
thumbnail := resizeImage(img, opt.Width, opt.Height)
buf := bytes.NewBuffer(nil)
switch opt.Format {
case formatPng:
err = png.Encode(buf, thumbnail)
case formatJpeg:
err = jpeg.Encode(buf, thumbnail, nil)
case formatGif:
err = gif.Encode(buf, thumbnail, nil)
}
if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
return "", err
} }
return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
} }

Write Preview
Loading…
Cancel
Save