feat: S3 server cache (#1329)
* optimize scheduled deletion * optimize scheduled deletion * optimize scheduled deletion * optimize scheduled deletion * minio cache * fix: conflicts * feat: minio cache * feat: cache optimize * feat: cache optimize * feat: cache optimize * feat: cache optimize * feat: cache optimizepull/1333/head
chao
8 months ago
committed by
GitHub
parent
62e9980f3c
commit
cb0bf64435
@ -0,0 +1,190 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/dtm-labs/rockscache"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
|
||||
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ObjectCache interface {
|
||||
metaCache
|
||||
GetName(ctx context.Context, name string) (*relationtb.ObjectModel, error)
|
||||
DelObjectName(names ...string) ObjectCache
|
||||
}
|
||||
|
||||
func NewObjectCacheRedis(rdb redis.UniversalClient, objDB relationtb.ObjectInfoModelInterface) ObjectCache {
|
||||
rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions())
|
||||
return &objectCacheRedis{
|
||||
rcClient: rcClient,
|
||||
expireTime: time.Hour * 12,
|
||||
objDB: objDB,
|
||||
metaCache: NewMetaCacheRedis(rcClient),
|
||||
}
|
||||
}
|
||||
|
||||
type objectCacheRedis struct {
|
||||
metaCache
|
||||
objDB relationtb.ObjectInfoModelInterface
|
||||
rcClient *rockscache.Client
|
||||
expireTime time.Duration
|
||||
}
|
||||
|
||||
func (g *objectCacheRedis) NewCache() ObjectCache {
|
||||
return &objectCacheRedis{
|
||||
rcClient: g.rcClient,
|
||||
expireTime: g.expireTime,
|
||||
objDB: g.objDB,
|
||||
metaCache: NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...),
|
||||
}
|
||||
}
|
||||
|
||||
func (g *objectCacheRedis) DelObjectName(names ...string) ObjectCache {
|
||||
objectCache := g.NewCache()
|
||||
keys := make([]string, 0, len(names))
|
||||
for _, name := range names {
|
||||
keys = append(keys, g.getObjectKey(name))
|
||||
}
|
||||
objectCache.AddKeys(keys...)
|
||||
return objectCache
|
||||
}
|
||||
|
||||
func (g *objectCacheRedis) getObjectKey(name string) string {
|
||||
return "OBJECT:" + name
|
||||
}
|
||||
|
||||
func (g *objectCacheRedis) GetName(ctx context.Context, name string) (*relationtb.ObjectModel, error) {
|
||||
return getCache(ctx, g.rcClient, g.getObjectKey(name), g.expireTime, func(ctx context.Context) (*relationtb.ObjectModel, error) {
|
||||
return g.objDB.Take(ctx, name)
|
||||
})
|
||||
}
|
||||
|
||||
type S3Cache interface {
|
||||
metaCache
|
||||
GetKey(ctx context.Context, engine string, key string) (*s3.ObjectInfo, error)
|
||||
DelS3Key(engine string, keys ...string) S3Cache
|
||||
}
|
||||
|
||||
func NewS3Cache(rdb redis.UniversalClient, s3 s3.Interface) S3Cache {
|
||||
rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions())
|
||||
return &s3CacheRedis{
|
||||
rcClient: rcClient,
|
||||
expireTime: time.Hour * 12,
|
||||
s3: s3,
|
||||
metaCache: NewMetaCacheRedis(rcClient),
|
||||
}
|
||||
}
|
||||
|
||||
type s3CacheRedis struct {
|
||||
metaCache
|
||||
s3 s3.Interface
|
||||
rcClient *rockscache.Client
|
||||
expireTime time.Duration
|
||||
}
|
||||
|
||||
func (g *s3CacheRedis) NewCache() S3Cache {
|
||||
return &s3CacheRedis{
|
||||
rcClient: g.rcClient,
|
||||
expireTime: g.expireTime,
|
||||
s3: g.s3,
|
||||
metaCache: NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...),
|
||||
}
|
||||
}
|
||||
|
||||
func (g *s3CacheRedis) DelS3Key(engine string, keys ...string) S3Cache {
|
||||
s3cache := g.NewCache()
|
||||
ks := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
ks = append(ks, g.getS3Key(engine, key))
|
||||
}
|
||||
s3cache.AddKeys(ks...)
|
||||
return s3cache
|
||||
}
|
||||
|
||||
func (g *s3CacheRedis) getS3Key(engine string, name string) string {
|
||||
return "S3:" + engine + ":" + name
|
||||
}
|
||||
|
||||
func (g *s3CacheRedis) GetKey(ctx context.Context, engine string, name string) (*s3.ObjectInfo, error) {
|
||||
return getCache(ctx, g.rcClient, g.getS3Key(engine, name), g.expireTime, func(ctx context.Context) (*s3.ObjectInfo, error) {
|
||||
return g.s3.StatObject(ctx, name)
|
||||
})
|
||||
}
|
||||
|
||||
type MinioCache interface {
|
||||
metaCache
|
||||
GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*MinioImageInfo, error)) (*MinioImageInfo, error)
|
||||
GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error)
|
||||
DelObjectImageInfoKey(keys ...string) MinioCache
|
||||
DelImageThumbnailKey(key string, format string, width int, height int) MinioCache
|
||||
}
|
||||
|
||||
func NewMinioCache(rdb redis.UniversalClient) MinioCache {
|
||||
rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions())
|
||||
return &minioCacheRedis{
|
||||
rcClient: rcClient,
|
||||
expireTime: time.Hour * 24 * 7,
|
||||
metaCache: NewMetaCacheRedis(rcClient),
|
||||
}
|
||||
}
|
||||
|
||||
type minioCacheRedis struct {
|
||||
metaCache
|
||||
rcClient *rockscache.Client
|
||||
expireTime time.Duration
|
||||
}
|
||||
|
||||
func (g *minioCacheRedis) NewCache() MinioCache {
|
||||
return &minioCacheRedis{
|
||||
rcClient: g.rcClient,
|
||||
expireTime: g.expireTime,
|
||||
metaCache: NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...),
|
||||
}
|
||||
}
|
||||
|
||||
func (g *minioCacheRedis) DelObjectImageInfoKey(keys ...string) MinioCache {
|
||||
s3cache := g.NewCache()
|
||||
ks := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
ks = append(ks, g.getObjectImageInfoKey(key))
|
||||
}
|
||||
s3cache.AddKeys(ks...)
|
||||
return s3cache
|
||||
}
|
||||
|
||||
func (g *minioCacheRedis) DelImageThumbnailKey(key string, format string, width int, height int) MinioCache {
|
||||
s3cache := g.NewCache()
|
||||
s3cache.AddKeys(g.getMinioImageThumbnailKey(key, format, width, height))
|
||||
return s3cache
|
||||
}
|
||||
|
||||
func (g *minioCacheRedis) getObjectImageInfoKey(key string) string {
|
||||
return "MINIO:IMAGE:" + key
|
||||
}
|
||||
|
||||
func (g *minioCacheRedis) getMinioImageThumbnailKey(key string, format string, width int, height int) string {
|
||||
return "MINIO:THUMBNAIL:" + format + ":w" + strconv.Itoa(width) + ":h" + strconv.Itoa(height) + ":" + key
|
||||
}
|
||||
|
||||
func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*MinioImageInfo, error)) (*MinioImageInfo, error) {
|
||||
info, err := getCache(ctx, g.rcClient, g.getObjectImageInfoKey(key), g.expireTime, fn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error) {
|
||||
return getCache(ctx, g.rcClient, g.getMinioImageThumbnailKey(key, format, width, height), g.expireTime, minioCache)
|
||||
}
|
||||
|
||||
type MinioImageInfo struct {
|
||||
IsImg bool `json:"isImg"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
Format string `json:"format"`
|
||||
Etag string `json:"etag"`
|
||||
}
|
||||
@ -1,22 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package minio
|
||||
|
||||
type minioImageInfo struct {
|
||||
NotImage bool `json:"notImage,omitempty"`
|
||||
Width int `json:"width,omitempty"`
|
||||
Height int `json:"height,omitempty"`
|
||||
Format string `json:"format,omitempty"`
|
||||
}
|
||||
@ -0,0 +1,134 @@
|
||||
package minio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/tools/errs"
|
||||
"github.com/OpenIMSDK/tools/log"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
|
||||
"image"
|
||||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (m *Minio) getImageThumbnailURL(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
|
||||
var img image.Image
|
||||
info, err := m.cache.GetImageObjectKeyInfo(ctx, name, func(ctx context.Context) (info *cache.MinioImageInfo, err error) {
|
||||
info, img, err = m.getObjectImageInfo(ctx, name)
|
||||
return
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.IsImg {
|
||||
return "", errs.ErrData.Wrap("object not image")
|
||||
}
|
||||
if opt.Width > info.Width || opt.Width <= 0 {
|
||||
opt.Width = info.Width
|
||||
}
|
||||
if opt.Height > info.Height || opt.Height <= 0 {
|
||||
opt.Height = info.Height
|
||||
}
|
||||
opt.Format = strings.ToLower(opt.Format)
|
||||
if opt.Format == formatJpg {
|
||||
opt.Format = formatJpeg
|
||||
}
|
||||
switch opt.Format {
|
||||
case formatPng, formatJpeg, formatGif:
|
||||
default:
|
||||
opt.Format = ""
|
||||
}
|
||||
reqParams := make(url.Values)
|
||||
if opt.Width == info.Width && opt.Height == info.Height && (opt.Format == info.Format || opt.Format == "") {
|
||||
reqParams.Set("response-content-type", "image/"+info.Format)
|
||||
return m.PresignedGetObject(ctx, name, expire, reqParams)
|
||||
}
|
||||
if opt.Format == "" {
|
||||
switch opt.Format {
|
||||
case formatGif:
|
||||
opt.Format = formatGif
|
||||
case formatJpeg:
|
||||
opt.Format = formatJpeg
|
||||
case formatPng:
|
||||
opt.Format = formatPng
|
||||
default:
|
||||
opt.Format = formatPng
|
||||
}
|
||||
}
|
||||
key, err := m.cache.GetThumbnailKey(ctx, name, opt.Format, opt.Width, opt.Height, func(ctx context.Context) (string, error) {
|
||||
if img == nil {
|
||||
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer reader.Close()
|
||||
img, _, err = ImageStat(reader)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
thumbnail := resizeImage(img, opt.Width, opt.Height)
|
||||
buf := bytes.NewBuffer(nil)
|
||||
switch opt.Format {
|
||||
case formatPng:
|
||||
err = png.Encode(buf, thumbnail)
|
||||
case formatJpeg:
|
||||
err = jpeg.Encode(buf, thumbnail, nil)
|
||||
case formatGif:
|
||||
err = gif.Encode(buf, thumbnail, nil)
|
||||
}
|
||||
cacheKey := filepath.Join(imageThumbnailPath, info.Etag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
|
||||
if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return cacheKey, nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
reqParams.Set("response-content-type", "image/"+opt.Format)
|
||||
return m.PresignedGetObject(ctx, key, expire, reqParams)
|
||||
}
|
||||
|
||||
func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.MinioImageInfo, image.Image, error) {
|
||||
fileInfo, err := m.StatObject(ctx, name)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if fileInfo.Size > maxImageSize {
|
||||
return nil, nil, errors.New("file size too large")
|
||||
}
|
||||
imageData, err := m.getObjectData(ctx, name, fileInfo.Size)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var info cache.MinioImageInfo
|
||||
imageInfo, format, err := ImageStat(bytes.NewReader(imageData))
|
||||
if err == nil {
|
||||
info.IsImg = true
|
||||
info.Format = format
|
||||
info.Width, info.Height = ImageWidthHeight(imageInfo)
|
||||
} else {
|
||||
info.IsImg = false
|
||||
}
|
||||
info.Etag = fileInfo.ETag
|
||||
return &info, imageInfo, nil
|
||||
}
|
||||
|
||||
func (m *Minio) delObjectImageInfoKey(ctx context.Context, key string, size int64) {
|
||||
if size > 0 && size > maxImageSize {
|
||||
return
|
||||
}
|
||||
if err := m.cache.DelObjectImageInfoKey(key).ExecDel(ctx); err != nil {
|
||||
log.ZError(ctx, "DelObjectImageInfoKey failed", err, "key", key)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue