From 94be8fc0c47f61270cca07820b7436b0fad61b14 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 7 Aug 2023 12:02:54 +0800 Subject: [PATCH] feat: minio --- config/config.yaml | 1 + go.mod | 2 +- go.sum | 4 ++-- internal/api/third.go | 9 +++++++- internal/rpc/third/s3.go | 19 +++++++++++++++- pkg/common/config/config.go | 15 +++++++------ pkg/common/db/controller/s3.go | 15 ++++++++----- pkg/common/db/s3/cos/cos.go | 14 ++++++------ pkg/common/db/s3/minio/minio.go | 40 ++++++++++++++++++++------------- pkg/common/db/s3/oss/oss.go | 20 ++++++++--------- pkg/common/db/s3/s3.go | 8 +++---- 11 files changed, 93 insertions(+), 54 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index ec813d69c..bb0e909a5 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -135,6 +135,7 @@ object: sessionToken: "" signEndpoint: "http://127.0.0.1:10005" thumbnailApi: "http://127.0.0.1:10003" + thumbnailUseSignEndpoint: false cos: bucketURL: "https://temp-1252357374.cos.ap-chengdu.myqcloud.com" secretID: "" diff --git a/go.mod b/go.mod index cbaca83b8..31b1860fd 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( require github.com/google/uuid v1.3.0 require ( - github.com/OpenIMSDK/protocol v0.0.3 + github.com/OpenIMSDK/protocol v0.0.6 github.com/OpenIMSDK/tools v0.0.13 github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible github.com/go-redis/redis v6.15.9+incompatible diff --git a/go.sum b/go.sum index 0e6352156..262df5d79 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,8 @@ cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7Biccwk firebase.google.com/go v3.13.0+incompatible h1:3TdYC3DDi6aHn20qoRkxwGqNgdjtblwVAyRLQwGn/+4= firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/OpenIMSDK/protocol v0.0.3 h1:CFQtmnyW+1dYKVFaVaHcJ6oYuMiMdNfU2gC1xz3K/9I= -github.com/OpenIMSDK/protocol v0.0.3/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y= +github.com/OpenIMSDK/protocol v0.0.6 h1:KjaItOEww7vjrhwyxHnVzhw80pnjcNukpskadqW6gnA= +github.com/OpenIMSDK/protocol v0.0.6/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y= github.com/OpenIMSDK/tools v0.0.13 h1:rcw4HS8S2DPZR9UOBxD8/ol9UBMzXBypzOVEytDRIMo= github.com/OpenIMSDK/tools v0.0.13/go.mod h1:eg+q4A34Qmu73xkY0mt37FHGMCMfC6CtmOnm0kFEGFI= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= diff --git a/internal/api/third.go b/internal/api/third.go index e480d4c11..e1f6a3810 100644 --- a/internal/api/third.go +++ b/internal/api/third.go @@ -81,7 +81,14 @@ func (o *ThirdApi) ObjectRedirect(c *gin.Context) { operationID = strconv.Itoa(rand.Int()) } ctx := mcontext.SetOperationID(c, operationID) - resp, err := o.Client.AccessURL(ctx, &third.AccessURLReq{Name: name}) + query := make(map[string]string) + for key, values := range c.Request.URL.Query() { + if len(values) == 0 { + continue + } + query[key] = values[0] + } + resp, err := o.Client.AccessURL(ctx, &third.AccessURLReq{Name: name, Query: query}) if err != nil { if errs.ErrArgs.Is(err) { c.String(http.StatusBadRequest, err.Error()) diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 38b0eb03c..01891b6c8 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -16,6 +16,8 @@ package third import ( "context" + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3" + "strconv" "time" "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/cont" @@ -151,7 +153,22 @@ func (t *thirdServer) CompleteMultipartUpload(ctx context.Context, req *third.Co } func (t *thirdServer) AccessURL(ctx context.Context, req *third.AccessURLReq) (*third.AccessURLResp, error) { - expireTime, rawURL, err := t.s3dataBase.AccessURL(ctx, req.Name, t.defaultExpire) + opt := &s3.AccessURLOption{} + if len(req.Query) > 0 { + switch req.Query["type"] { + case "image": + opt.Image.Format = req.Query["format"] + opt.Image.Width, _ = strconv.Atoi(req.Query["width"]) + opt.Image.Height, _ = strconv.Atoi(req.Query["height"]) + case "video": + opt.Video.Format = req.Query["format"] + opt.Video.Width, _ = strconv.Atoi(req.Query["width"]) + opt.Video.Height, _ = strconv.Atoi(req.Query["height"]) + millisecond, _ := strconv.Atoi(req.Query["time"]) + opt.Video.Time = time.Millisecond * time.Duration(millisecond) + } + } + expireTime, rawURL, err := t.s3dataBase.AccessURL(ctx, req.Name, t.defaultExpire, opt) if err != nil { return nil, err } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index da7942cf8..35dfea4c4 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -114,13 +114,14 @@ type configStruct struct { Enable string `yaml:"enable"` ApiURL string `yaml:"apiURL"` Minio struct { - Bucket string `yaml:"bucket"` - Endpoint string `yaml:"endpoint"` - AccessKeyID string `yaml:"accessKeyID"` - SecretAccessKey string `yaml:"secretAccessKey"` - SessionToken string `yaml:"sessionToken"` - ThumbnailApi string `yaml:"thumbnailApi"` - SignEndpoint string `yaml:"signEndpoint"` + Bucket string `yaml:"bucket"` + Endpoint string `yaml:"endpoint"` + AccessKeyID string `yaml:"accessKeyID"` + SecretAccessKey string `yaml:"secretAccessKey"` + SessionToken string `yaml:"sessionToken"` + SignEndpoint string `yaml:"signEndpoint"` + ThumbnailApi string `yaml:"thumbnailApi"` + ThumbnailUseSignEndpoint bool `yaml:"thumbnailUseSignEndpoint"` } `yaml:"minio"` Cos struct { BucketURL string `yaml:"bucketURL"` diff --git a/pkg/common/db/controller/s3.go b/pkg/common/db/controller/s3.go index 220e8e070..1ced644c2 100644 --- a/pkg/common/db/controller/s3.go +++ b/pkg/common/db/controller/s3.go @@ -30,7 +30,7 @@ type S3Database interface { AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error) CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error) - AccessURL(ctx context.Context, name string, expire time.Duration) (time.Time, string, error) + AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (time.Time, string, error) SetObject(ctx context.Context, info *relation.ObjectModel) error } @@ -70,14 +70,19 @@ func (s *s3Database) SetObject(ctx context.Context, info *relation.ObjectModel) return s.obj.SetObject(ctx, info) } -func (s *s3Database) AccessURL(ctx context.Context, name string, expire time.Duration) (time.Time, string, error) { +func (s *s3Database) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (time.Time, string, error) { obj, err := s.obj.Take(ctx, name) if err != nil { return time.Time{}, "", err } - opt := &s3.AccessURLOption{ - ContentType: obj.ContentType, - Filename: filepath.Base(obj.Name), + if opt == nil { + opt = &s3.AccessURLOption{} + } + if opt.ContentType == "" { + opt.ContentType = obj.ContentType + } + if opt.Filename == "" { + opt.Filename = filepath.Base(obj.Name) } expireTime := time.Now().Add(expire) rawURL, err := s.s3.AccessURL(ctx, obj.Key, expire, opt) diff --git a/pkg/common/db/s3/cos/cos.go b/pkg/common/db/s3/cos/cos.go index 0cbc7b3ea..cbdbd3722 100644 --- a/pkg/common/db/s3/cos/cos.go +++ b/pkg/common/db/s3/cos/cos.go @@ -308,19 +308,19 @@ func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, sec = 0 } query.Set("time", strconv.FormatFloat(sec, 'f', 3, 64)) - switch opt.Video.ImageFormat { + switch opt.Video.Format { case videoSnapshotImagePng, videoSnapshotImageJpg: default: - opt.Video.ImageFormat = videoSnapshotImageJpg + opt.Video.Format = videoSnapshotImageJpg } - query.Set("format", opt.Video.ImageFormat) - opt.ContentType = "image/" + opt.Video.ImageFormat + query.Set("format", opt.Video.Format) + opt.ContentType = "image/" + opt.Video.Format if opt.Filename == "" { - opt.Filename = filepath.Base(name) + "." + opt.Video.ImageFormat - } else if filepath.Ext(opt.Filename) != "."+opt.Video.ImageFormat { - opt.Filename += "." + opt.Video.ImageFormat + opt.Filename = filepath.Base(name) + "." + opt.Video.Format + } else if filepath.Ext(opt.Filename) != "."+opt.Video.Format { + opt.Filename += "." + opt.Video.Format } if opt.Video.Width > 0 { query.Set("width", strconv.Itoa(opt.Video.Width)) diff --git a/pkg/common/db/s3/minio/minio.go b/pkg/common/db/s3/minio/minio.go index 5773c5d04..19e722807 100644 --- a/pkg/common/db/s3/minio/minio.go +++ b/pkg/common/db/s3/minio/minio.go @@ -68,12 +68,13 @@ func NewMinio() (s3.Interface, error) { imageApi += "image?" } m := &Minio{ - bucket: conf.Bucket, - bucketURL: conf.Endpoint + "/" + conf.Bucket + "/", - imageApi: imageApi, - core: &minio.Core{Client: client}, - lock: &sync.Mutex{}, - init: false, + bucket: conf.Bucket, + bucketURL: conf.Endpoint + "/" + conf.Bucket + "/", + imageApi: imageApi, + imageUseSignAddr: conf.ThumbnailUseSignEndpoint, + core: &minio.Core{Client: client}, + lock: &sync.Mutex{}, + init: false, } if conf.SignEndpoint == "" { m.sign = m.core.Client @@ -100,15 +101,16 @@ func NewMinio() (s3.Interface, error) { } type Minio struct { - bucket string - bucketURL string - imageApi string - location string - opts *minio.Options - core *minio.Core - sign *minio.Client - lock sync.Locker - init bool + bucket string + bucketURL string + imageApi string + imageUseSignAddr bool + location string + opts *minio.Options + core *minio.Core + sign *minio.Client + lock sync.Locker + init bool } func (m *Minio) initMinio(ctx context.Context) error { @@ -369,7 +371,13 @@ func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration } else if expire < time.Second { expire = time.Second } - u, err := m.sign.PresignedGetObject(ctx, m.bucket, name, expire, reqParams) + var client *minio.Client + if m.imageUseSignAddr { + client = m.sign + } else { + client = m.core.Client + } + u, err := client.PresignedGetObject(ctx, m.bucket, name, expire, reqParams) if err != nil { return "", err } diff --git a/pkg/common/db/s3/oss/oss.go b/pkg/common/db/s3/oss/oss.go index 38034cfbc..08ade23c9 100644 --- a/pkg/common/db/s3/oss/oss.go +++ b/pkg/common/db/s3/oss/oss.go @@ -291,9 +291,9 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, } opt.ContentType = "image/" + format if opt.Filename == "" { - opt.Filename = filepath.Base(name) + "." + opt.Video.ImageFormat - } else if filepath.Ext(opt.Filename) != "."+opt.Video.ImageFormat { - opt.Filename += "." + opt.Video.ImageFormat + opt.Filename = filepath.Base(name) + "." + opt.Video.Format + } else if filepath.Ext(opt.Filename) != "."+opt.Video.Format { + opt.Filename += "." + opt.Video.Format } // https://oss-console-img-demo-cn-hangzhou.oss-cn-hangzhou.aliyuncs.com/example.jpg?x-oss-process=image/resize,h_100,m_lfit process := "image/resize,m_lfit" @@ -313,18 +313,18 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, if millisecond < 0 { millisecond = 0 } - switch opt.Video.ImageFormat { + switch opt.Video.Format { case videoSnapshotImageJpg, videoSnapshotImagePng: default: - opt.Video.ImageFormat = videoSnapshotImageJpg + opt.Video.Format = videoSnapshotImageJpg } - opt.ContentType = "image/" + opt.Video.ImageFormat + opt.ContentType = "image/" + opt.Video.Format if opt.Filename == "" { - opt.Filename = filepath.Base(name) + "." + opt.Video.ImageFormat - } else if filepath.Ext(opt.Filename) != "."+opt.Video.ImageFormat { - opt.Filename += "." + opt.Video.ImageFormat + opt.Filename = filepath.Base(name) + "." + opt.Video.Format + } else if filepath.Ext(opt.Filename) != "."+opt.Video.Format { + opt.Filename += "." + opt.Video.Format } - process := "video/snapshot,t_" + strconv.Itoa(millisecond) + ",f_" + opt.Video.ImageFormat + process := "video/snapshot,t_" + strconv.Itoa(millisecond) + ",f_" + opt.Video.Format if opt.Video.Width > 0 { process += ",w_" + strconv.Itoa(opt.Video.Width) } diff --git a/pkg/common/db/s3/s3.go b/pkg/common/db/s3/s3.go index 0fd4ccfe9..cb08813c6 100644 --- a/pkg/common/db/s3/s3.go +++ b/pkg/common/db/s3/s3.go @@ -123,10 +123,10 @@ type Image struct { } type Video struct { - Width int `json:"width"` - Height int `json:"height"` - Time time.Duration `json:"time"` - ImageFormat string `json:"format"` + Width int `json:"width"` + Height int `json:"height"` + Time time.Duration `json:"time"` + Format string `json:"format"` } type AccessURLOption struct {