From 56cf9906450d6246dfdc7a3074d7eaf09ab67ff8 Mon Sep 17 00:00:00 2001 From: Darren Yu Date: Thu, 16 Oct 2025 03:14:40 +0800 Subject: [PATCH] revoke(oss): downgrade to SDK v1, switch sign method to v4 --- go.mod | 2 +- go.sum | 4 +- pkg/filemanager/driver/oss/media.go | 7 +- pkg/filemanager/driver/oss/oss.go | 289 +++++++++++----------------- 4 files changed, 116 insertions(+), 186 deletions(-) diff --git a/go.mod b/go.mod index 259da672..e975e6f6 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( entgo.io/ent v0.13.0 github.com/Masterminds/semver/v3 v3.3.1 github.com/abslant/gzip v0.0.9 - github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.3.0 + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/aws/aws-sdk-go v1.31.5 github.com/bodgit/sevenzip v1.6.0 github.com/cloudflare/cfssl v1.6.1 diff --git a/go.sum b/go.sum index 72b414e5..ed10902b 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.3.0 h1:wQlqotpyjYPjJz+Noh5bRu7Snmydk8SKC5Z6u1CR20Y= -github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.3.0/go.mod h1:FTzydeQVmR24FI0D6XWUOMKckjXehM/jgMn1xC+DA9M= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 h1:8PmGpDEZl9yDpcdEr6Odf23feCxK3LNUNMxjXg41pZQ= github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= diff --git a/pkg/filemanager/driver/oss/media.go b/pkg/filemanager/driver/oss/media.go index 11db9483..22b8c48d 100644 --- a/pkg/filemanager/driver/oss/media.go +++ b/pkg/filemanager/driver/oss/media.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" + "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver" "github.com/cloudreve/Cloudreve/v4/pkg/mediameta" "github.com/cloudreve/Cloudreve/v4/pkg/request" @@ -266,14 +266,13 @@ func (handler *Driver) extractImageMeta(ctx context.Context, path string) ([]dri // extractMediaInfo Sends API calls to OSS IMM service to extract media info. func (handler *Driver) extractMediaInfo(ctx context.Context, path string, category string, forceSign bool) (string, error) { + mediaOption := []oss.Option{oss.Process(category)} mediaInfoExpire := time.Now().Add(mediaInfoTTL) thumbURL, err := handler.signSourceURL( ctx, path, &mediaInfoExpire, - &oss.GetObjectRequest{ - Process: oss.Ptr(category), - }, + mediaOption, forceSign, ) if err != nil { diff --git a/pkg/filemanager/driver/oss/oss.go b/pkg/filemanager/driver/oss/oss.go index 178eca78..3a4fb41d 100644 --- a/pkg/filemanager/driver/oss/oss.go +++ b/pkg/filemanager/driver/oss/oss.go @@ -15,8 +15,7 @@ import ( "strings" "time" - "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" - "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials" + "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/cloudreve/Cloudreve/v4/ent" "github.com/cloudreve/Cloudreve/v4/inventory/types" "github.com/cloudreve/Cloudreve/v4/pkg/boolset" @@ -53,6 +52,7 @@ type Driver struct { policy *ent.StoragePolicy client *oss.Client + bucket *oss.Bucket settings setting.Provider l logging.Logger config conf.ConfigProvider @@ -65,8 +65,12 @@ type Driver struct { type key int const ( - chunkRetrySleep = time.Duration(5) * time.Second - maxDeleteBatch = 1000 + chunkRetrySleep = time.Duration(5) * time.Second + uploadIdParam = "uploadId" + partNumberParam = "partNumber" + callbackParam = "callback" + completeAllHeader = "x-oss-complete-all" + maxDeleteBatch = 1000 // MultiPartUploadThreshold 服务端使用分片上传的阈值 MultiPartUploadThreshold int64 = 5 * (1 << 30) // 5GB @@ -98,27 +102,21 @@ func New(ctx context.Context, policy *ent.StoragePolicy, settings setting.Provid // CORS 创建跨域策略 func (handler *Driver) CORS() error { - _, err := handler.client.PutBucketCors(context.Background(), &oss.PutBucketCorsRequest{ - Bucket: &handler.policy.BucketName, - CORSConfiguration: &oss.CORSConfiguration{ - CORSRules: []oss.CORSRule{ - { - AllowedOrigins: []string{"*"}, - AllowedMethods: []string{ - "GET", - "POST", - "PUT", - "DELETE", - "HEAD", - }, - ExposeHeaders: []string{}, - AllowedHeaders: []string{"*"}, - MaxAgeSeconds: oss.Ptr(int64(3600)), - }, + return handler.client.SetBucketCORS(handler.policy.BucketName, []oss.CORSRule{ + { + AllowedOrigin: []string{"*"}, + AllowedMethod: []string{ + "GET", + "POST", + "PUT", + "DELETE", + "HEAD", }, - }}) - - return err + ExposeHeader: []string{}, + AllowedHeader: []string{"*"}, + MaxAgeSeconds: 3600, + }, + }) } // InitOSSClient 初始化OSS鉴权客户端 @@ -127,28 +125,38 @@ func (handler *Driver) InitOSSClient(forceUsePublicEndpoint bool) error { return errors.New("empty policy") } + opt := make([]oss.ClientOption, 0) + // 决定是否使用内网 Endpoint endpoint := handler.policy.Server - useCname := false if handler.policy.Settings.ServerSideEndpoint != "" && !forceUsePublicEndpoint { endpoint = handler.policy.Settings.ServerSideEndpoint } else if handler.policy.Settings.UseCname { - useCname = true + opt = append(opt, oss.UseCname(true)) } if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") { endpoint = "https://" + endpoint } - cfg := oss.LoadDefaultConfig(). - WithCredentialsProvider(credentials.NewStaticCredentialsProvider(handler.policy.AccessKey, handler.policy.SecretKey, "")). - WithEndpoint(endpoint). - WithRegion(handler.policy.Settings.Region). - WithUseCName(useCname) + // 设置签名版本 & 存储桶 region + opt = append(opt, oss.AuthVersion(oss.AuthV4)) + opt = append(opt, oss.Region(handler.policy.Settings.Region)) // 初始化客户端 - client := oss.NewClient(cfg) + client, err := oss.New(endpoint, handler.policy.AccessKey, handler.policy.SecretKey, opt...) + if err != nil { + return err + } handler.client = client + + // 初始化存储桶 + bucket, err := client.Bucket(handler.policy.BucketName) + if err != nil { + return err + } + handler.bucket = bucket + return nil } @@ -162,40 +170,38 @@ func (handler *Driver) List(ctx context.Context, base string, onProgress driver. var ( delimiter string + marker string objects []oss.ObjectProperties - commons []oss.CommonPrefix + commons []string ) if !recursive { delimiter = "/" } - p := handler.client.NewListObjectsPaginator(&oss.ListObjectsRequest{ - Bucket: &handler.policy.BucketName, - Prefix: &base, - MaxKeys: 1000, - Delimiter: &delimiter, - }) - - for p.HasNext() { - page, err := p.NextPage(ctx) + for { + subRes, err := handler.bucket.ListObjects(oss.Marker(marker), oss.Prefix(base), + oss.MaxKeys(1000), oss.Delimiter(delimiter)) if err != nil { return nil, err } - - objects = append(objects, page.Contents...) - commons = append(commons, page.CommonPrefixes...) + objects = append(objects, subRes.Objects...) + commons = append(commons, subRes.CommonPrefixes...) + marker = subRes.NextMarker + if marker == "" { + break + } } // 处理列取结果 res := make([]fs.PhysicalObject, 0, len(objects)+len(commons)) // 处理目录 for _, object := range commons { - rel, err := filepath.Rel(base, *object.Prefix) + rel, err := filepath.Rel(base, object) if err != nil { continue } res = append(res, fs.PhysicalObject{ - Name: path.Base(*object.Prefix), + Name: path.Base(object), RelativePath: filepath.ToSlash(rel), Size: 0, IsDir: true, @@ -206,17 +212,17 @@ func (handler *Driver) List(ctx context.Context, base string, onProgress driver. // 处理文件 for _, object := range objects { - rel, err := filepath.Rel(base, *object.Key) + rel, err := filepath.Rel(base, object.Key) if err != nil { continue } res = append(res, fs.PhysicalObject{ - Name: path.Base(*object.Key), - Source: *object.Key, + Name: path.Base(object.Key), + Source: object.Key, RelativePath: filepath.ToSlash(rel), Size: object.Size, IsDir: false, - LastModify: *object.LastModified, + LastModify: object.LastModified, }) } onProgress(len(res)) @@ -243,34 +249,25 @@ func (handler *Driver) Put(ctx context.Context, file *fs.UploadRequest) error { // 是否允许覆盖 overwrite := file.Mode&fs.ModeOverwrite == fs.ModeOverwrite - forbidOverwrite := oss.Ptr(strconv.FormatBool(!overwrite)) - exipires := oss.Ptr(time.Now().Add(credentialTTL * time.Second).Format(time.RFC3339)) + options := []oss.Option{ + oss.WithContext(ctx), + oss.Expires(time.Now().Add(credentialTTL * time.Second)), + oss.ForbidOverWrite(!overwrite), + oss.ContentType(mimeType), + } // 小文件直接上传 if file.Props.Size < MultiPartUploadThreshold { - _, err := handler.client.PutObject(ctx, &oss.PutObjectRequest{ - Bucket: &handler.policy.BucketName, - Key: &file.Props.SavePath, - Body: file, - ForbidOverwrite: forbidOverwrite, - ContentType: oss.Ptr(mimeType), - }) - return err + return handler.bucket.PutObject(file.Props.SavePath, file, options...) } // 超过阈值时使用分片上传 - imur, err := handler.client.InitiateMultipartUpload(ctx, &oss.InitiateMultipartUploadRequest{ - Bucket: &handler.policy.BucketName, - Key: &file.Props.SavePath, - ContentType: oss.Ptr(mimeType), - ForbidOverwrite: forbidOverwrite, - Expires: exipires, - }) + imur, err := handler.bucket.InitiateMultipartUpload(file.Props.SavePath, options...) if err != nil { return fmt.Errorf("failed to initiate multipart upload: %w", err) } - parts := make([]*oss.UploadPartResult, 0) + parts := make([]oss.UploadPart, 0) chunks := chunk.NewChunkGroup(file, handler.chunkSize, &backoff.ConstantBackoff{ Max: handler.settings.ChunkRetryLimit(ctx), @@ -278,13 +275,7 @@ func (handler *Driver) Put(ctx context.Context, file *fs.UploadRequest) error { }, handler.settings.UseChunkBuffer(ctx), handler.l, handler.settings.TempPath(ctx)) uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error { - part, err := handler.client.UploadPart(ctx, &oss.UploadPartRequest{ - Bucket: &handler.policy.BucketName, - Key: &file.Props.SavePath, - UploadId: imur.UploadId, - PartNumber: int32(current.Index() + 1), - Body: content, - }) + part, err := handler.bucket.UploadPart(imur, content, current.Length(), current.Index()+1, oss.WithContext(ctx)) if err == nil { parts = append(parts, part) } @@ -293,27 +284,14 @@ func (handler *Driver) Put(ctx context.Context, file *fs.UploadRequest) error { for chunks.Next() { if err := chunks.Process(uploadFunc); err != nil { - handler.cancelUpload(*imur) + handler.cancelUpload(imur) return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err) } } - _, err = handler.client.CompleteMultipartUpload(ctx, &oss.CompleteMultipartUploadRequest{ - Bucket: &handler.policy.BucketName, - Key: imur.Key, - UploadId: imur.UploadId, - CompleteMultipartUpload: &oss.CompleteMultipartUpload{ - Parts: lo.Map(parts, func(part *oss.UploadPartResult, i int) oss.UploadPart { - return oss.UploadPart{ - PartNumber: int32(i + 1), - ETag: part.ETag, - } - }), - }, - ForbidOverwrite: oss.Ptr(strconv.FormatBool(!overwrite)), - }) + _, err = handler.bucket.CompleteMultipartUpload(imur, parts, oss.ForbidOverWrite(!overwrite), oss.WithContext(ctx)) if err != nil { - handler.cancelUpload(*imur) + handler.cancelUpload(imur) } return err @@ -328,12 +306,7 @@ func (handler *Driver) Delete(ctx context.Context, files ...string) ([]string, e for index, group := range groups { handler.l.Debug("Process delete group #%d: %v", index, group) // 删除文件 - delRes, err := handler.client.DeleteMultipleObjects(ctx, &oss.DeleteMultipleObjectsRequest{ - Bucket: &handler.policy.BucketName, - Objects: lo.Map(group, func(v string, i int) oss.DeleteObject { - return oss.DeleteObject{Key: &v} - }), - }) + delRes, err := handler.bucket.DeleteObjects(group) if err != nil { failed = append(failed, group...) lastError = err @@ -341,14 +314,7 @@ func (handler *Driver) Delete(ctx context.Context, files ...string) ([]string, e } // 统计未删除的文件 - failed = append( - failed, - util.SliceDifference(files, - lo.Map(delRes.DeletedObjects, func(v oss.DeletedInfo, i int) string { - return *v.Key - }), - )..., - ) + failed = append(failed, util.SliceDifference(files, delRes.DeletedObjects)...) } if len(failed) > 0 && lastError == nil { @@ -381,14 +347,12 @@ func (handler *Driver) Thumb(ctx context.Context, expire *time.Time, ext string, thumbParam += fmt.Sprintf("/format,%s", enco.Format) } - req := &oss.GetObjectRequest{ - Process: oss.Ptr(thumbParam), - } + thumbOption := []oss.Option{oss.Process(thumbParam)} thumbURL, err := handler.signSourceURL( ctx, e.Source(), expire, - req, + thumbOption, false, ) if err != nil { @@ -410,11 +374,11 @@ func (handler *Driver) Source(ctx context.Context, e fs.Entity, args *driver.Get } // 添加各项设置 - req := &oss.GetObjectRequest{} + var signOptions = make([]oss.Option, 0, 2) if args.IsDownload { encodedFilename := url.PathEscape(args.DisplayName) - req.ResponseContentDisposition = oss.Ptr(fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, - encodedFilename, encodedFilename)) + signOptions = append(signOptions, oss.ResponseContentDisposition(fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, + encodedFilename, encodedFilename))) } if args.Speed > 0 { // Byte 转换为 bit @@ -427,33 +391,25 @@ func (handler *Driver) Source(ctx context.Context, e fs.Entity, args *driver.Get if args.Speed > 838860800 { args.Speed = 838860800 } - req.TrafficLimit = args.Speed + signOptions = append(signOptions, oss.TrafficLimitParam(args.Speed)) } - return handler.signSourceURL(ctx, e.Source(), args.Expire, req, false) + return handler.signSourceURL(ctx, e.Source(), args.Expire, signOptions, false) } -func (handler *Driver) signSourceURL(ctx context.Context, path string, expire *time.Time, req *oss.GetObjectRequest, forceSign bool) (string, error) { - ttl := time.Duration(24) * time.Hour * 365 * 20 +func (handler *Driver) signSourceURL(ctx context.Context, path string, expire *time.Time, options []oss.Option, forceSign bool) (string, error) { + ttl := int64(86400 * 7) if expire != nil { - ttl = time.Until(*expire) - } - - if req == nil { - req = &oss.GetObjectRequest{} + ttl = int64(time.Until(*expire).Seconds()) } - req.Bucket = &handler.policy.BucketName - req.Key = &path - - // signedURL, err := handler.client.Presign(path, oss.HTTPGet, ttl, options...) - result, err := handler.client.Presign(ctx, req, oss.PresignExpires(ttl)) + signedURL, err := handler.bucket.SignURL(path, oss.HTTPGet, ttl, options...) if err != nil { return "", err } // 将最终生成的签名URL域名换成用户自定义的加速域名（如果有） - finalURL, err := url.Parse(result.URL) + finalURL, err := url.Parse(signedURL) if err != nil { return "", err } @@ -502,41 +458,34 @@ func (handler *Driver) Token(ctx context.Context, uploadSession *fs.UploadSessio } // 初始化分片上传 - imur, err := handler.client.InitiateMultipartUpload(ctx, &oss.InitiateMultipartUploadRequest{ - Bucket: &handler.policy.BucketName, - Key: &file.Props.SavePath, - ContentType: oss.Ptr(mimeType), - ForbidOverwrite: oss.Ptr(strconv.FormatBool(true)), - Expires: oss.Ptr(uploadSession.Props.ExpireAt.Format(time.RFC3339)), - }) + options := []oss.Option{ + oss.WithContext(ctx), + oss.Expires(uploadSession.Props.ExpireAt), + oss.ForbidOverWrite(true), + oss.ContentType(mimeType), + } + imur, err := handler.bucket.InitiateMultipartUpload(file.Props.SavePath, options...) if err != nil { return nil, fmt.Errorf("failed to initialize multipart upload: %w", err) } - uploadSession.UploadID = *imur.UploadId + uploadSession.UploadID = imur.UploadID // 为每个分片签名上传 URL chunks := chunk.NewChunkGroup(file, handler.chunkSize, &backoff.ConstantBackoff{}, false, handler.l, "") urls := make([]string, chunks.Num()) - ttl := time.Until(uploadSession.Props.ExpireAt) + ttl := int64(time.Until(uploadSession.Props.ExpireAt).Seconds()) for chunks.Next() { err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error { - signedURL, err := handler.client.Presign(ctx, &oss.UploadPartRequest{ - Bucket: &handler.policy.BucketName, - Key: &file.Props.SavePath, - UploadId: imur.UploadId, - PartNumber: int32(c.Index() + 1), - Body: chunk, - RequestCommon: oss.RequestCommon{ - Headers: map[string]string{ - "Content-Type": "application/octet-stream", - }, - }, - }, oss.PresignExpires(ttl)) + signedURL, err := handler.bucket.SignURL(file.Props.SavePath, oss.HTTPPut, + ttl, + oss.AddParam(partNumberParam, strconv.Itoa(c.Index()+1)), + oss.AddParam(uploadIdParam, imur.UploadID), + oss.ContentType("application/octet-stream")) if err != nil { return err } - urls[c.Index()] = signedURL.URL + urls[c.Index()] = signedURL return nil }) if err != nil { @@ -545,43 +494,29 @@ func (handler *Driver) Token(ctx context.Context, uploadSession *fs.UploadSessio } // 签名完成分片上传的URL - completeURL, err := handler.client.Presign(ctx, &oss.CompleteMultipartUploadRequest{ - Bucket: &handler.policy.BucketName, - Key: &file.Props.SavePath, - UploadId: imur.UploadId, - RequestCommon: oss.RequestCommon{ - Parameters: map[string]string{ - "callback": callbackPolicyEncoded, - }, - Headers: map[string]string{ - "Content-Type": "application/octet-stream", - "x-oss-complete-all": "yes", - "x-oss-forbid-overwrite": "true", - }, - }, - }, oss.PresignExpires(ttl)) + completeURL, err := handler.bucket.SignURL(file.Props.SavePath, oss.HTTPPost, ttl, + oss.ContentType("application/octet-stream"), + oss.AddParam(uploadIdParam, imur.UploadID), + oss.Expires(time.Now().Add(time.Duration(ttl)*time.Second)), + oss.SetHeader(completeAllHeader, "yes"), + oss.ForbidOverWrite(true), + oss.AddParam(callbackParam, callbackPolicyEncoded)) if err != nil { return nil, err } return &fs.UploadCredential{ - UploadID: *imur.UploadId, + UploadID: imur.UploadID, UploadURLs: urls, - CompleteURL: completeURL.URL, + CompleteURL: completeURL, SessionID: uploadSession.Props.UploadSessionID, ChunkSize: handler.chunkSize, - Callback: callbackPolicyEncoded, }, nil } // 取消上传凭证 func (handler *Driver) CancelToken(ctx context.Context, uploadSession *fs.UploadSession) error { - _, err := handler.client.AbortMultipartUpload(ctx, &oss.AbortMultipartUploadRequest{ - Bucket: &handler.policy.BucketName, - Key: &uploadSession.Props.SavePath, - UploadId: &uploadSession.UploadID, - }) - return err + return handler.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{UploadID: uploadSession.UploadID, Key: uploadSession.Props.SavePath}, oss.WithContext(ctx)) } func (handler *Driver) CompleteUpload(ctx context.Context, session *fs.UploadSession) error { @@ -625,11 +560,7 @@ func (handler *Driver) LocalPath(ctx context.Context, path string) string { } func (handler *Driver) cancelUpload(imur oss.InitiateMultipartUploadResult) { - if _, err := handler.client.AbortMultipartUpload(context.Background(), &oss.AbortMultipartUploadRequest{ - Bucket: &handler.policy.BucketName, - Key: imur.Key, - UploadId: imur.UploadId, - }); err != nil { + if err := handler.bucket.AbortMultipartUpload(imur); err != nil { handler.l.Warning("failed to abort multipart upload: %s", err) } }