From 1f6225db41a070ad0af71c29461d03079bb81122 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 12 Jul 2023 13:39:08 +0800 Subject: [PATCH 1/5] s3 cos copy --- pkg/common/db/s3/cos/cos.go | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/pkg/common/db/s3/cos/cos.go b/pkg/common/db/s3/cos/cos.go index e1d1b3a49..365f6b153 100644 --- a/pkg/common/db/s3/cos/cos.go +++ b/pkg/common/db/s3/cos/cos.go @@ -77,7 +77,7 @@ func (c *Cos) CompleteMultipartUpload(ctx context.Context, uploadID string, name for i, part := range parts { opts.Parts[i] = cos.Object{ PartNumber: part.PartNumber, - ETag: strings.ToLower(part.ETag), + ETag: strings.ReplaceAll(part.ETag, `"`, ``), } } result, _, err := c.client.Object.CompleteMultipartUpload(ctx, name, uploadID, opts) @@ -109,27 +109,6 @@ func (c *Cos) PartSize(ctx context.Context, size int64) (int64, error) { return partSize, nil } -func (c *Cos) ClientUploadPart(ctx context.Context, partSize int64, expire time.Duration, upload *s3.InitiateMultipartUploadResult) (*s3.MultipartUploadRequest, error) { - uri := c.client.BaseURL.BucketURL.String() + "/" + cos.EncodeURIComponent(upload.Key) - req, err := http.NewRequestWithContext(ctx, http.MethodPut, uri, nil) - if err != nil { - return nil, err - } - cos.AddAuthorizationHeader(c.credential.SecretID, c.credential.SecretKey, c.credential.SessionToken, req, cos.NewAuthTime(expire)) - return &s3.MultipartUploadRequest{ - UploadID: upload.UploadID, - Bucket: upload.Bucket, - Key: upload.Key, - Method: req.Method, - URL: uri, - Query: url.Values{"uploadId": {upload.UploadID}}, - Header: req.Header, - PartKey: "partNumber", - PartSize: partSize, - FirstPart: 1, - }, nil -} - func (c *Cos) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) { result := s3.AuthSignResult{ URL: c.client.BaseURL.BucketURL.String() + "/" + cos.EncodeURIComponent(name), @@ -200,13 +179,14 @@ func (c *Cos) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, erro } func (c *Cos) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) { - result, _, err := c.client.Object.Copy(ctx, dst, c.copyURL+src, &cos.ObjectCopyOptions{}) + sourceURL := c.copyURL + src + result, _, err := c.client.Object.Copy(ctx, dst, sourceURL, nil) if err != nil { return nil, err } return &s3.CopyObjectInfo{ Key: dst, - ETag: result.ETag, + ETag: strings.ReplaceAll(result.ETag, `""`, ""), }, nil } From 75af5447ae864cb1f8dab269a25bab2f3f5c83b8 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 12 Jul 2023 13:45:45 +0800 Subject: [PATCH 2/5] s3 cos copy --- pkg/common/db/s3/cos/cos.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/db/s3/cos/cos.go b/pkg/common/db/s3/cos/cos.go index 365f6b153..d355b2f0b 100644 --- a/pkg/common/db/s3/cos/cos.go +++ b/pkg/common/db/s3/cos/cos.go @@ -186,7 +186,7 @@ func (c *Cos) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyO } return &s3.CopyObjectInfo{ Key: dst, - ETag: strings.ReplaceAll(result.ETag, `""`, ""), + ETag: strings.ReplaceAll(result.ETag, `"`, ``), }, nil } From 84f8074d6a027bfa15de14d20a920a32eff67478 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 12 Jul 2023 13:55:17 +0800 Subject: [PATCH 3/5] s3 url --- internal/rpc/third/s3.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 2b534aefb..6cc1398da 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -34,7 +34,7 @@ func (t *thirdServer) InitiateMultipartUpload(ctx context.Context, req *third.In if err := checkUploadName(ctx, req.Name); err != nil { return nil, err } - result, err := t.s3dataBase.InitiateMultipartUpload(ctx, req.Hash, req.Size, time.Hour*24, int(req.MaxParts)) + result, err := t.s3dataBase.InitiateMultipartUpload(ctx, req.Hash, req.Size, time.Hour*24*7, int(req.MaxParts)) if err != nil { if haErr, ok := errs.Unwrap(err).(*cont.HashAlreadyExistsError); ok { obj := &relation.ObjectModel{ @@ -51,7 +51,7 @@ func (t *thirdServer) InitiateMultipartUpload(ctx context.Context, req *third.In return nil, err } return &third.InitiateMultipartUploadResp{ - Url: t.apiAddress(obj.Key), + Url: t.apiAddress(obj.Name), }, nil } return nil, err From 406a77d5b8d658e16a2ea57819e7b39572c60f13 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 12 Jul 2023 13:58:20 +0800 Subject: [PATCH 4/5] s3 url --- pkg/common/db/s3/cos/cos.go | 16 ++++++++-------- pkg/common/db/s3/minio/minio.go | 16 ++++++++-------- pkg/common/db/s3/oss/oss.go | 16 ++++++++-------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pkg/common/db/s3/cos/cos.go b/pkg/common/db/s3/cos/cos.go index d355b2f0b..a6c958813 100644 --- a/pkg/common/db/s3/cos/cos.go +++ b/pkg/common/db/s3/cos/cos.go @@ -233,14 +233,14 @@ func (c *Cos) ListUploadedParts(ctx context.Context, uploadID string, name strin func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) { reqParams := make(url.Values) - if opt != nil { - if opt.ContentType != "" { - reqParams.Set("Content-Type", opt.ContentType) - } - if opt.ContentDisposition != "" { - reqParams.Set("Content-Disposition", opt.ContentDisposition) - } - } + //if opt != nil { + // if opt.ContentType != "" { + // reqParams.Set("Content-Type", opt.ContentType) + // } + // if opt.ContentDisposition != "" { + // reqParams.Set("Content-Disposition", opt.ContentDisposition) + // } + //} if expire <= 0 { expire = time.Hour * 24 * 365 * 99 // 99 years } else if expire < time.Second { diff --git a/pkg/common/db/s3/minio/minio.go b/pkg/common/db/s3/minio/minio.go index 6949bcb4c..c6070bc02 100644 --- a/pkg/common/db/s3/minio/minio.go +++ b/pkg/common/db/s3/minio/minio.go @@ -229,14 +229,14 @@ func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name str func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) { reqParams := make(url.Values) - if opt != nil { - if opt.ContentType != "" { - reqParams.Set("Content-Type", opt.ContentType) - } - if opt.ContentDisposition != "" { - reqParams.Set("Content-Disposition", opt.ContentDisposition) - } - } + //if opt != nil { + // if opt.ContentType != "" { + // reqParams.Set("Content-Type", opt.ContentType) + // } + // if opt.ContentDisposition != "" { + // reqParams.Set("Content-Disposition", opt.ContentDisposition) + // } + //} if expire <= 0 { expire = time.Hour * 24 * 365 * 99 // 99 years } else if expire < time.Second { diff --git a/pkg/common/db/s3/oss/oss.go b/pkg/common/db/s3/oss/oss.go index 51abcb46c..cd54c341b 100644 --- a/pkg/common/db/s3/oss/oss.go +++ b/pkg/common/db/s3/oss/oss.go @@ -242,14 +242,14 @@ func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name strin func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) { var opts []oss.Option - if opt != nil { - if opt.ContentType != "" { - opts = append(opts, oss.ContentType(opt.ContentType)) - } - if opt.ContentDisposition != "" { - opts = append(opts, oss.ContentDisposition(opt.ContentDisposition)) - } - } + //if opt != nil { + // if opt.ContentType != "" { + // opts = append(opts, oss.ContentType(opt.ContentType)) + // } + // if opt.ContentDisposition != "" { + // opts = append(opts, oss.ContentDisposition(opt.ContentDisposition)) + // } + //} if expire <= 0 { expire = time.Hour * 24 * 365 * 99 // 99 years } else if expire < time.Second { From bdf93565a317deeb10a9bad71bff8b700888b571 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 12 Jul 2023 14:01:48 +0800 Subject: [PATCH 5/5] s3 AccessURL --- pkg/common/db/s3/cos/cos.go | 4 ++-- pkg/common/db/s3/minio/minio.go | 4 ++-- pkg/common/db/s3/oss/oss.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/common/db/s3/cos/cos.go b/pkg/common/db/s3/cos/cos.go index a6c958813..4b1b4aa4d 100644 --- a/pkg/common/db/s3/cos/cos.go +++ b/pkg/common/db/s3/cos/cos.go @@ -232,7 +232,7 @@ func (c *Cos) ListUploadedParts(ctx context.Context, uploadID string, name strin } func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) { - reqParams := make(url.Values) + //reqParams := make(url.Values) //if opt != nil { // if opt.ContentType != "" { // reqParams.Set("Content-Type", opt.ContentType) @@ -246,7 +246,7 @@ func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, } else if expire < time.Second { expire = time.Second } - rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, reqParams) + rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, nil) if err != nil { return "", err } diff --git a/pkg/common/db/s3/minio/minio.go b/pkg/common/db/s3/minio/minio.go index c6070bc02..367cbe8a8 100644 --- a/pkg/common/db/s3/minio/minio.go +++ b/pkg/common/db/s3/minio/minio.go @@ -228,7 +228,7 @@ func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name str } func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) { - reqParams := make(url.Values) + //reqParams := make(url.Values) //if opt != nil { // if opt.ContentType != "" { // reqParams.Set("Content-Type", opt.ContentType) @@ -242,7 +242,7 @@ func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration } else if expire < time.Second { expire = time.Second } - u, err := m.core.Client.PresignedGetObject(ctx, m.bucket, name, expire, reqParams) + u, err := m.core.Client.PresignedGetObject(ctx, m.bucket, name, expire, nil) if err != nil { return "", err } diff --git a/pkg/common/db/s3/oss/oss.go b/pkg/common/db/s3/oss/oss.go index cd54c341b..f2b50da50 100644 --- a/pkg/common/db/s3/oss/oss.go +++ b/pkg/common/db/s3/oss/oss.go @@ -241,7 +241,7 @@ func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name strin } func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) { - var opts []oss.Option + //var opts []oss.Option //if opt != nil { // if opt.ContentType != "" { // opts = append(opts, oss.ContentType(opt.ContentType)) @@ -255,5 +255,5 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, } else if expire < time.Second { expire = time.Second } - return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...) + return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second)) }