From 9b84e417a027986b957262b851f85f545008ae0c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 26 Dec 2023 14:54:52 +0800 Subject: [PATCH] s3 form data --- go.mod | 2 +- go.sum | 10 ---- pkg/common/db/s3/cos/cos.go | 55 +++++++------------ pkg/common/db/s3/minio/minio.go | 6 ++- pkg/common/db/s3/minio/minio_test.go | 79 ---------------------------- pkg/common/db/s3/oss/oss.go | 16 ++++-- 6 files changed, 35 insertions(+), 133 deletions(-) delete mode 100644 pkg/common/db/s3/minio/minio_test.go diff --git a/go.mod b/go.mod index 8e8b425a4..904764096 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require github.com/google/uuid v1.3.1 require ( github.com/IBM/sarama v1.41.3 - github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible + github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible github.com/go-redis/redis v6.15.9+incompatible github.com/redis/go-redis/v9 v9.2.1 github.com/tencentyun/cos-go-sdk-v5 v0.7.45 diff --git a/go.sum b/go.sum index 6faa5d97c..125bc28df 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,6 @@ github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBb github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible h1:Sg/2xHwDrioHpxTN6WMiwbXTpUEinBpHsN7mG21Rc2k= github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible h1:so4m5rRA32Tc5GgKg/5gKUu0CRsYmVO3ThMP6T3CwLc= -github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -91,7 +89,6 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= @@ -199,10 +196,8 @@ github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -255,7 +250,6 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mozillazg/go-httpheader v0.4.0 h1:aBn6aRXtFzyDLZ4VIRLsZbbJloagQfMnCiYgOq6hK4w= github.com/mozillazg/go-httpheader v0.4.0/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -384,7 +378,6 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -456,8 +449,6 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -471,7 +462,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/common/db/s3/cos/cos.go b/pkg/common/db/s3/cos/cos.go index 15d7c57d3..7d2c0befe 100644 --- a/pkg/common/db/s3/cos/cos.go +++ b/pkg/common/db/s3/cos/cos.go @@ -23,7 +23,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "net/http" "net/url" "strconv" @@ -336,68 +335,52 @@ func (c *Cos) getPresignedURL(ctx context.Context, name string, expire time.Dura } func (c *Cos) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { - //res, _, err := c.client.Object.ListUploads(ctx, &cos.ObjectListUploadsOptions{}) - //if err != nil { - // return nil, err - //} + // https://cloud.tencent.com/document/product/436/14690 now := time.Now() expiration := now.Add(duration) keyTime := fmt.Sprintf("%d;%d", now.Unix(), expiration.Unix()) conditions := []any{ - map[string]string{"success_action_status": strconv.Itoa(successCode)}, - []any{"content-length-range", 0, size}, + map[string]string{"q-sign-algorithm": "sha1"}, + map[string]string{"q-ak": c.credential.SecretID}, map[string]string{"q-sign-time": keyTime}, map[string]string{"key": name}, - map[string]string{"q-sign-algorithm": "sha1"}, } - if c.credential.SessionToken != "" { - conditions = append(conditions, map[string]string{"x-cos-security-token": c.credential.SessionToken}) + if contentType != "" { + conditions = append(conditions, map[string]string{"Content-Type": contentType}) } policy := map[string]any{ "expiration": expiration.Format("2006-01-02T15:04:05.000Z"), - "conditions": []any{ - //map[string]string{"bucket": res.Bucket}, - //map[string]string{"acl": "default"}, - map[string]string{"q-sign-algorithm": "sha1"}, - map[string]string{"q-ak": c.credential.SecretID}, - map[string]string{"q-sign-time": keyTime}, - - //map[string]string{"success_action_status": strconv.Itoa(successCode)}, - //[]any{"content-length-range", 0, size}, - //map[string]string{"key": name}, - //map[string]string{"x-cos-security-token": c.credential.SessionToken}, - }, + "conditions": conditions, } policyJson, err := json.Marshal(policy) if err != nil { return nil, err } - //signKey := hmacSha1val(c.credential.SecretKey, keyTime) - //policyStr := hmacSha1val(sha1val(signKey), string(policyJson)) + signKey := hmacSha1val(c.credential.SecretKey, keyTime) + strToSign := sha1val(string(policyJson)) + signature := hmacSha1val(signKey, strToSign) - policyStr := base64.StdEncoding.EncodeToString(policyJson) - h := hmac.New(sha1.New, []byte(c.credential.SecretKey)) - if _, err := io.WriteString(h, policyStr); err != nil { - return nil, err - } fd := &s3.FormData{ URL: c.client.BaseURL.BucketURL.String(), File: "file", Expires: expiration, FormData: map[string]string{ - "key": name, - "policy": policyStr, - "q-sign-algorithm": "sha1", - "q-ak": c.credential.SecretID, - "q-key-time": keyTime, - "q-signature": hex.EncodeToString(h.Sum(nil)), + "policy": base64.StdEncoding.EncodeToString(policyJson), + "q-sign-algorithm": "sha1", + "q-ak": c.credential.SecretID, + "q-key-time": keyTime, + "q-signature": signature, + "key": name, + "success_action_status": strconv.Itoa(successCode), }, SuccessCodes: []int{successCode}, } + if contentType != "" { + fd.FormData["Content-Type"] = contentType + } if c.credential.SessionToken != "" { fd.FormData["x-cos-security-token"] = c.credential.SessionToken } - // 2019-08-30 17:38:12 return fd, nil } diff --git a/pkg/common/db/s3/minio/minio.go b/pkg/common/db/s3/minio/minio.go index 430052058..4bd2acb3b 100644 --- a/pkg/common/db/s3/minio/minio.go +++ b/pkg/common/db/s3/minio/minio.go @@ -456,8 +456,10 @@ func (m *Minio) FormData(ctx context.Context, name string, size int64, contentTy if err := policy.SetExpires(expires); err != nil { return nil, err } - if err := policy.SetContentLengthRange(0, size); err != nil { - return nil, err + if size > 0 { + if err := policy.SetContentLengthRange(0, size); err != nil { + return nil, err + } } if err := policy.SetSuccessStatusAction(strconv.Itoa(successCode)); err != nil { return nil, err diff --git a/pkg/common/db/s3/minio/minio_test.go b/pkg/common/db/s3/minio/minio_test.go deleted file mode 100644 index 0780c4021..000000000 --- a/pkg/common/db/s3/minio/minio_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package minio - -import ( - "bytes" - "context" - "errors" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3" - "io" - "mime/multipart" - "net/http" - "path" - "testing" - "time" -) - -func TestName(t *testing.T) { - config.Config.Object.Minio.Bucket = "openim" - config.Config.Object.Minio.AccessKeyID = "root" - config.Config.Object.Minio.SecretAccessKey = "openIM123" - config.Config.Object.Minio.Endpoint = "http://172.16.8.38:10005" - tmp, err := NewMinio(nil) - if err != nil { - panic(err) - } - min := tmp.(*Minio) - - text := []byte("hello world!") - name := "posttest.txt" - - u, err := min.FormData(context.Background(), "posttest.txt", int64(len(text)), "image/png", time.Second*1000) - if err != nil { - panic(err) - } - t.Log(u.URL) - for k, v := range u.FormData { - t.Log(k, v) - } - if err := PostFile(u, name, text); err != nil { - t.Error(err) - } -} - -func PostFile(fd *s3.FormData, name string, data []byte) error { - var body bytes.Buffer - writer := multipart.NewWriter(&body) - for k, v := range fd.FormData { - if err := writer.WriteField(k, v); err != nil { - return err - } - } - fileWriter, err := writer.CreateFormFile(fd.File, path.Base(name)) - if err != nil { - return err - } - if _, err := fileWriter.Write(data); err != nil { - return nil - } - defer writer.Close() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - reqBody := body.Bytes() - req, err := http.NewRequestWithContext(ctx, http.MethodPost, fd.URL, bytes.NewReader(reqBody)) - req.Header.Set("Content-Type", writer.FormDataContentType()) - req.ContentLength = int64(len(reqBody)) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK { - return errors.New(string(respBody)) - } - return nil -} diff --git a/pkg/common/db/s3/oss/oss.go b/pkg/common/db/s3/oss/oss.go index 710991bb6..8fa2a538e 100644 --- a/pkg/common/db/s3/oss/oss.go +++ b/pkg/common/db/s3/oss/oss.go @@ -338,13 +338,16 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, func (o *OSS) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { // https://help.aliyun.com/zh/oss/developer-reference/postobject?spm=a2c4g.11186623.0.0.1cb83cebkP55nn expires := time.Now().Add(duration) + conditions := []any{ + map[string]string{"bucket": o.bucket.BucketName}, + map[string]string{"key": name}, + } + if size > 0 { + conditions = append(conditions, []any{"content-length-range", 0, size}) + } policy := map[string]any{ "expiration": expires.Format("2006-01-02T15:04:05.000Z"), - "conditions": []any{ - map[string]string{"bucket": o.bucket.BucketName}, - []any{"content-length-range", 0, size}, - //map[string]string{"content-type": contentType}, - }, + "conditions": conditions, } policyJson, err := json.Marshal(policy) if err != nil { @@ -368,5 +371,8 @@ func (o *OSS) FormData(ctx context.Context, name string, size int64, contentType }, SuccessCodes: []int{successCode}, } + if contentType != "" { + fd.FormData["x-oss-content-type"] = contentType + } return fd, nil }