From 67a4af8d5c20528d7260bf7e9f7778e1605ef301 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 25 Dec 2023 18:36:14 +0800 Subject: [PATCH] s3 form data --- go.mod | 4 +- go.sum | 10 ++++ pkg/common/db/s3/cos/cos.go | 84 +++++++++++++++++++++++++++- pkg/common/db/s3/minio/minio.go | 20 ++++--- pkg/common/db/s3/minio/minio_test.go | 66 ++++++++++++++++++---- pkg/common/db/s3/oss/oss.go | 41 +++++++++++++- pkg/common/db/s3/s3.go | 11 ++-- 7 files changed, 208 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 4a0520e67..8e8b425a4 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require github.com/google/uuid v1.3.1 require ( github.com/IBM/sarama v1.41.3 - github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible + github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible github.com/go-redis/redis v6.15.9+incompatible github.com/redis/go-redis/v9 v9.2.1 github.com/tencentyun/cos-go-sdk-v5 v0.7.45 @@ -132,7 +132,7 @@ require ( golang.org/x/oauth2 v0.13.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.3.0 // indirect + golang.org/x/time v0.5.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect diff --git a/go.sum b/go.sum index 9684c1079..6faa5d97c 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,8 @@ github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBb github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible h1:Sg/2xHwDrioHpxTN6WMiwbXTpUEinBpHsN7mG21Rc2k= github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible h1:so4m5rRA32Tc5GgKg/5gKUu0CRsYmVO3ThMP6T3CwLc= +github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -89,6 +91,7 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= @@ -196,8 +199,10 @@ github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -250,6 +255,7 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mozillazg/go-httpheader v0.4.0 h1:aBn6aRXtFzyDLZ4VIRLsZbbJloagQfMnCiYgOq6hK4w= github.com/mozillazg/go-httpheader v0.4.0/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -378,6 +384,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -451,6 +458,8 @@ golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -462,6 +471,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/common/db/s3/cos/cos.go b/pkg/common/db/s3/cos/cos.go index ca8eb1694..15d7c57d3 100644 --- a/pkg/common/db/s3/cos/cos.go +++ b/pkg/common/db/s3/cos/cos.go @@ -16,8 +16,14 @@ package cos import ( "context" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/hex" + "encoding/json" "errors" "fmt" + "io" "net/http" "net/url" "strconv" @@ -44,6 +50,8 @@ const ( imageWebp = "webp" ) +const successCode = http.StatusOK + const ( videoSnapshotImagePng = "png" videoSnapshotImageJpg = "jpg" @@ -328,5 +336,79 @@ func (c *Cos) getPresignedURL(ctx context.Context, name string, expire time.Dura } func (c *Cos) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { - return nil, errors.New("cos temporarily not supported") + //res, _, err := c.client.Object.ListUploads(ctx, &cos.ObjectListUploadsOptions{}) + //if err != nil { + // return nil, err + //} + now := time.Now() + expiration := now.Add(duration) + keyTime := fmt.Sprintf("%d;%d", now.Unix(), expiration.Unix()) + conditions := []any{ + map[string]string{"success_action_status": strconv.Itoa(successCode)}, + []any{"content-length-range", 0, size}, + map[string]string{"q-sign-time": keyTime}, + map[string]string{"key": name}, + map[string]string{"q-sign-algorithm": "sha1"}, + } + if c.credential.SessionToken != "" { + conditions = append(conditions, map[string]string{"x-cos-security-token": c.credential.SessionToken}) + } + policy := map[string]any{ + "expiration": expiration.Format("2006-01-02T15:04:05.000Z"), + "conditions": []any{ + //map[string]string{"bucket": res.Bucket}, + //map[string]string{"acl": "default"}, + map[string]string{"q-sign-algorithm": "sha1"}, + map[string]string{"q-ak": c.credential.SecretID}, + map[string]string{"q-sign-time": keyTime}, + + //map[string]string{"success_action_status": strconv.Itoa(successCode)}, + //[]any{"content-length-range", 0, size}, + //map[string]string{"key": name}, + //map[string]string{"x-cos-security-token": c.credential.SessionToken}, + }, + } + policyJson, err := json.Marshal(policy) + if err != nil { + return nil, err + } + //signKey := hmacSha1val(c.credential.SecretKey, keyTime) + //policyStr := hmacSha1val(sha1val(signKey), string(policyJson)) + + policyStr := base64.StdEncoding.EncodeToString(policyJson) + h := hmac.New(sha1.New, []byte(c.credential.SecretKey)) + if _, err := io.WriteString(h, policyStr); err != nil { + return nil, err + } + fd := &s3.FormData{ + URL: c.client.BaseURL.BucketURL.String(), + File: "file", + Expires: expiration, + FormData: map[string]string{ + "key": name, + "policy": policyStr, + "q-sign-algorithm": "sha1", + "q-ak": c.credential.SecretID, + "q-key-time": keyTime, + "q-signature": hex.EncodeToString(h.Sum(nil)), + }, + SuccessCodes: []int{successCode}, + } + if c.credential.SessionToken != "" { + fd.FormData["x-cos-security-token"] = c.credential.SessionToken + } + // 2019-08-30 17:38:12 + return fd, nil +} + +func hmacSha1val(key, msg string) string { + v := hmac.New(sha1.New, []byte(key)) + v.Write([]byte(msg)) + return hex.EncodeToString(v.Sum(nil)) +} + +func sha1val(msg string) string { + sha1Hash := sha1.New() + sha1Hash.Write([]byte(msg)) + return hex.EncodeToString(sha1Hash.Sum(nil)) } diff --git a/pkg/common/db/s3/minio/minio.go b/pkg/common/db/s3/minio/minio.go index 914e2ae54..430052058 100644 --- a/pkg/common/db/s3/minio/minio.go +++ b/pkg/common/db/s3/minio/minio.go @@ -57,6 +57,8 @@ const ( imageThumbnailPath = "openim/thumbnail" ) +const successCode = http.StatusOK + func NewMinio(cache cache.MinioCache) (s3.Interface, error) { u, err := url.Parse(config.Config.Object.Minio.Endpoint) if err != nil { @@ -457,23 +459,27 @@ func (m *Minio) FormData(ctx context.Context, name string, size int64, contentTy if err := policy.SetContentLengthRange(0, size); err != nil { return nil, err } + if err := policy.SetSuccessStatusAction(strconv.Itoa(successCode)); err != nil { + return nil, err + } if contentType != "" { if err := policy.SetContentType(contentType); err != nil { return nil, err } } - if err := policy.SetBucket(config.Config.Object.Minio.Bucket); err != nil { + if err := policy.SetBucket(m.bucket); err != nil { return nil, err } u, fd, err := m.core.PresignedPostPolicy(ctx, policy) if err != nil { - panic(err) + return nil, err } return &s3.FormData{ - URL: u.String(), - File: "file", - Header: nil, - FormData: fd, - Expires: expires, + URL: u.String(), + File: "file", + Header: nil, + FormData: fd, + Expires: expires, + SuccessCodes: []int{successCode}, }, nil } diff --git a/pkg/common/db/s3/minio/minio_test.go b/pkg/common/db/s3/minio/minio_test.go index 2c721eb25..0780c4021 100644 --- a/pkg/common/db/s3/minio/minio_test.go +++ b/pkg/common/db/s3/minio/minio_test.go @@ -1,9 +1,15 @@ package minio import ( + "bytes" "context" - "github.com/minio/minio-go/v7" + "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3" + "io" + "mime/multipart" + "net/http" + "path" "testing" "time" ) @@ -18,20 +24,56 @@ func TestName(t *testing.T) { panic(err) } min := tmp.(*Minio) - cli := min.core.Client - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - policy := minio.NewPostPolicy() - _ = policy.SetExpires(time.Now().Add(time.Hour)) - _ = policy.SetKey("test.txt") - _ = policy.SetBucket(config.Config.Object.Minio.Bucket) - policy.SetContentType("text/plain") - u, fd, err := cli.PresignedPostPolicy(ctx, policy) + + text := []byte("hello world!") + name := "posttest.txt" + + u, err := min.FormData(context.Background(), "posttest.txt", int64(len(text)), "image/png", time.Second*1000) if err != nil { panic(err) } - t.Log(u) - for k, v := range fd { + t.Log(u.URL) + for k, v := range u.FormData { t.Log(k, v) } + if err := PostFile(u, name, text); err != nil { + t.Error(err) + } +} + +func PostFile(fd *s3.FormData, name string, data []byte) error { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + for k, v := range fd.FormData { + if err := writer.WriteField(k, v); err != nil { + return err + } + } + fileWriter, err := writer.CreateFormFile(fd.File, path.Base(name)) + if err != nil { + return err + } + if _, err := fileWriter.Write(data); err != nil { + return nil + } + defer writer.Close() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + reqBody := body.Bytes() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, fd.URL, bytes.NewReader(reqBody)) + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.ContentLength = int64(len(reqBody)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return errors.New(string(respBody)) + } + return nil } diff --git a/pkg/common/db/s3/oss/oss.go b/pkg/common/db/s3/oss/oss.go index a4c34fa82..710991bb6 100644 --- a/pkg/common/db/s3/oss/oss.go +++ b/pkg/common/db/s3/oss/oss.go @@ -16,8 +16,13 @@ package oss import ( "context" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/json" "errors" "fmt" + "io" "net/http" "net/url" "reflect" @@ -45,6 +50,8 @@ const ( imageWebp = "webp" ) +const successCode = http.StatusOK + const ( videoSnapshotImagePng = "png" videoSnapshotImageJpg = "jpg" @@ -329,5 +336,37 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, } func (o *OSS) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { - return nil, errors.New("oss temporarily not supported") + // https://help.aliyun.com/zh/oss/developer-reference/postobject?spm=a2c4g.11186623.0.0.1cb83cebkP55nn + expires := time.Now().Add(duration) + policy := map[string]any{ + "expiration": expires.Format("2006-01-02T15:04:05.000Z"), + "conditions": []any{ + map[string]string{"bucket": o.bucket.BucketName}, + []any{"content-length-range", 0, size}, + //map[string]string{"content-type": contentType}, + }, + } + policyJson, err := json.Marshal(policy) + if err != nil { + return nil, err + } + policyStr := base64.StdEncoding.EncodeToString(policyJson) + h := hmac.New(sha1.New, []byte(o.credentials.GetAccessKeySecret())) + if _, err := io.WriteString(h, policyStr); err != nil { + return nil, err + } + fd := &s3.FormData{ + URL: o.bucketURL, + File: "file", + Expires: expires, + FormData: map[string]string{ + "key": name, + "policy": policyStr, + "OSSAccessKeyId": o.credentials.GetAccessKeyID(), + "success_action_status": strconv.Itoa(successCode), + "signature": base64.StdEncoding.EncodeToString(h.Sum(nil)), + }, + SuccessCodes: []int{successCode}, + } + return fd, nil } diff --git a/pkg/common/db/s3/s3.go b/pkg/common/db/s3/s3.go index 3ca29943b..0352004b5 100644 --- a/pkg/common/db/s3/s3.go +++ b/pkg/common/db/s3/s3.go @@ -75,11 +75,12 @@ type CopyObjectInfo struct { } type FormData struct { - URL string `json:"url"` - File string `json:"file"` - Header http.Header `json:"header"` - FormData map[string]string `json:"form"` - Expires time.Time `json:"expires"` + URL string `json:"url"` + File string `json:"file"` + Header http.Header `json:"header"` + FormData map[string]string `json:"form"` + Expires time.Time `json:"expires"` + SuccessCodes []int `json:"successActionStatus"` } type SignPart struct {