msb-public
/
Open-IM-Server
mirror of https://github.com/OpenIMSDK/Open-IM-Server
2
0
Code Issues Projects Releases Wiki Activity

s3 form data

Browse Source
pull/1614/head
withchao 2 years ago
parent 00f0d9fb42
commit 67a4af8d5c
7 changed files with 208 additions and 28 deletions
Show Stats Download Patch File Download Diff File

4
go.mod
Unescape View File

@ -35,7 +35,7 @@ require github.com/google/uuid v1.3.1
require (
github.com/IBM/sarama v1.41.3
github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible
github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible
github.com/go-redis/redis v6.15.9+incompatible
github.com/redis/go-redis/v9 v9.2.1
github.com/tencentyun/cos-go-sdk-v5 v0.7.45
@ -132,7 +132,7 @@ require (
golang.org/x/oauth2 v0.13.0 // indirect
golang.org/x/sys v0.14.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect

10
go.sum
Unescape View File

@ -25,6 +25,8 @@ github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBb
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible h1:Sg/2xHwDrioHpxTN6WMiwbXTpUEinBpHsN7mG21Rc2k=
github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible h1:so4m5rRA32Tc5GgKg/5gKUu0CRsYmVO3ThMP6T3CwLc=
github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
@ -89,6 +91,7 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
@ -196,8 +199,10 @@ github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@ -250,6 +255,7 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
github.com/mozillazg/go-httpheader v0.4.0 h1:aBn6aRXtFzyDLZ4VIRLsZbbJloagQfMnCiYgOq6hK4w=
github.com/mozillazg/go-httpheader v0.4.0/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@ -378,6 +384,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -451,6 +458,8 @@ golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@ -462,6 +471,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

84
pkg/common/db/s3/cos/cos.go
Unescape View File

@ -16,8 +16,14 @@ package cos
import (
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
@ -44,6 +50,8 @@ const (
imageWebp = "webp"
)
const successCode = http.StatusOK
const (
videoSnapshotImagePng = "png"
videoSnapshotImageJpg = "jpg"
@ -328,5 +336,79 @@ func (c *Cos) getPresignedURL(ctx context.Context, name string, expire time.Dura
}
func (c *Cos) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
return nil, errors.New("cos temporarily not supported")
//res, _, err := c.client.Object.ListUploads(ctx, &cos.ObjectListUploadsOptions{})
//if err != nil {
// return nil, err
//}
now := time.Now()
expiration := now.Add(duration)
keyTime := fmt.Sprintf("%d;%d", now.Unix(), expiration.Unix())
conditions := []any{
map[string]string{"success_action_status": strconv.Itoa(successCode)},
[]any{"content-length-range", 0, size},
map[string]string{"q-sign-time": keyTime},
map[string]string{"key": name},
map[string]string{"q-sign-algorithm": "sha1"},
}
if c.credential.SessionToken != "" {
conditions = append(conditions, map[string]string{"x-cos-security-token": c.credential.SessionToken})
}
policy := map[string]any{
"expiration": expiration.Format("2006-01-02T15:04:05.000Z"),
"conditions": []any{
//map[string]string{"bucket": res.Bucket},
//map[string]string{"acl": "default"},
map[string]string{"q-sign-algorithm": "sha1"},
map[string]string{"q-ak": c.credential.SecretID},
map[string]string{"q-sign-time": keyTime},
//map[string]string{"success_action_status": strconv.Itoa(successCode)},
//[]any{"content-length-range", 0, size},
//map[string]string{"key": name},
//map[string]string{"x-cos-security-token": c.credential.SessionToken},
},
}
policyJson, err := json.Marshal(policy)
if err != nil {
return nil, err
}
//signKey := hmacSha1val(c.credential.SecretKey, keyTime)
//policyStr := hmacSha1val(sha1val(signKey), string(policyJson))
policyStr := base64.StdEncoding.EncodeToString(policyJson)
h := hmac.New(sha1.New, []byte(c.credential.SecretKey))
if _, err := io.WriteString(h, policyStr); err != nil {
return nil, err
}
fd := &s3.FormData{
URL: c.client.BaseURL.BucketURL.String(),
File: "file",
Expires: expiration,
FormData: map[string]string{
"key": name,
"policy": policyStr,
"q-sign-algorithm": "sha1",
"q-ak": c.credential.SecretID,
"q-key-time": keyTime,
"q-signature": hex.EncodeToString(h.Sum(nil)),
},
SuccessCodes: []int{successCode},
}
if c.credential.SessionToken != "" {
fd.FormData["x-cos-security-token"] = c.credential.SessionToken
}
// 2019-08-30 17:38:12
return fd, nil
}
func hmacSha1val(key, msg string) string {
v := hmac.New(sha1.New, []byte(key))
v.Write([]byte(msg))
return hex.EncodeToString(v.Sum(nil))
}
func sha1val(msg string) string {
sha1Hash := sha1.New()
sha1Hash.Write([]byte(msg))
return hex.EncodeToString(sha1Hash.Sum(nil))
}

10
pkg/common/db/s3/minio/minio.go
Unescape View File

@ -57,6 +57,8 @@ const (
imageThumbnailPath = "openim/thumbnail"
)
const successCode = http.StatusOK
func NewMinio(cache cache.MinioCache) (s3.Interface, error) {
u, err := url.Parse(config.Config.Object.Minio.Endpoint)
if err != nil {
@ -457,17 +459,20 @@ func (m *Minio) FormData(ctx context.Context, name string, size int64, contentTy
if err := policy.SetContentLengthRange(0, size); err != nil {
return nil, err
}
if err := policy.SetSuccessStatusAction(strconv.Itoa(successCode)); err != nil {
return nil, err
}
if contentType != "" {
if err := policy.SetContentType(contentType); err != nil {
return nil, err
}
}
if err := policy.SetBucket(config.Config.Object.Minio.Bucket); err != nil {
if err := policy.SetBucket(m.bucket); err != nil {
return nil, err
}
u, fd, err := m.core.PresignedPostPolicy(ctx, policy)
if err != nil {
panic(err)
return nil, err
}
return &s3.FormData{
URL: u.String(),
@ -475,5 +480,6 @@ func (m *Minio) FormData(ctx context.Context, name string, size int64, contentTy
Header: nil,
FormData: fd,
Expires: expires,
SuccessCodes: []int{successCode},
}, nil
}

66
pkg/common/db/s3/minio/minio_test.go
Unescape View File

@ -1,9 +1,15 @@
package minio
import (
"bytes"
"context"
"github.com/minio/minio-go/v7"
"errors"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
"io"
"mime/multipart"
"net/http"
"path"
"testing"
"time"
)
@ -18,20 +24,56 @@ func TestName(t *testing.T) {
panic(err)
}
min := tmp.(*Minio)
cli := min.core.Client
ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
defer cancel()
policy := minio.NewPostPolicy()
_ = policy.SetExpires(time.Now().Add(time.Hour))
_ = policy.SetKey("test.txt")
_ = policy.SetBucket(config.Config.Object.Minio.Bucket)
policy.SetContentType("text/plain")
u, fd, err := cli.PresignedPostPolicy(ctx, policy)
text := []byte("hello world!")
name := "posttest.txt"
u, err := min.FormData(context.Background(), "posttest.txt", int64(len(text)), "image/png", time.Second*1000)
if err != nil {
panic(err)
}
t.Log(u)
for k, v := range fd {
t.Log(u.URL)
for k, v := range u.FormData {
t.Log(k, v)
}
if err := PostFile(u, name, text); err != nil {
t.Error(err)
}
}
func PostFile(fd *s3.FormData, name string, data []byte) error {
var body bytes.Buffer
writer := multipart.NewWriter(&body)
for k, v := range fd.FormData {
if err := writer.WriteField(k, v); err != nil {
return err
}
}
fileWriter, err := writer.CreateFormFile(fd.File, path.Base(name))
if err != nil {
return err
}
if _, err := fileWriter.Write(data); err != nil {
return nil
}
defer writer.Close()
ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
defer cancel()
reqBody := body.Bytes()
req, err := http.NewRequestWithContext(ctx, http.MethodPost, fd.URL, bytes.NewReader(reqBody))
req.Header.Set("Content-Type", writer.FormDataContentType())
req.ContentLength = int64(len(reqBody))
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return errors.New(string(respBody))
}
return nil
}

41
pkg/common/db/s3/oss/oss.go
Unescape View File

@ -16,8 +16,13 @@ package oss
import (
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"reflect"
@ -45,6 +50,8 @@ const (
imageWebp = "webp"
)
const successCode = http.StatusOK
const (
videoSnapshotImagePng = "png"
videoSnapshotImageJpg = "jpg"
@ -329,5 +336,37 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration,
}
func (o *OSS) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
return nil, errors.New("oss temporarily not supported")
// https://help.aliyun.com/zh/oss/developer-reference/postobject?spm=a2c4g.11186623.0.0.1cb83cebkP55nn
expires := time.Now().Add(duration)
policy := map[string]any{
"expiration": expires.Format("2006-01-02T15:04:05.000Z"),
"conditions": []any{
map[string]string{"bucket": o.bucket.BucketName},
[]any{"content-length-range", 0, size},
//map[string]string{"content-type": contentType},
},
}
policyJson, err := json.Marshal(policy)
if err != nil {
return nil, err
}
policyStr := base64.StdEncoding.EncodeToString(policyJson)
h := hmac.New(sha1.New, []byte(o.credentials.GetAccessKeySecret()))
if _, err := io.WriteString(h, policyStr); err != nil {
return nil, err
}
fd := &s3.FormData{
URL: o.bucketURL,
File: "file",
Expires: expires,
FormData: map[string]string{
"key": name,
"policy": policyStr,
"OSSAccessKeyId": o.credentials.GetAccessKeyID(),
"success_action_status": strconv.Itoa(successCode),
"signature": base64.StdEncoding.EncodeToString(h.Sum(nil)),
},
SuccessCodes: []int{successCode},
}
return fd, nil
}

1
pkg/common/db/s3/s3.go
Unescape View File

@ -80,6 +80,7 @@ type FormData struct {
Header http.Header `json:"header"`
FormData map[string]string `json:"form"`
Expires time.Time `json:"expires"`
SuccessCodes []int `json:"successActionStatus"`
}
type SignPart struct {

Write Preview
Loading…
Cancel
Save