feat: s3 FormData upload (#1614)

* upgrade package and rtc convert

* upgrade package and rtc convert

* upgrade package and rtc convert

* upgrade package and rtc convert

* friend user

* s3 form data

* s3 form data

* s3 form data

* s3 form data

* s3 form data

* s3 form data

* s3 form data

* s3 form data

* s3 form data
pull/1612/head
chao 6 months ago committed by GitHub
parent 47dd6b17f6
commit cd1235fb32
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -4,6 +4,8 @@ go 1.19
require (
firebase.google.com/go v3.13.0+incompatible
github.com/OpenIMSDK/protocol v0.0.39
github.com/OpenIMSDK/tools v0.0.21
github.com/bwmarrin/snowflake v0.3.0 // indirect
github.com/dtm-labs/rockscache v0.1.1
github.com/gin-gonic/gin v1.9.1
@ -33,8 +35,6 @@ require github.com/google/uuid v1.3.1
require (
github.com/IBM/sarama v1.41.3
github.com/OpenIMSDK/protocol v0.0.36
github.com/OpenIMSDK/tools v0.0.21
github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible
github.com/go-redis/redis v6.15.9+incompatible
github.com/redis/go-redis/v9 v9.2.1
@ -133,7 +133,7 @@ require (
golang.org/x/oauth2 v0.13.0 // indirect
golang.org/x/sys v0.14.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect
@ -156,5 +156,3 @@ require (
golang.org/x/crypto v0.14.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
)
replace github.com/OpenIMSDK/protocol v0.0.36 => github.com/luhaoling/protocol v0.0.0-20231222100538-d625562d53d5

@ -18,6 +18,8 @@ firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIw
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/IBM/sarama v1.41.3 h1:MWBEJ12vHC8coMjdEXFq/6ftO6DUZnQlFYcxtOJFa7c=
github.com/IBM/sarama v1.41.3/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ=
github.com/OpenIMSDK/protocol v0.0.39 h1:DfvFcNGBcfj2vtT7W3uw4U/ipnI7NecTzQdlSYGuQz8=
github.com/OpenIMSDK/protocol v0.0.39/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y=
github.com/OpenIMSDK/tools v0.0.21 h1:iTapc2mIEVH/xl5Nd6jfwPub11Pgp44tVcE1rjB3a48=
github.com/OpenIMSDK/tools v0.0.21/go.mod h1:eg+q4A34Qmu73xkY0mt37FHGMCMfC6CtmOnm0kFEGFI=
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
@ -225,8 +227,6 @@ github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205Ah
github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw=
github.com/lithammer/shortuuid v3.0.0+incompatible h1:NcD0xWW/MZYXEHa6ITy6kaXN5nwm/V115vj2YXfhS0w=
github.com/lithammer/shortuuid v3.0.0+incompatible/go.mod h1:FR74pbAuElzOUuenUHTK2Tciko1/vKuIKS9dSkDrA4w=
github.com/luhaoling/protocol v0.0.0-20231222100538-d625562d53d5 h1:nmrJmAgQsCAxKgw109kaTcBV4rMWDRvqOson0ehw708=
github.com/luhaoling/protocol v0.0.0-20231222100538-d625562d53d5/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
@ -453,8 +453,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

@ -171,6 +171,8 @@ func NewGinRouter(discov discoveryregistry.SvcDiscoveryRegistry, rdb redis.Unive
objectGroup.POST("/auth_sign", t.AuthSign)
objectGroup.POST("/complete_multipart_upload", t.CompleteMultipartUpload)
objectGroup.POST("/access_url", t.AccessURL)
objectGroup.POST("/initiate_form_data", t.InitiateFormData)
objectGroup.POST("/complete_form_data", t.CompleteFormData)
objectGroup.GET("/*name", t.ObjectRedirect)
}
// Message

@ -71,6 +71,14 @@ func (o *ThirdApi) AccessURL(c *gin.Context) {
a2r.Call(third.ThirdClient.AccessURL, o.Client, c)
}
func (o *ThirdApi) InitiateFormData(c *gin.Context) {
a2r.Call(third.ThirdClient.InitiateFormData, o.Client, c)
}
func (o *ThirdApi) CompleteFormData(c *gin.Context) {
a2r.Call(third.ThirdClient.CompleteFormData, o.Client, c)
}
func (o *ThirdApi) ObjectRedirect(c *gin.Context) {
name := c.Param("name")
if name == "" {

@ -54,8 +54,7 @@ type friendServer struct {
}
func (s *friendServer) UpdateFriends(ctx context.Context, req *pbfriend.UpdateFriendsReq) (*pbfriend.UpdateFriendsResp, error) {
//TODO implement me
panic("implement me")
return nil, errs.ErrInternalServer.Wrap("not implemented")
}
func Start(client registry.SvcDiscoveryRegistry, server *grpc.Server) error {

@ -16,6 +16,12 @@ package third
import (
"context"
"encoding/base64"
"encoding/hex"
"encoding/json"
"github.com/google/uuid"
"github.com/openimsdk/open-im-server/v3/pkg/authverify"
"path"
"strconv"
"time"
@ -179,6 +185,113 @@ func (t *thirdServer) AccessURL(ctx context.Context, req *third.AccessURLReq) (*
}, nil
}
func (t *thirdServer) InitiateFormData(ctx context.Context, req *third.InitiateFormDataReq) (*third.InitiateFormDataResp, error) {
if req.Name == "" {
return nil, errs.ErrArgs.Wrap("name is empty")
}
if req.Size <= 0 {
return nil, errs.ErrArgs.Wrap("size must be greater than 0")
}
if err := checkUploadName(ctx, req.Name); err != nil {
return nil, err
}
var duration time.Duration
opUserID := mcontext.GetOpUserID(ctx)
var key string
if authverify.IsManagerUserID(opUserID) {
if req.Millisecond <= 0 {
duration = time.Minute * 10
} else {
duration = time.Millisecond * time.Duration(req.Millisecond)
}
if req.Absolute {
key = req.Name
}
} else {
duration = time.Minute * 10
}
uid, err := uuid.NewRandom()
if err != nil {
return nil, err
}
if key == "" {
date := time.Now().Format("20060102")
key = path.Join(cont.DirectPath, date, opUserID, hex.EncodeToString(uid[:])+path.Ext(req.Name))
}
mate := FormDataMate{
Name: req.Name,
Size: req.Size,
ContentType: req.ContentType,
Group: req.Group,
Key: key,
}
mateData, err := json.Marshal(&mate)
if err != nil {
return nil, err
}
resp, err := t.s3dataBase.FormData(ctx, key, req.Size, req.ContentType, duration)
if err != nil {
return nil, err
}
return &third.InitiateFormDataResp{
Id: base64.RawStdEncoding.EncodeToString(mateData),
Url: resp.URL,
File: resp.File,
Header: toPbMapArray(resp.Header),
FormData: resp.FormData,
Expires: resp.Expires.UnixMilli(),
SuccessCodes: utils.Slice(resp.SuccessCodes, func(code int) int32 {
return int32(code)
}),
}, nil
}
func (t *thirdServer) CompleteFormData(ctx context.Context, req *third.CompleteFormDataReq) (*third.CompleteFormDataResp, error) {
if req.Id == "" {
return nil, errs.ErrArgs.Wrap("id is empty")
}
data, err := base64.RawStdEncoding.DecodeString(req.Id)
if err != nil {
return nil, errs.ErrArgs.Wrap("invalid id " + err.Error())
}
var mate FormDataMate
if err := json.Unmarshal(data, &mate); err != nil {
return nil, errs.ErrArgs.Wrap("invalid id " + err.Error())
}
if err := checkUploadName(ctx, mate.Name); err != nil {
return nil, err
}
info, err := t.s3dataBase.StatObject(ctx, mate.Key)
if err != nil {
return nil, err
}
if info.Size > 0 && info.Size != mate.Size {
return nil, errs.ErrData.Wrap("file size mismatch")
}
obj := &relation.ObjectModel{
Name: mate.Name,
UserID: mcontext.GetOpUserID(ctx),
Hash: "etag_" + info.ETag,
Key: info.Key,
Size: info.Size,
ContentType: mate.ContentType,
Group: mate.Group,
CreateTime: time.Now(),
}
if err := t.s3dataBase.SetObject(ctx, obj); err != nil {
return nil, err
}
return &third.CompleteFormDataResp{Url: t.apiAddress(mate.Name)}, nil
}
func (t *thirdServer) apiAddress(name string) string {
return t.apiURL + name
}
type FormDataMate struct {
Name string `json:"name"`
Size int64 `json:"size"`
ContentType string `json:"contentType"`
Group string `json:"group"`
Key string `json:"key"`
}

@ -101,16 +101,6 @@ type thirdServer struct {
defaultExpire time.Duration
}
func (t *thirdServer) InitiateFormData(ctx context.Context, req *third.InitiateFormDataReq) (*third.InitiateFormDataResp, error) {
//TODO implement me
panic("implement me")
}
func (t *thirdServer) CompleteFormData(ctx context.Context, req *third.CompleteFormDataReq) (*third.CompleteFormDataResp, error) {
//TODO implement me
panic("implement me")
}
func (t *thirdServer) FcmUpdateToken(ctx context.Context, req *third.FcmUpdateTokenReq) (resp *third.FcmUpdateTokenResp, err error) {
err = t.thirdDatabase.FcmUpdateToken(ctx, req.Account, int(req.PlatformID), req.FcmToken, req.ExpireTime)
if err != nil {

@ -29,6 +29,9 @@ import (
)
func toPbMapArray(m map[string][]string) []*third.KeyValues {
if len(m) == 0 {
return nil
}
res := make([]*third.KeyValues, 0, len(m))
for key := range m {
res = append(res, &third.KeyValues{

@ -57,6 +57,10 @@ type userServer struct {
RegisterCenter registry.SvcDiscoveryRegistry
}
func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUserInfoExReq) (*pbuser.UpdateUserInfoExResp, error) {
return nil, errs.ErrInternalServer.Wrap("not implemented")
}
func Start(client registry.SvcDiscoveryRegistry, server *grpc.Server) error {
rdb, err := cache.NewRedis()
if err != nil {
@ -484,11 +488,6 @@ func (s *userServer) SearchNotificationAccount(ctx context.Context, req *pbuser.
return &pbuser.SearchNotificationAccountResp{Total: total, NotificationAccounts: accounts}, nil
}
func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUserInfoExReq) (*pbuser.UpdateUserInfoExResp, error) {
//TODO implement me
panic("implement me")
}
func (s *userServer) GetNotificationAccount(ctx context.Context, req *pbuser.GetNotificationAccountReq) (*pbuser.GetNotificationAccountResp, error) {
if req.UserID == "" {
return nil, errs.ErrArgs.Wrap("userID is empty")

@ -45,7 +45,7 @@ type CmdOpts struct {
func WithCronTaskLogName() func(*CmdOpts) {
return func(opts *CmdOpts) {
opts.loggerPrefixName = "OpenIM.CronTask.log.all"
opts.loggerPrefixName = "openim.crontask.log.all"
}
}

@ -35,6 +35,8 @@ type S3Database interface {
CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error)
AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (time.Time, string, error)
SetObject(ctx context.Context, info *relation.ObjectModel) error
StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error)
FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error)
}
func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj relation.ObjectInfoModelInterface) S3Database {
@ -100,3 +102,11 @@ func (s *s3Database) AccessURL(ctx context.Context, name string, expire time.Dur
}
return expireTime, rawURL, nil
}
func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
return s.s3.StatObject(ctx, name)
}
func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
return s.s3.FormData(ctx, name, size, contentType, duration)
}

@ -17,6 +17,7 @@ package cont
const (
hashPath = "openim/data/hash/"
tempPath = "openim/temp/"
DirectPath = "openim/direct"
UploadTypeMultipart = 1 // 分片上传
UploadTypePresigned = 2 // 预签名上传
partSeparator = ","

@ -279,3 +279,7 @@ func (c *Controller) AccessURL(ctx context.Context, name string, expire time.Dur
}
return c.impl.AccessURL(ctx, name, expire, opt)
}
func (c *Controller) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
return c.impl.FormData(ctx, name, size, contentType, duration)
}

@ -16,6 +16,11 @@ package cos
import (
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/http"
@ -44,6 +49,8 @@ const (
imageWebp = "webp"
)
const successCode = http.StatusOK
const (
videoSnapshotImagePng = "png"
videoSnapshotImageJpg = "jpg"
@ -326,3 +333,65 @@ func (c *Cos) getPresignedURL(ctx context.Context, name string, expire time.Dura
}
return c.client.Object.GetObjectURL(name), nil
}
func (c *Cos) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
// https://cloud.tencent.com/document/product/436/14690
now := time.Now()
expiration := now.Add(duration)
keyTime := fmt.Sprintf("%d;%d", now.Unix(), expiration.Unix())
conditions := []any{
map[string]string{"q-sign-algorithm": "sha1"},
map[string]string{"q-ak": c.credential.SecretID},
map[string]string{"q-sign-time": keyTime},
map[string]string{"key": name},
}
if contentType != "" {
conditions = append(conditions, map[string]string{"Content-Type": contentType})
}
policy := map[string]any{
"expiration": expiration.Format("2006-01-02T15:04:05.000Z"),
"conditions": conditions,
}
policyJson, err := json.Marshal(policy)
if err != nil {
return nil, err
}
signKey := hmacSha1val(c.credential.SecretKey, keyTime)
strToSign := sha1val(string(policyJson))
signature := hmacSha1val(signKey, strToSign)
fd := &s3.FormData{
URL: c.client.BaseURL.BucketURL.String(),
File: "file",
Expires: expiration,
FormData: map[string]string{
"policy": base64.StdEncoding.EncodeToString(policyJson),
"q-sign-algorithm": "sha1",
"q-ak": c.credential.SecretID,
"q-key-time": keyTime,
"q-signature": signature,
"key": name,
"success_action_status": strconv.Itoa(successCode),
},
SuccessCodes: []int{successCode},
}
if contentType != "" {
fd.FormData["Content-Type"] = contentType
}
if c.credential.SessionToken != "" {
fd.FormData["x-cos-security-token"] = c.credential.SessionToken
}
return fd, nil
}
func hmacSha1val(key, msg string) string {
v := hmac.New(sha1.New, []byte(key))
v.Write([]byte(msg))
return hex.EncodeToString(v.Sum(nil))
}
func sha1val(msg string) string {
sha1Hash := sha1.New()
sha1Hash.Write([]byte(msg))
return hex.EncodeToString(sha1Hash.Sum(nil))
}

@ -57,6 +57,8 @@ const (
imageThumbnailPath = "openim/thumbnail"
)
const successCode = http.StatusOK
func NewMinio(cache cache.MinioCache) (s3.Interface, error) {
u, err := url.Parse(config.Config.Object.Minio.Endpoint)
if err != nil {
@ -441,3 +443,51 @@ func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]
}
return io.ReadAll(io.LimitReader(object, limit))
}
func (m *Minio) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
if err := m.initMinio(ctx); err != nil {
return nil, err
}
policy := minio.NewPostPolicy()
if err := policy.SetKey(name); err != nil {
return nil, err
}
expires := time.Now().Add(duration)
if err := policy.SetExpires(expires); err != nil {
return nil, err
}
if size > 0 {
if err := policy.SetContentLengthRange(0, size); err != nil {
return nil, err
}
}
if err := policy.SetSuccessStatusAction(strconv.Itoa(successCode)); err != nil {
return nil, err
}
if contentType != "" {
if err := policy.SetContentType(contentType); err != nil {
return nil, err
}
}
if err := policy.SetBucket(m.bucket); err != nil {
return nil, err
}
u, fd, err := m.core.PresignedPostPolicy(ctx, policy)
if err != nil {
return nil, err
}
sign, err := url.Parse(m.signEndpoint)
if err != nil {
return nil, err
}
u.Scheme = sign.Scheme
u.Host = sign.Host
return &s3.FormData{
URL: u.String(),
File: "file",
Header: nil,
FormData: fd,
Expires: expires,
SuccessCodes: []int{successCode},
}, nil
}

@ -16,8 +16,13 @@ package oss
import (
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"reflect"
@ -45,6 +50,8 @@ const (
imageWebp = "webp"
)
const successCode = http.StatusOK
const (
videoSnapshotImagePng = "png"
videoSnapshotImageJpg = "jpg"
@ -327,3 +334,45 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration,
params := getURLParams(*o.bucket.Client.Conn, rawParams)
return getURL(o.um, o.bucket.BucketName, name, params).String(), nil
}
func (o *OSS) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
// https://help.aliyun.com/zh/oss/developer-reference/postobject?spm=a2c4g.11186623.0.0.1cb83cebkP55nn
expires := time.Now().Add(duration)
conditions := []any{
map[string]string{"bucket": o.bucket.BucketName},
map[string]string{"key": name},
}
if size > 0 {
conditions = append(conditions, []any{"content-length-range", 0, size})
}
policy := map[string]any{
"expiration": expires.Format("2006-01-02T15:04:05.000Z"),
"conditions": conditions,
}
policyJson, err := json.Marshal(policy)
if err != nil {
return nil, err
}
policyStr := base64.StdEncoding.EncodeToString(policyJson)
h := hmac.New(sha1.New, []byte(o.credentials.GetAccessKeySecret()))
if _, err := io.WriteString(h, policyStr); err != nil {
return nil, err
}
fd := &s3.FormData{
URL: o.bucketURL,
File: "file",
Expires: expires,
FormData: map[string]string{
"key": name,
"policy": policyStr,
"OSSAccessKeyId": o.credentials.GetAccessKeyID(),
"success_action_status": strconv.Itoa(successCode),
"signature": base64.StdEncoding.EncodeToString(h.Sum(nil)),
},
SuccessCodes: []int{successCode},
}
if contentType != "" {
fd.FormData["x-oss-content-type"] = contentType
}
return fd, nil
}

@ -74,6 +74,15 @@ type CopyObjectInfo struct {
ETag string `json:"etag"`
}
type FormData struct {
URL string `json:"url"`
File string `json:"file"`
Header http.Header `json:"header"`
FormData map[string]string `json:"form"`
Expires time.Time `json:"expires"`
SuccessCodes []int `json:"successActionStatus"`
}
type SignPart struct {
PartNumber int `json:"partNumber"`
URL string `json:"url"`
@ -152,4 +161,6 @@ type Interface interface {
ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*ListUploadedPartsResult, error)
AccessURL(ctx context.Context, name string, expire time.Duration, opt *AccessURLOption) (string, error)
FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*FormData, error)
}

@ -432,7 +432,7 @@ func computeApproximateRequestSize(r *http.Request) int {
}
s += len(r.Host)
// r.Form and r.MultipartForm are assumed to be included in r.URL.
// r.FormData and r.MultipartForm are assumed to be included in r.URL.
if r.ContentLength != -1 {
s += int(r.ContentLength)

Loading…
Cancel
Save