From 8d437a451c8436fee2d56ad120b3c1f965ff3cbd Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Thu, 23 Jan 2020 12:38:32 +0800 Subject: [PATCH] Feat: COS credential / upload callback --- go.mod | 1 + go.sum | 8 ++ middleware/auth.go | 16 +++ models/policy.go | 4 +- pkg/filesystem/driver/cos/handller.go | 171 ++++++++++++++++++++++++++ pkg/filesystem/filesystem.go | 16 +++ pkg/filesystem/upload.go | 1 + pkg/serializer/upload.go | 6 +- routers/controllers/callback.go | 11 ++ routers/router.go | 6 + service/callback/upload.go | 46 ++++++- 11 files changed, 282 insertions(+), 4 deletions(-) create mode 100644 pkg/filesystem/driver/cos/handller.go diff --git a/go.mod b/go.mod index 5f6c186..11de968 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1 github.com/smartystreets/goconvey v1.6.4 // indirect github.com/stretchr/testify v1.4.0 + github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac github.com/upyun/go-sdk v2.1.0+incompatible golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 gopkg.in/go-playground/validator.v8 v8.18.2 diff --git a/go.sum b/go.sum index 9f0b83a..f773e66 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,7 @@ cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7h github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= @@ -73,8 +74,11 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/certificate-transparency-go v1.0.21 h1:Yf1aXowfZ2nuboBsg7iYGLmwsOARdV86pfH3g95wXmE= github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= @@ -126,6 +130,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2 h1:daZqE/T/yEoKIQNd3rwNeLsiS0VpZFfJulR0t/rtgAE= github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2/go.mod h1:wAQCKEc5bDujxKRmbT6/vTnTt5CjStQ8bRfPWUuz/iY= +github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= +github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= @@ -165,6 +171,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac h1:PSBhZblOjdwH7SIVgcue+7OlnLHkM45KuScLZ+PiVbQ= +github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac/go.mod h1:wQBO5HdAkLjj2q6XQiIfDSP8DXDNrppDRw2Kp/1BODA= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= diff --git a/middleware/auth.go b/middleware/auth.go index 2167b3e..731b517 100644 --- a/middleware/auth.go +++ b/middleware/auth.go @@ -293,3 +293,19 @@ func OneDriveCallbackAuth() gin.HandlerFunc { c.Next() } } + +// COSCallbackAuth 腾讯云COS回调签名验证 +// TODO 解耦 测试 +func COSCallbackAuth() gin.HandlerFunc { + return func(c *gin.Context) { + // 验证key并查找用户 + resp, _ := uploadCallbackCheck(c) + if resp.Code != 0 { + c.JSON(401, serializer.QiniuCallbackFailed{Error: resp.Msg}) + c.Abort() + return + } + + c.Next() + } +} diff --git a/models/policy.go b/models/policy.go index 6d7ecf5..5cf53f1 100644 --- a/models/policy.go +++ b/models/policy.go @@ -145,7 +145,7 @@ func (policy Policy) getOriginNameRule(origin string) string { return "$(fname)" case "local", "remote": return origin - case "oss": + case "oss", "cos": // OSS会将${filename}自动替换为原始文件名 return "${filename}" case "upyun": @@ -201,7 +201,7 @@ func (policy *Policy) GetUploadURL() string { controller, _ = url.Parse("/api/v3/file/upload") case "remote": controller, _ = url.Parse("/api/v3/slave/upload") - case "oss": + case "oss", "cos": return policy.BaseURL case "upyun": return "http://v0.api.upyun.com/" + policy.BucketName diff --git a/pkg/filesystem/driver/cos/handller.go b/pkg/filesystem/driver/cos/handller.go new file mode 100644 index 0000000..e688182 --- /dev/null +++ b/pkg/filesystem/driver/cos/handller.go @@ -0,0 +1,171 @@ +package cos + +import ( + "context" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + model "github.com/HFO4/cloudreve/models" + "github.com/HFO4/cloudreve/pkg/filesystem/fsctx" + "github.com/HFO4/cloudreve/pkg/filesystem/response" + "github.com/HFO4/cloudreve/pkg/serializer" + cossdk "github.com/tencentyun/cos-go-sdk-v5" + "io" + "net/url" + "time" +) + +// UploadPolicy 腾讯云COS上传策略 +type UploadPolicy struct { + Expiration string `json:"expiration"` + Conditions []interface{} `json:"conditions"` +} + +// MetaData 文件元信息 +type MetaData struct { + Size uint64 + CallbackKey string + CallbackURL string +} + +// Driver 腾讯云COS适配器模板 +type Driver struct { + Policy *model.Policy + Client *cossdk.Client +} + +// Get 获取文件 +func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) { + return nil, errors.New("未实现") +} + +// Put 将文件流保存到指定目录 +func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error { + return errors.New("未实现") +} + +// Delete 删除一个或多个文件， +// 返回未删除的文件，及遇到的最后一个错误 +func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) { + return []string{}, errors.New("未实现") +} + +// Thumb 获取文件缩略图 +func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) { + return nil, errors.New("未实现") +} + +// Source 获取外链URL +func (handler Driver) Source( + ctx context.Context, + path string, + baseURL url.URL, + ttl int64, + isDownload bool, + speed int, +) (string, error) { + return "", errors.New("未实现") +} + +// Token 获取上传策略和认证Token +func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) { + // 读取上下文中生成的存储路径 + savePath, ok := ctx.Value(fsctx.SavePathCtx).(string) + if !ok { + return serializer.UploadCredential{}, errors.New("无法获取存储路径") + } + + // 生成回调地址 + siteURL := model.GetSiteURL() + apiBaseURI, _ := url.Parse("/api/v3/callback/cos/" + key) + apiURL := siteURL.ResolveReference(apiBaseURI).String() + + // 上传策略 + startTime := time.Now() + endTime := startTime.Add(time.Duration(TTL) * time.Second) + keyTime := fmt.Sprintf("%d;%d", startTime.Unix(), endTime.Unix()) + postPolicy := UploadPolicy{ + Expiration: endTime.UTC().Format(time.RFC3339), + Conditions: []interface{}{ + map[string]string{"bucket": handler.Policy.BucketName}, + map[string]string{"$key": savePath}, + map[string]string{"x-cos-meta-callback": apiURL}, + map[string]string{"x-cos-meta-key": key}, + []interface{}{"content-length-range", 0, handler.Policy.MaxSize}, + map[string]string{"q-sign-algorithm": "sha1"}, + map[string]string{"q-ak": handler.Policy.AccessKey}, + map[string]string{"q-sign-time": keyTime}, + }, + } + + res, err := handler.getUploadCredential(ctx, postPolicy, keyTime) + if err == nil { + res.Callback = apiURL + res.Key = key + } + + return res, err + +} + +// Meta 获取文件信息 +func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) { + res, err := handler.Client.Object.Head(ctx, path, &cossdk.ObjectHeadOptions{}) + if err != nil { + return nil, err + } + return &MetaData{ + Size: uint64(res.ContentLength), + CallbackKey: res.Header.Get("x-cos-meta-key"), + CallbackURL: res.Header.Get("x-cos-meta-callback"), + }, nil +} + +func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, keyTime string) (serializer.UploadCredential, error) { + // 读取上下文中生成的存储路径 + savePath, ok := ctx.Value(fsctx.SavePathCtx).(string) + if !ok { + return serializer.UploadCredential{}, errors.New("无法获取存储路径") + } + + // 编码上传策略 + policyJSON, err := json.Marshal(policy) + if err != nil { + return serializer.UploadCredential{}, err + } + policyEncoded := base64.StdEncoding.EncodeToString(policyJSON) + + // 签名上传策略 + hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey)) + _, err = io.WriteString(hmacSign, keyTime) + if err != nil { + return serializer.UploadCredential{}, err + } + signKey := fmt.Sprintf("%x", hmacSign.Sum(nil)) + + sha1Sign := sha1.New() + _, err = sha1Sign.Write(policyJSON) + if err != nil { + return serializer.UploadCredential{}, err + } + stringToSign := fmt.Sprintf("%x", sha1Sign.Sum(nil)) + + // 最终签名 + hmacFinalSign := hmac.New(sha1.New, []byte(signKey)) + _, err = hmacFinalSign.Write([]byte(stringToSign)) + if err != nil { + return serializer.UploadCredential{}, err + } + signature := hmacFinalSign.Sum(nil) + + return serializer.UploadCredential{ + Policy: policyEncoded, + Path: savePath, + AccessKey: handler.Policy.AccessKey, + Token: fmt.Sprintf("%x", signature), + KeyTime: keyTime, + }, nil +} diff --git a/pkg/filesystem/filesystem.go b/pkg/filesystem/filesystem.go index a2599c3..12584a2 100644 --- a/pkg/filesystem/filesystem.go +++ b/pkg/filesystem/filesystem.go @@ -6,6 +6,7 @@ import ( "github.com/HFO4/cloudreve/models" "github.com/HFO4/cloudreve/pkg/auth" "github.com/HFO4/cloudreve/pkg/conf" + "github.com/HFO4/cloudreve/pkg/filesystem/driver/cos" "github.com/HFO4/cloudreve/pkg/filesystem/driver/local" "github.com/HFO4/cloudreve/pkg/filesystem/driver/onedrive" "github.com/HFO4/cloudreve/pkg/filesystem/driver/oss" @@ -16,7 +17,9 @@ import ( "github.com/HFO4/cloudreve/pkg/request" "github.com/HFO4/cloudreve/pkg/serializer" "github.com/gin-gonic/gin" + cossdk "github.com/tencentyun/cos-go-sdk-v5" "io" + "net/http" "net/url" "sync" ) @@ -191,6 +194,19 @@ func (fs *FileSystem) DispatchHandler() error { HTTPClient: request.HTTPClient{}, } return err + case "cos": + u, _ := url.Parse(currentPolicy.Server) + b := &cossdk.BaseURL{BucketURL: u} + fs.Handler = cos.Driver{ + Policy: currentPolicy, + Client: cossdk.NewClient(b, &http.Client{ + Transport: &cossdk.AuthorizationTransport{ + SecretID: currentPolicy.AccessKey, + SecretKey: currentPolicy.SecretKey, + }, + }), + } + return nil default: return ErrUnknownPolicyType } diff --git a/pkg/filesystem/upload.go b/pkg/filesystem/upload.go index 9733987..9e84868 100644 --- a/pkg/filesystem/upload.go +++ b/pkg/filesystem/upload.go @@ -165,6 +165,7 @@ func (fs *FileSystem) GetUploadToken(ctx context.Context, path string, size uint err = cache.Set( "callback_"+callbackKey, serializer.UploadSession{ + Key: callbackKey, UID: fs.User.ID, PolicyID: fs.User.GetPolicyID(), VirtualPath: path, diff --git a/pkg/serializer/upload.go b/pkg/serializer/upload.go index d031987..c473483 100644 --- a/pkg/serializer/upload.go +++ b/pkg/serializer/upload.go @@ -20,12 +20,16 @@ type UploadPolicy struct { type UploadCredential struct { Token string `json:"token"` Policy string `json:"policy"` - Path string `json:"path"` + Path string `json:"path"` // 存储路径 AccessKey string `json:"ak"` + KeyTime string `json:"key_time,omitempty"` // COS用有效期 + Callback string `json:"callback,omitempty"` // 回调地址 + Key string `json:"key,omitempty"` // 文件标识符，通常为回调key } // UploadSession 上传会话 type UploadSession struct { + Key string UID uint PolicyID uint VirtualPath string diff --git a/routers/controllers/callback.go b/routers/controllers/callback.go index 34f5bb1..c74796e 100644 --- a/routers/controllers/callback.go +++ b/routers/controllers/callback.go @@ -76,3 +76,14 @@ func OneDriveCallback(c *gin.Context) { c.JSON(200, ErrorResponse(err)) } } + +// COSCallback COS上传完成客户端回调 +func COSCallback(c *gin.Context) { + var callbackBody callback.COSCallback + if err := c.ShouldBindQuery(&callbackBody); err == nil { + res := callbackBody.PreProcess(c) + c.JSON(200, res) + } else { + c.JSON(200, ErrorResponse(err)) + } +} diff --git a/routers/router.go b/routers/router.go index e6a4155..bd24bc6 100644 --- a/routers/router.go +++ b/routers/router.go @@ -159,6 +159,12 @@ func InitMasterRouter() *gin.Engine { controllers.OneDriveCallback, ) } + // 腾讯云COS策略上传回调 + callback.GET( + "cos/:key", + middleware.COSCallbackAuth(), + controllers.COSCallback, + ) } // 需要登录保护的 diff --git a/service/callback/upload.go b/service/callback/upload.go index 226b79a..b261474 100644 --- a/service/callback/upload.go +++ b/service/callback/upload.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/HFO4/cloudreve/pkg/filesystem" + "github.com/HFO4/cloudreve/pkg/filesystem/driver/cos" "github.com/HFO4/cloudreve/pkg/filesystem/driver/local" "github.com/HFO4/cloudreve/pkg/filesystem/driver/onedrive" "github.com/HFO4/cloudreve/pkg/filesystem/fsctx" @@ -52,6 +53,12 @@ type OneDriveCallback struct { Meta *onedrive.FileInfo } +// COSCallback COS 客户端回调正文 +type COSCallback struct { + Bucket string `form:"bucket"` + Etag string `form:"etag"` +} + // GetBody 返回回调正文 func (service UpyunCallbackService) GetBody(session *serializer.UploadSession) serializer.UploadCallback { res := serializer.UploadCallback{ @@ -90,6 +97,16 @@ func (service OneDriveCallback) GetBody(session *serializer.UploadSession) seria } } +// GetBody 返回回调正文 +func (service COSCallback) GetBody(session *serializer.UploadSession) serializer.UploadCallback { + return serializer.UploadCallback{ + Name: session.Name, + SourceName: session.SavePath, + PicInfo: "", + Size: session.Size, + } +} + // ProcessCallback 处理上传结果回调 func ProcessCallback(service CallbackProcessService, c *gin.Context) serializer.Response { // 创建文件系统 @@ -168,9 +185,36 @@ func (service *OneDriveCallback) PreProcess(c *gin.Context) serializer.Response // 验证与回调会话中是否一致 actualPath := strings.TrimPrefix(callbackSession.SavePath, "/") if callbackSession.Size != info.Size || info.GetSourcePath() != actualPath { - // TODO 删除文件信息 + fs.Handler.(onedrive.Driver).Client.Delete(context.Background(), []string{info.GetSourcePath()}) return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err) } service.Meta = info return ProcessCallback(service, c) } + +// PreProcess 对COS客户端回调进行预处理 +func (service *COSCallback) PreProcess(c *gin.Context) serializer.Response { + // 创建文件系统 + fs, err := filesystem.NewFileSystemFromCallback(c) + if err != nil { + return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err) + } + defer fs.Recycle() + + // 获取回调会话 + callbackSessionRaw, _ := c.Get("callbackSession") + callbackSession := callbackSessionRaw.(*serializer.UploadSession) + + // 获取文件信息 + info, err := fs.Handler.(cos.Driver).Meta(context.Background(), callbackSession.SavePath) + if err != nil { + return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err) + } + + // 验证实际文件信息与回调会话中是否一致 + if callbackSession.Size != info.Size || callbackSession.Key != info.CallbackKey { + return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err) + } + + return ProcessCallback(service, c) +}