From dd198becce06e571e2ce1854bb869e6e8288ac5a Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Thu, 16 Jan 2020 13:36:13 +0800 Subject: [PATCH] Feat: client-upload file in oss --- go.mod | 1 + go.sum | 3 + middleware/auth.go | 18 ++++- middleware/auth_test.go | 50 ++++++++++++++ models/policy.go | 25 ++++--- models/policy_test.go | 73 ++++++++++++-------- pkg/filesystem/hooks_test.go | 29 ++++++++ pkg/filesystem/oss/handller.go | 118 +++++++++++++++++++++++++++++++- pkg/serializer/upload.go | 6 +- routers/controllers/callback.go | 13 +++- routers/router.go | 6 ++ service/callback/upload.go | 6 +- service/explorer/upload.go | 2 +- 13 files changed, 302 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index 2b01584..f7c56e5 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.12 require ( github.com/DATA-DOG/go-sqlmock v1.3.3 + github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7 github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4 github.com/fatih/color v1.7.0 diff --git a/go.sum b/go.sum index a230d6e..6b5dfa1 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7I github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible h1:A3oZlWPD/Poa19FvNbw+Zu4yKAurDBTjlRDilYGBiS4= +github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff h1:RmdPFa+slIr4SCBg4st/l/vZWVe9QJKMXGO60Bxbe04= @@ -213,6 +215,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/middleware/auth.go b/middleware/auth.go index 448a237..79788a2 100644 --- a/middleware/auth.go +++ b/middleware/auth.go @@ -165,7 +165,6 @@ func RemoteCallbackAuth() gin.HandlerFunc { } // QiniuCallbackAuth 七牛回调签名验证 -// TODO 测试 func QiniuCallbackAuth() gin.HandlerFunc { return func(c *gin.Context) { // 验证key并查找用户 @@ -194,3 +193,20 @@ func QiniuCallbackAuth() gin.HandlerFunc { c.Next() } } + +// OSSCallbackAuth 阿里云OSS回调签名验证 +func OSSCallbackAuth() gin.HandlerFunc { + return func(c *gin.Context) { + // 验证key并查找用户 + resp, _ := uploadCallbackCheck(c) + if resp.Code != 0 { + c.JSON(401, serializer.QiniuCallbackFailed{Error: resp.Msg}) + c.Abort() + return + } + + // TODO 验证OSS给出的签名 + + c.Next() + } +} diff --git a/middleware/auth_test.go b/middleware/auth_test.go index 67fb250..ca595b9 100644 --- a/middleware/auth_test.go +++ b/middleware/auth_test.go @@ -424,3 +424,53 @@ func TestQiniuCallbackAuth(t *testing.T) { asserts.True(c.IsAborted()) } } + +func TestOSSCallbackAuth(t *testing.T) { + asserts := assert.New(t) + rec := httptest.NewRecorder() + AuthFunc := OSSCallbackAuth() + + // Callback Key 相关验证失败 + { + c, _ := gin.CreateTestContext(rec) + c.Params = []gin.Param{ + {"key", "testOSSBackRemote"}, + } + c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/testQiniuBackRemote", nil) + AuthFunc(c) + asserts.True(c.IsAborted()) + } + + // 成功 + { + cache.Set( + "callback_testCallBackOSS", + serializer.UploadSession{ + UID: 1, + PolicyID: 2, + VirtualPath: "/", + }, + 0, + ) + cache.Deletes([]string{"1"}, "policy_") + mock.ExpectQuery("SELECT(.+)users(.+)"). + WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1)) + mock.ExpectQuery("SELECT(.+)groups(.+)"). + WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[2]")) + mock.ExpectQuery("SELECT(.+)policies(.+)"). + WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123")) + c, _ := gin.CreateTestContext(rec) + c.Params = []gin.Param{ + {"key", "testCallBackOSS"}, + } + c.Request, _ = http.NewRequest("POST", "/api/v3/callback/qiniu/testCallBackOSS", nil) + mac := qbox.NewMac("123", "123") + token, err := mac.SignRequest(c.Request) + asserts.NoError(err) + c.Request.Header["Authorization"] = []string{"QBox " + token} + AuthFunc(c) + asserts.NoError(mock.ExpectationsWereMet()) + asserts.False(c.IsAborted()) + } + +} diff --git a/models/policy.go b/models/policy.go index 39d8daa..c742b3e 100644 --- a/models/policy.go +++ b/models/policy.go @@ -111,7 +111,7 @@ func (policy *Policy) GeneratePath(uid uint, origin string) string { func (policy *Policy) GenerateFileName(uid uint, origin string) string { // 未开启自动重命名时，直接返回原始文件名 if !policy.AutoRename { - return origin + return policy.getOriginNameRule(origin) } fileRule := policy.FileNameRule @@ -125,28 +125,31 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string { "{date}": time.Now().Format("20060102"), } + replaceTable["{originname}"] = policy.getOriginNameRule(origin) + + fileRule = util.Replace(replaceTable, fileRule) + return fileRule +} + +func (policy Policy) getOriginNameRule(origin string) string { // 部分存储策略可以使用{origin}代表原始文件名 if origin == "" { // 如果上游未传回原始文件名，则使用占位符，让云存储端替换 switch policy.Type { case "qiniu": // 七牛会将$(fname)自动替换为原始文件名 - replaceTable["{originname}"] = "$(fname)" + return "$(fname)" case "local", "remote": - replaceTable["{originname}"] = origin + return origin case "oss": // OSS会将${filename}自动替换为原始文件名 - replaceTable["{originname}"] = "${filename}" + return "${filename}" case "upyun": // Upyun会将{filename}{.suffix}自动替换为原始文件名 - replaceTable["{originname}"] = "{filename}{.suffix}" + return "{filename}{.suffix}" } - } else { - replaceTable["{originname}"] = origin } - - fileRule = util.Replace(replaceTable, fileRule) - return fileRule + return origin } // IsDirectlyPreview 返回此策略下文件是否可以直接预览（不需要重定向） @@ -172,6 +175,8 @@ func (policy *Policy) GetUploadURL() string { controller, _ = url.Parse("/api/v3/file/upload") case "remote": controller, _ = url.Parse("/api/v3/slave/upload") + case "oss": + return policy.BaseURL default: controller, _ = url.Parse("") } diff --git a/models/policy_test.go b/models/policy_test.go index d30d7a4..30c3ae0 100644 --- a/models/policy_test.go +++ b/models/policy_test.go @@ -91,47 +91,62 @@ func TestPolicy_GeneratePath(t *testing.T) { func TestPolicy_GenerateFileName(t *testing.T) { asserts := assert.New(t) - testPolicy := Policy{ - AutoRename: true, + // 重命名关闭 + { + testPolicy := Policy{ + AutoRename: false, + } + testPolicy.FileNameRule = "{randomkey16}" + asserts.Equal("123.txt", testPolicy.GenerateFileName(1, "123.txt")) + + testPolicy.Type = "oss" + asserts.Equal("${filename}", testPolicy.GenerateFileName(1, "")) } - testPolicy.FileNameRule = "{randomkey16}" - asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 16) + // 重命名开启 + { + testPolicy := Policy{ + AutoRename: true, + } - testPolicy.FileNameRule = "{randomkey8}" - asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 8) + testPolicy.FileNameRule = "{randomkey16}" + asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 16) - testPolicy.FileNameRule = "{timestamp}" - asserts.Equal(testPolicy.GenerateFileName(1, "123.txt"), strconv.FormatInt(time.Now().Unix(), 10)) + testPolicy.FileNameRule = "{randomkey8}" + asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 8) - testPolicy.FileNameRule = "{uid}" - asserts.Equal(testPolicy.GenerateFileName(1, "123.txt"), strconv.Itoa(int(1))) + testPolicy.FileNameRule = "{timestamp}" + asserts.Equal(testPolicy.GenerateFileName(1, "123.txt"), strconv.FormatInt(time.Now().Unix(), 10)) - testPolicy.FileNameRule = "{datetime}" - asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 14) + testPolicy.FileNameRule = "{uid}" + asserts.Equal(testPolicy.GenerateFileName(1, "123.txt"), strconv.Itoa(int(1))) - testPolicy.FileNameRule = "{date}" - asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 8) + testPolicy.FileNameRule = "{datetime}" + asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 14) - testPolicy.FileNameRule = "123{date}ss{datetime}" - asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 27) + testPolicy.FileNameRule = "{date}" + asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 8) - // 支持{originname}的策略 - testPolicy.Type = "local" - testPolicy.FileNameRule = "123{originname}" - asserts.Equal("123123.txt", testPolicy.GenerateFileName(1, "123.txt")) + testPolicy.FileNameRule = "123{date}ss{datetime}" + asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 27) - testPolicy.Type = "qiniu" - testPolicy.FileNameRule = "{uid}123{originname}" - asserts.Equal("1123123.txt", testPolicy.GenerateFileName(1, "123.txt")) + // 支持{originname}的策略 + testPolicy.Type = "local" + testPolicy.FileNameRule = "123{originname}" + asserts.Equal("123123.txt", testPolicy.GenerateFileName(1, "123.txt")) - testPolicy.Type = "oss" - testPolicy.FileNameRule = "{uid}123{originname}" - asserts.Equal("1123${filename}", testPolicy.GenerateFileName(1, "")) + testPolicy.Type = "qiniu" + testPolicy.FileNameRule = "{uid}123{originname}" + asserts.Equal("1123123.txt", testPolicy.GenerateFileName(1, "123.txt")) - testPolicy.Type = "upyun" - testPolicy.FileNameRule = "{uid}123{originname}" - asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, "")) + testPolicy.Type = "oss" + testPolicy.FileNameRule = "{uid}123{originname}" + asserts.Equal("1123${filename}", testPolicy.GenerateFileName(1, "")) + + testPolicy.Type = "upyun" + testPolicy.FileNameRule = "{uid}123{originname}" + asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, "")) + } } diff --git a/pkg/filesystem/hooks_test.go b/pkg/filesystem/hooks_test.go index d21456f..8eadb2e 100644 --- a/pkg/filesystem/hooks_test.go +++ b/pkg/filesystem/hooks_test.go @@ -414,6 +414,35 @@ func TestHookClearFileSize(t *testing.T) { } +func TestHookUpdateSourceName(t *testing.T) { + asserts := assert.New(t) + fs := &FileSystem{User: &model.User{ + Model: gorm.Model{ID: 1}, + }} + + // 成功 + { + originFile := model.File{ + Model: gorm.Model{ID: 1}, + SourceName: "new.txt", + } + ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile) + mock.ExpectBegin() + mock.ExpectExec("UPDATE(.+)").WithArgs("new.txt", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + err := HookUpdateSourceName(ctx, fs) + asserts.NoError(mock.ExpectationsWereMet()) + asserts.NoError(err) + } + + // 上下文错误 + { + ctx := context.Background() + err := HookUpdateSourceName(ctx, fs) + asserts.Error(err) + } +} + func TestGenericAfterUpdate(t *testing.T) { asserts := assert.New(t) fs := &FileSystem{User: &model.User{ diff --git a/pkg/filesystem/oss/handller.go b/pkg/filesystem/oss/handller.go index 450c301..43689f7 100644 --- a/pkg/filesystem/oss/handller.go +++ b/pkg/filesystem/oss/handller.go @@ -2,17 +2,64 @@ package oss import ( "context" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/json" "errors" + "fmt" model "github.com/HFO4/cloudreve/models" + "github.com/HFO4/cloudreve/pkg/filesystem/fsctx" "github.com/HFO4/cloudreve/pkg/filesystem/response" "github.com/HFO4/cloudreve/pkg/serializer" + "github.com/aliyun/aliyun-oss-go-sdk/oss" "io" "net/url" + "path" + "time" ) +// UploadPolicy 阿里云OSS上传策略 +type UploadPolicy struct { + Expiration string `json:"expiration"` + Conditions []interface{} `json:"conditions"` +} + +// CallbackPolicy 回调策略 +type CallbackPolicy struct { + CallbackURL string `json:"callbackUrl"` + CallbackBody string `json:"callbackBody"` + CallbackBodyType string `json:"callbackBodyType"` +} + // Handler 阿里云OSS策略适配器 type Handler struct { Policy *model.Policy + client *oss.Client + bucket *oss.Bucket +} + +// InitOSSClient 初始化OSS鉴权客户端 +func (handler *Handler) InitOSSClient() error { + if handler.Policy == nil { + return errors.New("存储策略为空") + } + + // 初始化客户端 + client, err := oss.New(handler.Policy.Server, handler.Policy.AccessKey, handler.Policy.SecretKey) + if err != nil { + return err + } + handler.client = client + + // 初始化存储桶 + bucket, err := client.Bucket(handler.Policy.BucketName) + if err != nil { + return err + } + handler.bucket = bucket + + return nil } // Get 获取文件 @@ -50,5 +97,74 @@ func (handler Handler) Source( // Token 获取上传策略和认证Token func (handler Handler) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) { - return serializer.UploadCredential{}, errors.New("未实现") + // 读取上下文中生成的存储路径 + savePath, ok := ctx.Value(fsctx.SavePathCtx).(string) + if !ok { + return serializer.UploadCredential{}, errors.New("无法获取存储路径") + } + + // 生成回调地址 + siteURL := model.GetSiteURL() + apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + key) + apiURL := siteURL.ResolveReference(apiBaseURI) + + // 回调策略 + callbackPolicy := CallbackPolicy{ + CallbackURL: apiURL.String(), + CallbackBody: `{"name":${x:fname},"source_name":${object},"size":${size},"pic_info":"${imageInfo.width},${imageInfo.height}"}`, + CallbackBodyType: "application/json", + } + + // 上传策略 + postPolicy := UploadPolicy{ + Expiration: time.Now().UTC().Add(time.Duration(TTL) * time.Second).Format(time.RFC3339), + Conditions: []interface{}{ + map[string]string{"bucket": handler.Policy.BucketName}, + []string{"starts-with", "$key", path.Dir(savePath)}, + []interface{}{"content-length-range", 0, handler.Policy.MaxSize}, + }, + } + + return handler.getUploadCredential(ctx, postPolicy, callbackPolicy, TTL) +} + +func (handler Handler) getUploadCredential(ctx context.Context, policy UploadPolicy, callback CallbackPolicy, TTL int64) (serializer.UploadCredential, error) { + // 读取上下文中生成的存储路径 + savePath, ok := ctx.Value(fsctx.SavePathCtx).(string) + if !ok { + return serializer.UploadCredential{}, errors.New("无法获取存储路径") + } + + // 处理回调策略 + callbackPolicyEncoded := "" + if callback.CallbackURL != "" { + callbackPolicyJSON, err := json.Marshal(callback) + if err != nil { + return serializer.UploadCredential{}, err + } + callbackPolicyEncoded = base64.StdEncoding.EncodeToString(callbackPolicyJSON) + policy.Conditions = append(policy.Conditions, map[string]string{"callback": callbackPolicyEncoded}) + } + + // 编码上传策略 + policyJSON, err := json.Marshal(policy) + if err != nil { + return serializer.UploadCredential{}, err + } + policyEncoded := base64.StdEncoding.EncodeToString(policyJSON) + + // 签名上传策略 + hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey)) + _, err = io.WriteString(hmacSign, policyEncoded) + if err != nil { + return serializer.UploadCredential{}, err + } + signature := base64.StdEncoding.EncodeToString(hmacSign.Sum(nil)) + + return serializer.UploadCredential{ + Policy: fmt.Sprintf("%s:%s", callbackPolicyEncoded, policyEncoded), + Path: savePath, + AccessKey: handler.Policy.AccessKey, + Token: signature, + }, nil } diff --git a/pkg/serializer/upload.go b/pkg/serializer/upload.go index 5ae0fad..98d6471 100644 --- a/pkg/serializer/upload.go +++ b/pkg/serializer/upload.go @@ -18,8 +18,10 @@ type UploadPolicy struct { // UploadCredential 返回给客户端的上传凭证 type UploadCredential struct { - Token string `json:"token"` - Policy string `json:"policy"` + Token string `json:"token"` + Policy string `json:"policy"` + Path string `json:"path"` + AccessKey string `json:"ak"` } // UploadSession 上传会话 diff --git a/routers/controllers/callback.go b/routers/controllers/callback.go index 830596a..d26e327 100644 --- a/routers/controllers/callback.go +++ b/routers/controllers/callback.go @@ -19,7 +19,7 @@ func RemoteCallback(c *gin.Context) { // QiniuCallback 七牛上传回调 func QiniuCallback(c *gin.Context) { - var callbackBody callback.QiniuUploadCallbackService + var callbackBody callback.UploadCallbackService if err := c.ShouldBindJSON(&callbackBody); err == nil { res := callback.ProcessCallback(callbackBody, c) if res.Code != 0 { @@ -31,3 +31,14 @@ func QiniuCallback(c *gin.Context) { c.JSON(401, ErrorResponse(err)) } } + +// OSSCallback 阿里云OSS上传回调 +func OSSCallback(c *gin.Context) { + var callbackBody callback.UploadCallbackService + if err := c.ShouldBindJSON(&callbackBody); err == nil { + res := callback.ProcessCallback(callbackBody, c) + c.JSON(200, res) + } else { + c.JSON(200, ErrorResponse(err)) + } +} diff --git a/routers/router.go b/routers/router.go index d5b1f37..699694b 100644 --- a/routers/router.go +++ b/routers/router.go @@ -138,6 +138,12 @@ func InitMasterRouter() *gin.Engine { middleware.QiniuCallbackAuth(), controllers.QiniuCallback, ) + // 阿里云OSS策略上传回调 + callback.POST( + "oss/:key", + middleware.OSSCallbackAuth(), + controllers.OSSCallback, + ) } // 需要登录保护的 diff --git a/service/callback/upload.go b/service/callback/upload.go index d891dda..2657579 100644 --- a/service/callback/upload.go +++ b/service/callback/upload.go @@ -25,8 +25,8 @@ func (service RemoteUploadCallbackService) GetBody() serializer.UploadCallback { return service.Data } -// QiniuUploadCallbackService 七牛存储上传回调请求服务 -type QiniuUploadCallbackService struct { +// UploadCallbackService 云存储上传回调请求服务 +type UploadCallbackService struct { Name string `json:"name"` SourceName string `json:"source_name"` PicInfo string `json:"pic_info"` @@ -34,7 +34,7 @@ type QiniuUploadCallbackService struct { } // GetBody 返回回调正文 -func (service QiniuUploadCallbackService) GetBody() serializer.UploadCallback { +func (service UploadCallbackService) GetBody() serializer.UploadCallback { return serializer.UploadCallback{ Name: service.Name, SourceName: service.SourceName, diff --git a/service/explorer/upload.go b/service/explorer/upload.go index 1a5be04..5ec15b4 100644 --- a/service/explorer/upload.go +++ b/service/explorer/upload.go @@ -11,7 +11,7 @@ import ( // UploadCredentialService 获取上传凭证服务 type UploadCredentialService struct { Path string `form:"path" binding:"required"` - Size uint64 `form:"size" binding:"required,min=0"` + Size uint64 `form:"size" binding:"min=0"` } // Get 获取新的上传凭证