Feat: client-upload file in oss

pull/247/head
HFO4 5 years ago
parent 0e62665d7f
commit dd198becce

@ -4,6 +4,7 @@ go 1.12
require (
github.com/DATA-DOG/go-sqlmock v1.3.3
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible
github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7
github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4
github.com/fatih/color v1.7.0

@ -10,6 +10,8 @@ github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7I
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible h1:A3oZlWPD/Poa19FvNbw+Zu4yKAurDBTjlRDilYGBiS4=
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff h1:RmdPFa+slIr4SCBg4st/l/vZWVe9QJKMXGO60Bxbe04=
@ -213,6 +215,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

@ -165,7 +165,6 @@ func RemoteCallbackAuth() gin.HandlerFunc {
}
// QiniuCallbackAuth 七牛回调签名验证
// TODO 测试
func QiniuCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
// 验证key并查找用户
@ -194,3 +193,20 @@ func QiniuCallbackAuth() gin.HandlerFunc {
c.Next()
}
}
// OSSCallbackAuth 阿里云OSS回调签名验证
func OSSCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
// 验证key并查找用户
resp, _ := uploadCallbackCheck(c)
if resp.Code != 0 {
c.JSON(401, serializer.QiniuCallbackFailed{Error: resp.Msg})
c.Abort()
return
}
// TODO 验证OSS给出的签名
c.Next()
}
}

@ -424,3 +424,53 @@ func TestQiniuCallbackAuth(t *testing.T) {
asserts.True(c.IsAborted())
}
}
func TestOSSCallbackAuth(t *testing.T) {
asserts := assert.New(t)
rec := httptest.NewRecorder()
AuthFunc := OSSCallbackAuth()
// Callback Key 相关验证失败
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testOSSBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/testQiniuBackRemote", nil)
AuthFunc(c)
asserts.True(c.IsAborted())
}
// 成功
{
cache.Set(
"callback_testCallBackOSS",
serializer.UploadSession{
UID: 1,
PolicyID: 2,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[2]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackOSS"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/qiniu/testCallBackOSS", nil)
mac := qbox.NewMac("123", "123")
token, err := mac.SignRequest(c.Request)
asserts.NoError(err)
c.Request.Header["Authorization"] = []string{"QBox " + token}
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.False(c.IsAborted())
}
}

@ -111,7 +111,7 @@ func (policy *Policy) GeneratePath(uid uint, origin string) string {
func (policy *Policy) GenerateFileName(uid uint, origin string) string {
// 未开启自动重命名时,直接返回原始文件名
if !policy.AutoRename {
return origin
return policy.getOriginNameRule(origin)
}
fileRule := policy.FileNameRule
@ -125,28 +125,31 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string {
"{date}": time.Now().Format("20060102"),
}
replaceTable["{originname}"] = policy.getOriginNameRule(origin)
fileRule = util.Replace(replaceTable, fileRule)
return fileRule
}
func (policy Policy) getOriginNameRule(origin string) string {
// 部分存储策略可以使用{origin}代表原始文件名
if origin == "" {
// 如果上游未传回原始文件名,则使用占位符,让云存储端替换
switch policy.Type {
case "qiniu":
// 七牛会将$(fname)自动替换为原始文件名
replaceTable["{originname}"] = "$(fname)"
return "$(fname)"
case "local", "remote":
replaceTable["{originname}"] = origin
return origin
case "oss":
// OSS会将${filename}自动替换为原始文件名
replaceTable["{originname}"] = "${filename}"
return "${filename}"
case "upyun":
// Upyun会将{filename}{.suffix}自动替换为原始文件名
replaceTable["{originname}"] = "{filename}{.suffix}"
return "{filename}{.suffix}"
}
} else {
replaceTable["{originname}"] = origin
}
fileRule = util.Replace(replaceTable, fileRule)
return fileRule
return origin
}
// IsDirectlyPreview 返回此策略下文件是否可以直接预览(不需要重定向)
@ -172,6 +175,8 @@ func (policy *Policy) GetUploadURL() string {
controller, _ = url.Parse("/api/v3/file/upload")
case "remote":
controller, _ = url.Parse("/api/v3/slave/upload")
case "oss":
return policy.BaseURL
default:
controller, _ = url.Parse("")
}

@ -91,47 +91,62 @@ func TestPolicy_GeneratePath(t *testing.T) {
func TestPolicy_GenerateFileName(t *testing.T) {
asserts := assert.New(t)
testPolicy := Policy{
AutoRename: true,
// 重命名关闭
{
testPolicy := Policy{
AutoRename: false,
}
testPolicy.FileNameRule = "{randomkey16}"
asserts.Equal("123.txt", testPolicy.GenerateFileName(1, "123.txt"))
testPolicy.Type = "oss"
asserts.Equal("${filename}", testPolicy.GenerateFileName(1, ""))
}
testPolicy.FileNameRule = "{randomkey16}"
asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 16)
// 重命名开启
{
testPolicy := Policy{
AutoRename: true,
}
testPolicy.FileNameRule = "{randomkey8}"
asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 8)
testPolicy.FileNameRule = "{randomkey16}"
asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 16)
testPolicy.FileNameRule = "{timestamp}"
asserts.Equal(testPolicy.GenerateFileName(1, "123.txt"), strconv.FormatInt(time.Now().Unix(), 10))
testPolicy.FileNameRule = "{randomkey8}"
asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 8)
testPolicy.FileNameRule = "{uid}"
asserts.Equal(testPolicy.GenerateFileName(1, "123.txt"), strconv.Itoa(int(1)))
testPolicy.FileNameRule = "{timestamp}"
asserts.Equal(testPolicy.GenerateFileName(1, "123.txt"), strconv.FormatInt(time.Now().Unix(), 10))
testPolicy.FileNameRule = "{datetime}"
asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 14)
testPolicy.FileNameRule = "{uid}"
asserts.Equal(testPolicy.GenerateFileName(1, "123.txt"), strconv.Itoa(int(1)))
testPolicy.FileNameRule = "{date}"
asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 8)
testPolicy.FileNameRule = "{datetime}"
asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 14)
testPolicy.FileNameRule = "123{date}ss{datetime}"
asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 27)
testPolicy.FileNameRule = "{date}"
asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 8)
// 支持{originname}的策略
testPolicy.Type = "local"
testPolicy.FileNameRule = "123{originname}"
asserts.Equal("123123.txt", testPolicy.GenerateFileName(1, "123.txt"))
testPolicy.FileNameRule = "123{date}ss{datetime}"
asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 27)
testPolicy.Type = "qiniu"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123123.txt", testPolicy.GenerateFileName(1, "123.txt"))
// 支持{originname}的策略
testPolicy.Type = "local"
testPolicy.FileNameRule = "123{originname}"
asserts.Equal("123123.txt", testPolicy.GenerateFileName(1, "123.txt"))
testPolicy.Type = "oss"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123${filename}", testPolicy.GenerateFileName(1, ""))
testPolicy.Type = "qiniu"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123123.txt", testPolicy.GenerateFileName(1, "123.txt"))
testPolicy.Type = "upyun"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, ""))
testPolicy.Type = "oss"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123${filename}", testPolicy.GenerateFileName(1, ""))
testPolicy.Type = "upyun"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, ""))
}
}

@ -414,6 +414,35 @@ func TestHookClearFileSize(t *testing.T) {
}
func TestHookUpdateSourceName(t *testing.T) {
asserts := assert.New(t)
fs := &FileSystem{User: &model.User{
Model: gorm.Model{ID: 1},
}}
// 成功
{
originFile := model.File{
Model: gorm.Model{ID: 1},
SourceName: "new.txt",
}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WithArgs("new.txt", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := HookUpdateSourceName(ctx, fs)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
// 上下文错误
{
ctx := context.Background()
err := HookUpdateSourceName(ctx, fs)
asserts.Error(err)
}
}
func TestGenericAfterUpdate(t *testing.T) {
asserts := assert.New(t)
fs := &FileSystem{User: &model.User{

@ -2,17 +2,64 @@ package oss
import (
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"io"
"net/url"
"path"
"time"
)
// UploadPolicy 阿里云OSS上传策略
type UploadPolicy struct {
Expiration string `json:"expiration"`
Conditions []interface{} `json:"conditions"`
}
// CallbackPolicy 回调策略
type CallbackPolicy struct {
CallbackURL string `json:"callbackUrl"`
CallbackBody string `json:"callbackBody"`
CallbackBodyType string `json:"callbackBodyType"`
}
// Handler 阿里云OSS策略适配器
type Handler struct {
Policy *model.Policy
client *oss.Client
bucket *oss.Bucket
}
// InitOSSClient 初始化OSS鉴权客户端
func (handler *Handler) InitOSSClient() error {
if handler.Policy == nil {
return errors.New("存储策略为空")
}
// 初始化客户端
client, err := oss.New(handler.Policy.Server, handler.Policy.AccessKey, handler.Policy.SecretKey)
if err != nil {
return err
}
handler.client = client
// 初始化存储桶
bucket, err := client.Bucket(handler.Policy.BucketName)
if err != nil {
return err
}
handler.bucket = bucket
return nil
}
// Get 获取文件
@ -50,5 +97,74 @@ func (handler Handler) Source(
// Token 获取上传策略和认证Token
func (handler Handler) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
return serializer.UploadCredential{}, errors.New("未实现")
// 读取上下文中生成的存储路径
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
}
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 回调策略
callbackPolicy := CallbackPolicy{
CallbackURL: apiURL.String(),
CallbackBody: `{"name":${x:fname},"source_name":${object},"size":${size},"pic_info":"${imageInfo.width},${imageInfo.height}"}`,
CallbackBodyType: "application/json",
}
// 上传策略
postPolicy := UploadPolicy{
Expiration: time.Now().UTC().Add(time.Duration(TTL) * time.Second).Format(time.RFC3339),
Conditions: []interface{}{
map[string]string{"bucket": handler.Policy.BucketName},
[]string{"starts-with", "$key", path.Dir(savePath)},
[]interface{}{"content-length-range", 0, handler.Policy.MaxSize},
},
}
return handler.getUploadCredential(ctx, postPolicy, callbackPolicy, TTL)
}
func (handler Handler) getUploadCredential(ctx context.Context, policy UploadPolicy, callback CallbackPolicy, TTL int64) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
}
// 处理回调策略
callbackPolicyEncoded := ""
if callback.CallbackURL != "" {
callbackPolicyJSON, err := json.Marshal(callback)
if err != nil {
return serializer.UploadCredential{}, err
}
callbackPolicyEncoded = base64.StdEncoding.EncodeToString(callbackPolicyJSON)
policy.Conditions = append(policy.Conditions, map[string]string{"callback": callbackPolicyEncoded})
}
// 编码上传策略
policyJSON, err := json.Marshal(policy)
if err != nil {
return serializer.UploadCredential{}, err
}
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
// 签名上传策略
hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey))
_, err = io.WriteString(hmacSign, policyEncoded)
if err != nil {
return serializer.UploadCredential{}, err
}
signature := base64.StdEncoding.EncodeToString(hmacSign.Sum(nil))
return serializer.UploadCredential{
Policy: fmt.Sprintf("%s:%s", callbackPolicyEncoded, policyEncoded),
Path: savePath,
AccessKey: handler.Policy.AccessKey,
Token: signature,
}, nil
}

@ -18,8 +18,10 @@ type UploadPolicy struct {
// UploadCredential 返回给客户端的上传凭证
type UploadCredential struct {
Token string `json:"token"`
Policy string `json:"policy"`
Token string `json:"token"`
Policy string `json:"policy"`
Path string `json:"path"`
AccessKey string `json:"ak"`
}
// UploadSession 上传会话

@ -19,7 +19,7 @@ func RemoteCallback(c *gin.Context) {
// QiniuCallback 七牛上传回调
func QiniuCallback(c *gin.Context) {
var callbackBody callback.QiniuUploadCallbackService
var callbackBody callback.UploadCallbackService
if err := c.ShouldBindJSON(&callbackBody); err == nil {
res := callback.ProcessCallback(callbackBody, c)
if res.Code != 0 {
@ -31,3 +31,14 @@ func QiniuCallback(c *gin.Context) {
c.JSON(401, ErrorResponse(err))
}
}
// OSSCallback 阿里云OSS上传回调
func OSSCallback(c *gin.Context) {
var callbackBody callback.UploadCallbackService
if err := c.ShouldBindJSON(&callbackBody); err == nil {
res := callback.ProcessCallback(callbackBody, c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}

@ -138,6 +138,12 @@ func InitMasterRouter() *gin.Engine {
middleware.QiniuCallbackAuth(),
controllers.QiniuCallback,
)
// 阿里云OSS策略上传回调
callback.POST(
"oss/:key",
middleware.OSSCallbackAuth(),
controllers.OSSCallback,
)
}
// 需要登录保护的

@ -25,8 +25,8 @@ func (service RemoteUploadCallbackService) GetBody() serializer.UploadCallback {
return service.Data
}
// QiniuUploadCallbackService 七牛存储上传回调请求服务
type QiniuUploadCallbackService struct {
// UploadCallbackService 云存储上传回调请求服务
type UploadCallbackService struct {
Name string `json:"name"`
SourceName string `json:"source_name"`
PicInfo string `json:"pic_info"`
@ -34,7 +34,7 @@ type QiniuUploadCallbackService struct {
}
// GetBody 返回回调正文
func (service QiniuUploadCallbackService) GetBody() serializer.UploadCallback {
func (service UploadCallbackService) GetBody() serializer.UploadCallback {
return serializer.UploadCallback{
Name: service.Name,
SourceName: service.SourceName,

@ -11,7 +11,7 @@ import (
// UploadCredentialService 获取上传凭证服务
type UploadCredentialService struct {
Path string `form:"path" binding:"required"`
Size uint64 `form:"size" binding:"required,min=0"`
Size uint64 `form:"size" binding:"min=0"`
}
// Get 获取新的上传凭证

Loading…
Cancel
Save