Feat: file uploading token sign for remote policy

pull/247/head
HFO4 5 years ago
parent 132c7a8fcb
commit cf8b5f4d1e

2
.gitignore vendored

@ -21,4 +21,4 @@ version.lock
# Config file
*.ini
/conf/conf.ini
conf/conf.ini

@ -103,6 +103,8 @@ solid #e9e9e9;"bgcolor="#fff"><tbody><tr style="font-family: 'Helvetica Neue',He
{Name: "archive_timeout", Value: `30`, Type: "timeout"},
{Name: "download_timeout", Value: `30`, Type: "timeout"},
{Name: "doc_preview_timeout", Value: `60`, Type: "timeout"},
{Name: "upload_credential_timeout", Value: `1800`, Type: "timeout"},
{Name: "upload_session_timeout", Value: `86400`, Type: "timeout"},
{Name: "allowdVisitorDownload", Value: `false`, Type: "share"},
{Name: "login_captcha", Value: `0`, Type: "login"},
{Name: "qq_login", Value: `0`, Type: "login"},

@ -5,7 +5,9 @@ import (
"github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/conf"
"github.com/HFO4/cloudreve/pkg/filesystem/local"
"github.com/HFO4/cloudreve/pkg/filesystem/remote"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/gin-gonic/gin"
"io"
"net/url"
@ -39,6 +41,9 @@ type Handler interface {
// url - 站点本身地址,
// isDownload - 是否直接下载
Source(ctx context.Context, path string, url url.URL, ttl int64, isDownload bool) (string, error)
// Token 获取有效期为ttl的上传凭证和签名同时回调会话有效期为sessionTTL
Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error)
}
// FileSystem 管理文件的文件系统
@ -124,6 +129,11 @@ func (fs *FileSystem) dispatchHandler() error {
Policy: currentPolicy,
}
return nil
case "remote":
fs.Handler = remote.Handler{
Policy: currentPolicy,
}
return nil
default:
return ErrUnknownPolicyType
}

@ -4,6 +4,7 @@ import (
"github.com/DATA-DOG/go-sqlmock"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/filesystem/local"
"github.com/HFO4/cloudreve/pkg/filesystem/remote"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
"net/http/httptest"
@ -19,9 +20,17 @@ func TestNewFileSystem(t *testing.T) {
},
}
// 本地 成功
fs, err := NewFileSystem(&user)
asserts.NoError(err)
asserts.NotNil(fs.Handler)
asserts.IsType(local.Handler{}, fs.Handler)
// 远程
user.Policy.Type = "remote"
fs, err = NewFileSystem(&user)
asserts.NoError(err)
asserts.NotNil(fs.Handler)
asserts.IsType(remote.Handler{}, fs.Handler)
user.Policy.Type = "unknown"
fs, err = NewFileSystem(&user)

@ -160,3 +160,9 @@ func (handler Handler) Source(
finalURL := baseURL.ResolveReference(signedURI).String()
return finalURL, nil
}
// Token 获取上传策略和认证Token本地策略直接返回空值
// TODO 测试
func (handler Handler) Token(ctx context.Context, ttl int64, key string) (serializer.UploadCredential, error) {
return serializer.UploadCredential{}, nil
}

@ -184,3 +184,11 @@ func TestHandler_GetDownloadURL(t *testing.T) {
asserts.Empty(downloadURL)
}
}
func TestHandler_Token(t *testing.T) {
asserts := assert.New(t)
handler := Handler{}
ctx := context.Background()
_, err := handler.Token(ctx, 10, "123")
asserts.NoError(err)
}

@ -0,0 +1,92 @@
package remote
// TODO 测试
import (
"context"
"errors"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/auth"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
"github.com/HFO4/cloudreve/pkg/serializer"
"io"
"net/http"
"net/url"
"time"
)
// Handler 远程存储策略适配器
type Handler struct {
Policy *model.Policy
}
// Get 获取文件内容
func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser, error) {
return nil, nil
}
// Put 将文件流保存到指定目录
func (handler Handler) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
return errors.New("远程策略不支持此上传方式")
}
// Delete 删除一个或多个文件,
// 返回未删除的文件,及遇到的最后一个错误
func (handler Handler) Delete(ctx context.Context, files []string) ([]string, error) {
return []string{}, nil
}
// Thumb 获取文件缩略图
func (handler Handler) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
return nil, nil
}
// Source 获取外链URL
func (handler Handler) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
) (string, error) {
return "", errors.New("暂未实现")
}
// Token 获取上传策略和认证Token
// TODO 测试
func (handler Handler) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/upload/" + key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 生成上传策略
policy := serializer.UploadPolicy{
SavePath: handler.Policy.DirNameRule,
FileName: handler.Policy.FileNameRule,
AutoRename: handler.Policy.AutoRename,
MaxSize: handler.Policy.MaxSize,
AllowedExtension: handler.Policy.OptionsSerialized.FileType,
CallbackURL: apiURL.String(),
}
policyEncoded, err := policy.EncodeUploadPolicy()
if err != nil {
return serializer.UploadCredential{}, err
}
// 签名上传策略
uploadRequest, _ := http.NewRequest("POST", "/api/v3/slave/upload", nil)
uploadRequest.Header = map[string][]string{
"X-Policy": {policyEncoded},
}
auth.SignRequest(uploadRequest, time.Now().Unix()+TTL)
if credential, ok := uploadRequest.Header["Authorization"]; ok && len(credential) == 1 {
return serializer.UploadCredential{
Token: credential[0],
Policy: policyEncoded,
}, nil
}
return serializer.UploadCredential{}, errors.New("无法签名上传策略")
}

@ -0,0 +1,43 @@
package remote
import (
"context"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/auth"
"github.com/HFO4/cloudreve/pkg/cache"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/stretchr/testify/assert"
"testing"
)
func TestHandler_Token(t *testing.T) {
asserts := assert.New(t)
handler := Handler{
Policy: &model.Policy{
MaxSize: 10,
AutoRename: true,
DirNameRule: "dir",
FileNameRule: "file",
OptionsSerialized: model.PolicyOption{
FileType: []string{"txt"},
},
},
}
ctx := context.Background()
auth.General = auth.HMACAuth{SecretKey: []byte("test")}
// 成功
{
cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
credential, err := handler.Token(ctx, 10, "123")
asserts.NoError(err)
policy, err := serializer.DecodeUploadPolicy(credential.Policy)
asserts.NoError(err)
asserts.Equal(uint64(10), policy.MaxSize)
asserts.Equal(true, policy.AutoRename)
asserts.Equal("dir", policy.SavePath)
asserts.Equal("file", policy.FileName)
asserts.Equal([]string{"txt"}, policy.AllowedExtension)
}
}

@ -3,11 +3,13 @@ package filesystem
import (
"context"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/cache"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/HFO4/cloudreve/pkg/util"
"github.com/gin-gonic/gin"
"path/filepath"
"strconv"
)
/* ================
@ -135,3 +137,52 @@ func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file FileHe
}
}
// GetUploadToken 生成新的上传凭证
func (fs *FileSystem) GetUploadToken(ctx context.Context, path string, size uint64) (*serializer.UploadCredential, error) {
// 获取相关有效期设置
ttls := model.GetSettingByNames([]string{"upload_credential_timeout", "upload_session_timeout"})
var (
err error
credentialTTL int64 = 3600
callBackSessionTTL int64 = 86400
)
// 获取上传凭证的有效期
if ttlStr, ok := ttls["upload_credential_timeout"]; ok {
credentialTTL, err = strconv.ParseInt(ttlStr, 10, 64)
if err != nil {
return nil, serializer.NewError(serializer.CodeInternalSetting, "上传凭证有效期设置无效", err)
}
}
// 获取回调会话的有效期
if ttlStr, ok := ttls["upload_session_timeout"]; ok {
callBackSessionTTL, err = strconv.ParseInt(ttlStr, 10, 64)
if err != nil {
return nil, serializer.NewError(serializer.CodeInternalSetting, "上传会话有效期设置无效", err)
}
}
// 获取上传凭证
callbackKey := util.RandStringRunes(32)
credential, err := fs.Handler.Token(ctx, credentialTTL, callbackKey)
if err != nil {
return nil, serializer.NewError(serializer.CodeEncryptError, "无法获取上传凭证", err)
}
// 创建回调会话
err = cache.Set(
"callback_"+callbackKey,
serializer.UploadSession{
UID: fs.User.ID,
VirtualPath: path,
},
int(callBackSessionTTL),
)
if err != nil {
return nil, err
}
return &credential, nil
}

@ -4,6 +4,7 @@ import (
"context"
"errors"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/cache"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/filesystem/local"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
@ -48,6 +49,11 @@ func (m FileHeaderMock) Source(ctx context.Context, path string, url url.URL, ex
return args.Get(0).(string), args.Error(1)
}
func (m FileHeaderMock) Token(ctx context.Context, expires int64, key string) (serializer.UploadCredential, error) {
args := m.Called(ctx, expires, key)
return args.Get(0).(serializer.UploadCredential), args.Error(1)
}
func TestFileSystem_Upload(t *testing.T) {
asserts := assert.New(t)
@ -159,3 +165,60 @@ func TestFileSystem_GenerateSavePath_Anonymous(t *testing.T) {
asserts.Len(savePath, 26)
asserts.Contains(savePath, "test.test")
}
func TestFileSystem_GetUploadToken(t *testing.T) {
asserts := assert.New(t)
fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}}
ctx := context.Background()
// 成功
{
cache.SetSettings(map[string]string{
"upload_credential_timeout": "10",
"upload_session_timeout": "10",
}, "setting_")
testHandller := new(FileHeaderMock)
testHandller.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{Token: "test"}, nil)
fs.Handler = testHandller
res, err := fs.GetUploadToken(ctx, "/", 10)
testHandller.AssertExpectations(t)
asserts.NoError(err)
asserts.Equal("test", res.Token)
}
// 无法获取上传凭证
{
cache.SetSettings(map[string]string{
"upload_credential_timeout": "10",
"upload_session_timeout": "10",
}, "setting_")
testHandller := new(FileHeaderMock)
testHandller.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{}, errors.New("error"))
fs.Handler = testHandller
_, err := fs.GetUploadToken(ctx, "/", 10)
testHandller.AssertExpectations(t)
asserts.Error(err)
}
// 上传有效期错误
{
cache.SetSettings(map[string]string{
"upload_credential_timeout": "10",
"upload_session_timeout": "10",
}, "setting_")
_, err := fs.GetUploadToken(ctx, "/", 10)
asserts.Error(err)
}
// 上传会话有效期错误
{
cache.SetSettings(map[string]string{
"upload_credential_timeout": "10",
"upload_session_timeout": "10",
}, "setting_")
_, err := fs.GetUploadToken(ctx, "/", 10)
asserts.Error(err)
}
}

@ -2,6 +2,7 @@ package serializer
import (
"encoding/base64"
"encoding/gob"
"encoding/json"
)
@ -13,11 +14,25 @@ type UploadPolicy struct {
MaxSize uint64 `json:"max_size"`
AllowedExtension []string `json:"allowed_extension"`
CallbackURL string `json:"callback_url"`
CallbackKey string `json:"callback_key"`
}
// UploadCredential 返回给客户端的上传凭证
type UploadCredential struct {
Token string `json:"token"`
Policy string `json:"policy"`
}
// UploadSession 上传会话
type UploadSession struct {
UID uint
VirtualPath string
}
func init() {
gob.Register(UploadSession{})
}
// DecodeUploadPolicy 反序列化Header中携带的上传策略
// TODO 测试
func DecodeUploadPolicy(raw string) (*UploadPolicy, error) {
var res UploadPolicy
@ -33,3 +48,16 @@ func DecodeUploadPolicy(raw string) (*UploadPolicy, error) {
return &res, err
}
// EncodeUploadPolicy 序列化Header中携带的上传策略
// TODO 测试
func (policy *UploadPolicy) EncodeUploadPolicy() (string, error) {
jsonRes, err := json.Marshal(policy)
if err != nil {
return "", err
}
res := base64.StdEncoding.EncodeToString(jsonRes)
return res, nil
}

@ -33,10 +33,10 @@ func TestDecodeUploadPolicy(t *testing.T) {
&UploadPolicy{},
},
{
"eyJjYWxsYmFja19rZXkiOiJ0ZXN0In0=",
"eyJjYWxsYmFja191cmwiOiJ0ZXN0In0=",
false,
false,
&UploadPolicy{CallbackKey: "test"},
&UploadPolicy{CallbackURL: "test"},
},
}
@ -53,3 +53,11 @@ func TestDecodeUploadPolicy(t *testing.T) {
}
}
}
func TestUploadPolicy_EncodeUploadPolicy(t *testing.T) {
asserts := assert.New(t)
testPolicy := UploadPolicy{}
res, err := testPolicy.EncodeUploadPolicy()
asserts.NoError(err)
asserts.NotEmpty(res)
}

@ -283,3 +283,18 @@ func FileUploadStream(c *gin.Context) {
Code: 0,
})
}
// GetUploadCredential 获取上传凭证
func GetUploadCredential(c *gin.Context) {
// 创建上下文
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var service explorer.UploadCredentialService
if err := c.ShouldBindQuery(&service); err == nil {
res := service.Get(ctx, c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}

@ -139,6 +139,8 @@ func InitMasterRouter() *gin.Engine {
{
// 文件上传
file.POST("upload", controllers.FileUploadStream)
// 获取上传凭证
file.GET("upload/credential", controllers.GetUploadCredential)
// 更新文件
file.PUT("update/*path", controllers.PutContent)
// 创建文件下载会话

@ -0,0 +1,35 @@
package explorer
import (
"context"
"github.com/HFO4/cloudreve/pkg/filesystem"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/gin-gonic/gin"
)
// UploadCredentialService 获取上传凭证服务
type UploadCredentialService struct {
Path string `form:"path" binding:"required"`
Size uint64 `form:"size" binding:"required,min=0"`
}
// Get 获取新的上传凭证
func (service *UploadCredentialService) Get(ctx context.Context, c *gin.Context) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewFileSystemFromContext(c)
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
ctx = context.WithValue(ctx, fsctx.GinCtx, c)
credential, err := fs.GetUploadToken(ctx, service.Path, service.Size)
if err != nil {
return serializer.Err(serializer.CodeNotSet, err.Error(), err)
}
return serializer.Response{
Code: 0,
Data: credential,
}
}
Loading…
Cancel
Save