Feat: file uploading token sign for remote policy

.gitignore

@@ -21,4 +21,4 @@ version.lock
 
 # Config file
 *.ini
-/conf/conf.ini
+conf/conf.ini

models/migration.go

@@ -103,6 +103,8 @@ solid #e9e9e9;"bgcolor="#fff"><tbody><tr style="font-family: 'Helvetica Neue',He
 		{Name: "archive_timeout", Value: `30`, Type: "timeout"},
 		{Name: "download_timeout", Value: `30`, Type: "timeout"},
 		{Name: "doc_preview_timeout", Value: `60`, Type: "timeout"},
+		{Name: "upload_credential_timeout", Value: `1800`, Type: "timeout"},
+		{Name: "upload_session_timeout", Value: `86400`, Type: "timeout"},
 		{Name: "allowdVisitorDownload", Value: `false`, Type: "share"},
 		{Name: "login_captcha", Value: `0`, Type: "login"},
 		{Name: "qq_login", Value: `0`, Type: "login"},

pkg/filesystem/filesystem.go

@@ -5,7 +5,9 @@ import (
 	"github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/conf"
 	"github.com/HFO4/cloudreve/pkg/filesystem/local"
+	"github.com/HFO4/cloudreve/pkg/filesystem/remote"
 	"github.com/HFO4/cloudreve/pkg/filesystem/response"
+	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/gin-gonic/gin"
 	"io"
 	"net/url"
@@ -39,6 +41,9 @@ type Handler interface {
 	// url - 站点本身地址,
 	// isDownload - 是否直接下载
 	Source(ctx context.Context, path string, url url.URL, ttl int64, isDownload bool) (string, error)
+
+	// Token 获取有效期为ttl的上传凭证和签名，同时回调会话有效期为sessionTTL
+	Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error)
 }
 
 // FileSystem 管理文件的文件系统
@@ -124,6 +129,11 @@ func (fs *FileSystem) dispatchHandler() error {
 			Policy: currentPolicy,
 		}
 		return nil
+	case "remote":
+		fs.Handler = remote.Handler{
+			Policy: currentPolicy,
+		}
+		return nil
 	default:
 		return ErrUnknownPolicyType
 	}

pkg/filesystem/filesystem_test.go

@@ -4,6 +4,7 @@ import (
 	"github.com/DATA-DOG/go-sqlmock"
 	model "github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/filesystem/local"
+	"github.com/HFO4/cloudreve/pkg/filesystem/remote"
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/assert"
 	"net/http/httptest"
@@ -19,9 +20,17 @@ func TestNewFileSystem(t *testing.T) {
 		},
 	}
 
+	// 本地 成功
 	fs, err := NewFileSystem(&user)
 	asserts.NoError(err)
 	asserts.NotNil(fs.Handler)
+	asserts.IsType(local.Handler{}, fs.Handler)
+	// 远程
+	user.Policy.Type = "remote"
+	fs, err = NewFileSystem(&user)
+	asserts.NoError(err)
+	asserts.NotNil(fs.Handler)
+	asserts.IsType(remote.Handler{}, fs.Handler)
 
 	user.Policy.Type = "unknown"
 	fs, err = NewFileSystem(&user)

pkg/filesystem/local/handler.go

@@ -160,3 +160,9 @@ func (handler Handler) Source(
 	finalURL := baseURL.ResolveReference(signedURI).String()
 	return finalURL, nil
 }
+
+// Token 获取上传策略和认证Token，本地策略直接返回空值
+// TODO 测试
+func (handler Handler) Token(ctx context.Context, ttl int64, key string) (serializer.UploadCredential, error) {
+	return serializer.UploadCredential{}, nil
+}

pkg/filesystem/local/handller_test.go

@@ -184,3 +184,11 @@ func TestHandler_GetDownloadURL(t *testing.T) {
 		asserts.Empty(downloadURL)
 	}
 }
+
+func TestHandler_Token(t *testing.T) {
+	asserts := assert.New(t)
+	handler := Handler{}
+	ctx := context.Background()
+	_, err := handler.Token(ctx, 10, "123")
+	asserts.NoError(err)
+}

pkg/filesystem/remote/handler.go

@@ -0,0 +1,92 @@
+package remote
+
+// TODO 测试
+import (
+	"context"
+	"errors"
+	model "github.com/HFO4/cloudreve/models"
+	"github.com/HFO4/cloudreve/pkg/auth"
+	"github.com/HFO4/cloudreve/pkg/filesystem/response"
+	"github.com/HFO4/cloudreve/pkg/serializer"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+// Handler 远程存储策略适配器
+type Handler struct {
+	Policy *model.Policy
+}
+
+// Get 获取文件内容
+func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser, error) {
+
+	return nil, nil
+}
+
+// Put 将文件流保存到指定目录
+func (handler Handler) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
+	return errors.New("远程策略不支持此上传方式")
+}
+
+// Delete 删除一个或多个文件，
+// 返回未删除的文件，及遇到的最后一个错误
+func (handler Handler) Delete(ctx context.Context, files []string) ([]string, error) {
+	return []string{}, nil
+}
+
+// Thumb 获取文件缩略图
+func (handler Handler) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
+	return nil, nil
+}
+
+// Source 获取外链URL
+func (handler Handler) Source(
+	ctx context.Context,
+	path string,
+	baseURL url.URL,
+	ttl int64,
+	isDownload bool,
+) (string, error) {
+	return "", errors.New("暂未实现")
+}
+
+// Token 获取上传策略和认证Token
+// TODO 测试
+func (handler Handler) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
+	// 生成回调地址
+	siteURL := model.GetSiteURL()
+	apiBaseURI, _ := url.Parse("/api/v3/callback/upload/" + key)
+	apiURL := siteURL.ResolveReference(apiBaseURI)
+
+	// 生成上传策略
+	policy := serializer.UploadPolicy{
+		SavePath:         handler.Policy.DirNameRule,
+		FileName:         handler.Policy.FileNameRule,
+		AutoRename:       handler.Policy.AutoRename,
+		MaxSize:          handler.Policy.MaxSize,
+		AllowedExtension: handler.Policy.OptionsSerialized.FileType,
+		CallbackURL:      apiURL.String(),
+	}
+	policyEncoded, err := policy.EncodeUploadPolicy()
+	if err != nil {
+		return serializer.UploadCredential{}, err
+	}
+
+	// 签名上传策略
+	uploadRequest, _ := http.NewRequest("POST", "/api/v3/slave/upload", nil)
+	uploadRequest.Header = map[string][]string{
+		"X-Policy": {policyEncoded},
+	}
+	auth.SignRequest(uploadRequest, time.Now().Unix()+TTL)
+
+	if credential, ok := uploadRequest.Header["Authorization"]; ok && len(credential) == 1 {
+		return serializer.UploadCredential{
+			Token:  credential[0],
+			Policy: policyEncoded,
+		}, nil
+	}
+	return serializer.UploadCredential{}, errors.New("无法签名上传策略")
+
+}

pkg/filesystem/remote/handler_test.go

@@ -0,0 +1,43 @@
+package remote
+
+import (
+	"context"
+	model "github.com/HFO4/cloudreve/models"
+	"github.com/HFO4/cloudreve/pkg/auth"
+	"github.com/HFO4/cloudreve/pkg/cache"
+	"github.com/HFO4/cloudreve/pkg/serializer"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestHandler_Token(t *testing.T) {
+	asserts := assert.New(t)
+	handler := Handler{
+		Policy: &model.Policy{
+			MaxSize:      10,
+			AutoRename:   true,
+			DirNameRule:  "dir",
+			FileNameRule: "file",
+			OptionsSerialized: model.PolicyOption{
+				FileType: []string{"txt"},
+			},
+		},
+	}
+	ctx := context.Background()
+	auth.General = auth.HMACAuth{SecretKey: []byte("test")}
+
+	// 成功
+	{
+		cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
+		credential, err := handler.Token(ctx, 10, "123")
+		asserts.NoError(err)
+		policy, err := serializer.DecodeUploadPolicy(credential.Policy)
+		asserts.NoError(err)
+		asserts.Equal(uint64(10), policy.MaxSize)
+		asserts.Equal(true, policy.AutoRename)
+		asserts.Equal("dir", policy.SavePath)
+		asserts.Equal("file", policy.FileName)
+		asserts.Equal([]string{"txt"}, policy.AllowedExtension)
+	}
+
+}

pkg/filesystem/upload.go

@@ -3,11 +3,13 @@ package filesystem
 import (
 	"context"
 	model "github.com/HFO4/cloudreve/models"
+	"github.com/HFO4/cloudreve/pkg/cache"
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/HFO4/cloudreve/pkg/util"
 	"github.com/gin-gonic/gin"
 	"path/filepath"
+	"strconv"
 )
 
 /* ================
@@ -135,3 +137,52 @@ func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file FileHe
 
 	}
 }
+
+// GetUploadToken 生成新的上传凭证
+func (fs *FileSystem) GetUploadToken(ctx context.Context, path string, size uint64) (*serializer.UploadCredential, error) {
+	// 获取相关有效期设置
+	ttls := model.GetSettingByNames([]string{"upload_credential_timeout", "upload_session_timeout"})
+
+	var (
+		err                error
+		credentialTTL      int64 = 3600
+		callBackSessionTTL int64 = 86400
+	)
+	// 获取上传凭证的有效期
+	if ttlStr, ok := ttls["upload_credential_timeout"]; ok {
+		credentialTTL, err = strconv.ParseInt(ttlStr, 10, 64)
+		if err != nil {
+			return nil, serializer.NewError(serializer.CodeInternalSetting, "上传凭证有效期设置无效", err)
+		}
+	}
+
+	// 获取回调会话的有效期
+	if ttlStr, ok := ttls["upload_session_timeout"]; ok {
+		callBackSessionTTL, err = strconv.ParseInt(ttlStr, 10, 64)
+		if err != nil {
+			return nil, serializer.NewError(serializer.CodeInternalSetting, "上传会话有效期设置无效", err)
+		}
+	}
+
+	// 获取上传凭证
+	callbackKey := util.RandStringRunes(32)
+	credential, err := fs.Handler.Token(ctx, credentialTTL, callbackKey)
+	if err != nil {
+		return nil, serializer.NewError(serializer.CodeEncryptError, "无法获取上传凭证", err)
+	}
+
+	// 创建回调会话
+	err = cache.Set(
+		"callback_"+callbackKey,
+		serializer.UploadSession{
+			UID:         fs.User.ID,
+			VirtualPath: path,
+		},
+		int(callBackSessionTTL),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &credential, nil
+}

pkg/filesystem/upload_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	model "github.com/HFO4/cloudreve/models"
+	"github.com/HFO4/cloudreve/pkg/cache"
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
 	"github.com/HFO4/cloudreve/pkg/filesystem/local"
 	"github.com/HFO4/cloudreve/pkg/filesystem/response"
@@ -48,6 +49,11 @@ func (m FileHeaderMock) Source(ctx context.Context, path string, url url.URL, ex
 	return args.Get(0).(string), args.Error(1)
 }
 
+func (m FileHeaderMock) Token(ctx context.Context, expires int64, key string) (serializer.UploadCredential, error) {
+	args := m.Called(ctx, expires, key)
+	return args.Get(0).(serializer.UploadCredential), args.Error(1)
+}
+
 func TestFileSystem_Upload(t *testing.T) {
 	asserts := assert.New(t)
 
@@ -159,3 +165,60 @@ func TestFileSystem_GenerateSavePath_Anonymous(t *testing.T) {
 	asserts.Len(savePath, 26)
 	asserts.Contains(savePath, "test.test")
 }
+
+func TestFileSystem_GetUploadToken(t *testing.T) {
+	asserts := assert.New(t)
+	fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}}
+	ctx := context.Background()
+
+	// 成功
+	{
+		cache.SetSettings(map[string]string{
+			"upload_credential_timeout": "10",
+			"upload_session_timeout":    "10",
+		}, "setting_")
+		testHandller := new(FileHeaderMock)
+		testHandller.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{Token: "test"}, nil)
+		fs.Handler = testHandller
+		res, err := fs.GetUploadToken(ctx, "/", 10)
+		testHandller.AssertExpectations(t)
+		asserts.NoError(err)
+		asserts.Equal("test", res.Token)
+	}
+
+	// 无法获取上传凭证
+	{
+		cache.SetSettings(map[string]string{
+			"upload_credential_timeout": "10",
+			"upload_session_timeout":    "10",
+		}, "setting_")
+		testHandller := new(FileHeaderMock)
+		testHandller.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{}, errors.New("error"))
+		fs.Handler = testHandller
+		_, err := fs.GetUploadToken(ctx, "/", 10)
+		testHandller.AssertExpectations(t)
+		asserts.Error(err)
+	}
+
+	// 上传有效期错误
+	{
+		cache.SetSettings(map[string]string{
+			"upload_credential_timeout": "？10",
+			"upload_session_timeout":    "10",
+		}, "setting_")
+
+		_, err := fs.GetUploadToken(ctx, "/", 10)
+		asserts.Error(err)
+	}
+
+	// 上传会话有效期错误
+	{
+		cache.SetSettings(map[string]string{
+			"upload_credential_timeout": "10",
+			"upload_session_timeout":    "？10",
+		}, "setting_")
+
+		_, err := fs.GetUploadToken(ctx, "/", 10)
+		asserts.Error(err)
+	}
+}

pkg/serializer/upload.go

@@ -2,6 +2,7 @@ package serializer
 
 import (
 	"encoding/base64"
+	"encoding/gob"
 	"encoding/json"
 )
 
@@ -13,11 +14,25 @@ type UploadPolicy struct {
 	MaxSize          uint64   `json:"max_size"`
 	AllowedExtension []string `json:"allowed_extension"`
 	CallbackURL      string   `json:"callback_url"`
-	CallbackKey      string   `json:"callback_key"`
+}
+
+// UploadCredential 返回给客户端的上传凭证
+type UploadCredential struct {
+	Token  string `json:"token"`
+	Policy string `json:"policy"`
+}
+
+// UploadSession 上传会话
+type UploadSession struct {
+	UID         uint
+	VirtualPath string
+}
+
+func init() {
+	gob.Register(UploadSession{})
 }
 
 // DecodeUploadPolicy 反序列化Header中携带的上传策略
-// TODO 测试
 func DecodeUploadPolicy(raw string) (*UploadPolicy, error) {
 	var res UploadPolicy
 
@@ -33,3 +48,16 @@ func DecodeUploadPolicy(raw string) (*UploadPolicy, error) {
 
 	return &res, err
 }
+
+// EncodeUploadPolicy 序列化Header中携带的上传策略
+// TODO 测试
+func (policy *UploadPolicy) EncodeUploadPolicy() (string, error) {
+	jsonRes, err := json.Marshal(policy)
+	if err != nil {
+		return "", err
+	}
+
+	res := base64.StdEncoding.EncodeToString(jsonRes)
+	return res, nil
+
+}

pkg/serializer/upload_test.go

@@ -33,10 +33,10 @@ func TestDecodeUploadPolicy(t *testing.T) {
 			&UploadPolicy{},
 		},
 		{
-			"eyJjYWxsYmFja19rZXkiOiJ0ZXN0In0=",
+			"eyJjYWxsYmFja191cmwiOiJ0ZXN0In0=",
 			false,
 			false,
-			&UploadPolicy{CallbackKey: "test"},
+			&UploadPolicy{CallbackURL: "test"},
 		},
 	}
 
@@ -53,3 +53,11 @@ func TestDecodeUploadPolicy(t *testing.T) {
 		}
 	}
 }
+
+func TestUploadPolicy_EncodeUploadPolicy(t *testing.T) {
+	asserts := assert.New(t)
+	testPolicy := UploadPolicy{}
+	res, err := testPolicy.EncodeUploadPolicy()
+	asserts.NoError(err)
+	asserts.NotEmpty(res)
+}

routers/controllers/file.go

@@ -283,3 +283,18 @@ func FileUploadStream(c *gin.Context) {
 		Code: 0,
 	})
 }
+
+// GetUploadCredential 获取上传凭证
+func GetUploadCredential(c *gin.Context) {
+	// 创建上下文
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var service explorer.UploadCredentialService
+	if err := c.ShouldBindQuery(&service); err == nil {
+		res := service.Get(ctx, c)
+		c.JSON(200, res)
+	} else {
+		c.JSON(200, ErrorResponse(err))
+	}
+}

routers/router.go

@@ -139,6 +139,8 @@ func InitMasterRouter() *gin.Engine {
 			{
 				// 文件上传
 				file.POST("upload", controllers.FileUploadStream)
+				// 获取上传凭证
+				file.GET("upload/credential", controllers.GetUploadCredential)
 				// 更新文件
 				file.PUT("update/*path", controllers.PutContent)
 				// 创建文件下载会话

service/explorer/upload.go

@@ -0,0 +1,35 @@
+package explorer
+
+import (
+	"context"
+	"github.com/HFO4/cloudreve/pkg/filesystem"
+	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
+	"github.com/HFO4/cloudreve/pkg/serializer"
+	"github.com/gin-gonic/gin"
+)
+
+// UploadCredentialService 获取上传凭证服务
+type UploadCredentialService struct {
+	Path string `form:"path" binding:"required"`
+	Size uint64 `form:"size" binding:"required,min=0"`
+}
+
+// Get 获取新的上传凭证
+func (service *UploadCredentialService) Get(ctx context.Context, c *gin.Context) serializer.Response {
+	// 创建文件系统
+	fs, err := filesystem.NewFileSystemFromContext(c)
+	if err != nil {
+		return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
+	}
+
+	ctx = context.WithValue(ctx, fsctx.GinCtx, c)
+	credential, err := fs.GetUploadToken(ctx, service.Path, service.Size)
+	if err != nil {
+		return serializer.Err(serializer.CodeNotSet, err.Error(), err)
+	}
+
+	return serializer.Response{
+		Code: 0,
+		Data: credential,
+	}
+}