Feat: create upload session and pre-upload check

pkg/filesystem/driver/cos/handler.go

@@ -324,7 +324,7 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
 }
 
 // Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
+func (handler Driver) Token(ctx context.Context, TTL int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
 	// 读取上下文中生成的存储路径
 	savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
 	if !ok {
@@ -333,7 +333,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
 
 	// 生成回调地址
 	siteURL := model.GetSiteURL()
-	apiBaseURI, _ := url.Parse("/api/v3/callback/cos/" + key)
+	apiBaseURI, _ := url.Parse("/api/v3/callback/cos/" + uploadSession.Key)
 	apiURL := siteURL.ResolveReference(apiBaseURI).String()
 
 	// 上传策略
@@ -346,7 +346,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
 			map[string]string{"bucket": handler.Policy.BucketName},
 			map[string]string{"$key": savePath},
 			map[string]string{"x-cos-meta-callback": apiURL},
-			map[string]string{"x-cos-meta-key": key},
+			map[string]string{"x-cos-meta-key": uploadSession.Key},
 			map[string]string{"q-sign-algorithm": "sha1"},
 			map[string]string{"q-ak": handler.Policy.AccessKey},
 			map[string]string{"q-sign-time": keyTime},
@@ -361,7 +361,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
 	res, err := handler.getUploadCredential(ctx, postPolicy, keyTime)
 	if err == nil {
 		res.Callback = apiURL
-		res.Key = key
+		res.Key = uploadSession.Key
 	}
 
 	return res, err

pkg/filesystem/driver/handler.go

@@ -29,8 +29,8 @@ type Handler interface {
 	// isDownload - 是否直接下载
 	Source(ctx context.Context, path string, url url.URL, ttl int64, isDownload bool, speed int) (string, error)
 
-	// Token 获取有效期为ttl的上传凭证和签名，同时回调会话有效期为sessionTTL
+	// Token 获取有效期为ttl的上传凭证和签名
-	Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error)
+	Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error)
 
 	// List 递归列取远程端path路径下文件、目录，不包含path本身，
 	// 返回的对象路径以path作为起始根目录.

pkg/filesystem/driver/local/handler.go

@@ -79,22 +79,9 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
 		return nil, err
 	}
 
-	// 开启一个协程，用于请求结束后关闭reader
-	// go closeReader(ctx, file)
-
 	return file, nil
 }
 
-// closeReader 用于在请求结束后关闭reader
-// TODO 让业务代码自己关闭
-func closeReader(ctx context.Context, closer io.Closer) {
-	select {
-	case <-ctx.Done():
-		_ = closer.Close()
-
-	}
-}
-
 // Put 将文件流保存到指定目录
 func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
 	defer file.Close()
@@ -227,6 +214,8 @@ func (handler Driver) Source(
 }
 
 // Token 获取上传策略和认证Token，本地策略直接返回空值
-func (handler Driver) Token(ctx context.Context, ttl int64, key string) (serializer.UploadCredential, error) {
+func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
-	return serializer.UploadCredential{}, nil
+	return serializer.UploadCredential{
+		SessionID: uploadSession.Key,
+	}, nil
 }

pkg/filesystem/driver/onedrive/handler.go

@@ -223,7 +223,7 @@ func (handler Driver) replaceSourceHost(origin string) (string, error) {
 }
 
 // Token 获取上传会话URL
-func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
+func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
 
 	// 读取上下文中生成的存储路径和文件大小
 	savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
@@ -242,7 +242,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
 
 	// 生成回调地址
 	siteURL := model.GetSiteURL()
-	apiBaseURI, _ := url.Parse("/api/v3/callback/onedrive/finish/" + key)
+	apiBaseURI, _ := url.Parse("/api/v3/callback/onedrive/finish/" + uploadSession.Key)
 	apiURL := siteURL.ResolveReference(apiBaseURI)
 
 	uploadURL, err := handler.Client.CreateUploadSession(ctx, savePath, WithConflictBehavior("fail"))
@@ -251,7 +251,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
 	}
 
 	// 监控回调及上传
-	go handler.Client.MonitorUpload(uploadURL, key, savePath, fileSize, TTL)
+	go handler.Client.MonitorUpload(uploadURL, uploadSession.Key, savePath, fileSize, ttl)
 
 	return serializer.UploadCredential{
 		Policy: uploadURL,

pkg/filesystem/driver/oss/handler.go

@@ -397,7 +397,7 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
 }
 
 // Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
+func (handler Driver) Token(ctx context.Context, TTL int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
 	// 读取上下文中生成的存储路径
 	savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
 	if !ok {
@@ -406,7 +406,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
 
 	// 生成回调地址
 	siteURL := model.GetSiteURL()
-	apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + key)
+	apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + uploadSession.Key)
 	apiURL := siteURL.ResolveReference(apiBaseURI)
 
 	// 回调策略

pkg/filesystem/driver/qiniu/handler.go

@@ -274,10 +274,10 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64)
 }
 
 // Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
+func (handler Driver) Token(ctx context.Context, TTL int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
 	// 生成回调地址
 	siteURL := model.GetSiteURL()
-	apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + key)
+	apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + uploadSession.Key)
 	apiURL := siteURL.ResolveReference(apiBaseURI)
 
 	// 读取上下文中生成的存储路径

pkg/filesystem/driver/remote/handler.go

@@ -305,10 +305,10 @@ func (handler Driver) Source(
 }
 
 // Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
+func (handler Driver) Token(ctx context.Context, TTL int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
 	// 生成回调地址
 	siteURL := model.GetSiteURL()
-	apiBaseURI, _ := url.Parse("/api/v3/callback/remote/" + key)
+	apiBaseURI, _ := url.Parse("/api/v3/callback/remote/" + uploadSession.Key)
 	apiURL := siteURL.ResolveReference(apiBaseURI)
 
 	// 生成上传策略

pkg/filesystem/driver/s3/handler.go

@@ -324,7 +324,7 @@ func (handler Driver) Source(
 }
 
 // Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
+func (handler Driver) Token(ctx context.Context, TTL int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
 
 	// 读取上下文中生成的存储路径和文件大小
 	savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
@@ -334,7 +334,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
 
 	// 生成回调地址
 	siteURL := model.GetSiteURL()
-	apiBaseURI, _ := url.Parse("/api/v3/callback/s3/" + key)
+	apiBaseURI, _ := url.Parse("/api/v3/callback/s3/" + uploadSession.Key)
 	apiURL := siteURL.ResolveReference(apiBaseURI)
 
 	// 上传策略

pkg/filesystem/driver/shadow/masterinslave/handler.go

@@ -47,7 +47,7 @@ func (d *Driver) Source(ctx context.Context, path string, url url.URL, ttl int64
 	return "", ErrNotImplemented
 }
 
-func (d *Driver) Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error) {
+func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
 	return serializer.UploadCredential{}, ErrNotImplemented
 }
 

pkg/filesystem/driver/shadow/slaveinmaster/handler.go

@@ -112,7 +112,7 @@ func (d *Driver) Source(ctx context.Context, path string, url url.URL, ttl int64
 	return "", ErrNotImplemented
 }
 
-func (d *Driver) Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error) {
+func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
 	return serializer.UploadCredential{}, ErrNotImplemented
 }
 

pkg/filesystem/driver/template/handler.go

@@ -1,55 +0,0 @@
-package template
-
-import (
-	"context"
-	"errors"
-	"io"
-	"net/url"
-
-	model "github.com/cloudreve/Cloudreve/v3/models"
-	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
-	"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
-)
-
-// Driver 适配器模板
-type Driver struct {
-	Policy *model.Policy
-}
-
-// Get 获取文件
-func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
-	return nil, errors.New("未实现")
-}
-
-// Put 将文件流保存到指定目录
-func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
-	return errors.New("未实现")
-}
-
-// Delete 删除一个或多个文件，
-// 返回未删除的文件，及遇到的最后一个错误
-func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
-	return []string{}, errors.New("未实现")
-}
-
-// Thumb 获取文件缩略图
-func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
-	return nil, errors.New("未实现")
-}
-
-// Source 获取外链URL
-func (handler Driver) Source(
-	ctx context.Context,
-	path string,
-	baseURL url.URL,
-	ttl int64,
-	isDownload bool,
-	speed int,
-) (string, error) {
-	return "", errors.New("未实现")
-}
-
-// Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
-	return serializer.UploadCredential{}, errors.New("未实现")
-}

pkg/filesystem/driver/upyun/handler.go

@@ -311,7 +311,7 @@ func (handler Driver) signURL(ctx context.Context, path *url.URL, TTL int64) (st
 }
 
 // Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
+func (handler Driver) Token(ctx context.Context, TTL int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
 	// 读取上下文中生成的存储路径和文件大小
 	savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
 	if !ok {
@@ -326,7 +326,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
 
 	// 生成回调地址
 	siteURL := model.GetSiteURL()
-	apiBaseURI, _ := url.Parse("/api/v3/callback/upyun/" + key)
+	apiBaseURI, _ := url.Parse("/api/v3/callback/upyun/" + uploadSession.Key)
 	apiURL := siteURL.ResolveReference(apiBaseURI)
 
 	// 上传策略

pkg/filesystem/upload.go

@@ -14,6 +14,7 @@ import (
 	"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
 	"github.com/cloudreve/Cloudreve/v3/pkg/util"
 	"github.com/gin-gonic/gin"
+	"github.com/gofrs/uuid"
 )
 
 /* ================
@@ -146,32 +147,47 @@ func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file FileHe
 	}
 }
 
-// GetUploadToken 生成新的上传凭证
+// CreateUploadSession 创建上传会话
-func (fs *FileSystem) GetUploadToken(ctx context.Context, path string, size uint64, name string) (*serializer.UploadCredential, error) {
+func (fs *FileSystem) CreateUploadSession(ctx context.Context, path string, size uint64, name string) (*serializer.UploadCredential, error) {
 	// 获取相关有效期设置
 	credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
 	callBackSessionTTL := model.GetIntSetting("upload_session_timeout", 86400)
 
 	var err error
 
-	// 检查文件大小
+	// 进行文件上传预检查
-	if fs.Policy.MaxSize != 0 {
+
-		if size > fs.Policy.MaxSize {
+	// 创建上下文环境
-			return nil, ErrFileSizeTooBig
+	ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, local.FileStream{
-		}
+		Size: size,
+		Name: name,
+	})
+
+	// 检查上传请求合法性
+	if err := HookValidateFile(ctx, fs); err != nil {
+		return nil, err
 	}
 
-	// 是否需要预先生成存储路径
+	if err := HookValidateCapacityWithoutIncrease(ctx, fs); err != nil {
-	var savePath string
+		return nil, err
-	if fs.Policy.IsPathGenerateNeeded() {
+	}
-		savePath = fs.GenerateSavePath(ctx, local.FileStream{Name: name, VirtualPath: path})
+
-		ctx = context.WithValue(ctx, fsctx.SavePathCtx, savePath)
+	// 生成存储路径
+	savePath := fs.GenerateSavePath(ctx, local.FileStream{Name: name, VirtualPath: path})
+
+	callbackKey := uuid.Must(uuid.NewV4()).String()
+	uploadSession := &serializer.UploadSession{
+		Key:         callbackKey,
+		UID:         fs.User.ID,
+		PolicyID:    fs.Policy.ID,
+		VirtualPath: path,
+		Name:        name,
+		Size:        size,
+		SavePath:    savePath,
 	}
-	ctx = context.WithValue(ctx, fsctx.FileSizeCtx, size)
 
 	// 获取上传凭证
-	callbackKey := util.RandStringRunes(32)
+	credential, err := fs.Handler.Token(ctx, int64(credentialTTL), uploadSession)
-	credential, err := fs.Handler.Token(ctx, int64(credentialTTL), callbackKey)
 	if err != nil {
 		return nil, serializer.NewError(serializer.CodeEncryptError, "无法获取上传凭证", err)
 	}
@@ -179,14 +195,7 @@ func (fs *FileSystem) GetUploadToken(ctx context.Context, path string, size uint
 	// 创建回调会话
 	err = cache.Set(
 		"callback_"+callbackKey,
-		serializer.UploadSession{
+		uploadSession,
-			Key:         callbackKey,
-			UID:         fs.User.ID,
-			VirtualPath: path,
-			Name:        name,
-			Size:        size,
-			SavePath:    savePath,
-		},
 		callBackSessionTTL,
 	)
 	if err != nil {

pkg/filesystem/upload_test.go

@@ -57,8 +57,8 @@ func (m FileHeaderMock) Source(ctx context.Context, path string, url url.URL, ex
 	return args.Get(0).(string), args.Error(1)
 }
 
-func (m FileHeaderMock) Token(ctx context.Context, expires int64, key string) (serializer.UploadCredential, error) {
+func (m FileHeaderMock) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
-	args := m.Called(ctx, expires, key)
+	args := m.Called(ctx, ttl, uploadSession)
 	return args.Get(0).(serializer.UploadCredential), args.Error(1)
 }
 
@@ -189,7 +189,7 @@ func TestFileSystem_GetUploadToken(t *testing.T) {
 		testHandler := new(FileHeaderMock)
 		testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{Token: "test"}, nil)
 		fs.Handler = testHandler
-		res, err := fs.GetUploadToken(ctx, "/", 10, "123")
+		res, err := fs.CreateUploadSession(ctx, "/", 10, "123")
 		testHandler.AssertExpectations(t)
 		asserts.NoError(err)
 		asserts.Equal("test", res.Token)
@@ -204,7 +204,7 @@ func TestFileSystem_GetUploadToken(t *testing.T) {
 		testHandler := new(FileHeaderMock)
 		testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{}, errors.New("error"))
 		fs.Handler = testHandler
-		_, err := fs.GetUploadToken(ctx, "/", 10, "123")
+		_, err := fs.CreateUploadSession(ctx, "/", 10, "123")
 		testHandler.AssertExpectations(t)
 		asserts.Error(err)
 	}

pkg/serializer/upload.go

@@ -18,6 +18,8 @@ type UploadPolicy struct {
 
 // UploadCredential 返回给客户端的上传凭证
 type UploadCredential struct {
+	SessionID string `json:"sessionID"`
+
 	Token     string `json:"token"`
 	Policy    string `json:"policy"`
 	Path      string `json:"path"` // 存储路径

routers/controllers/file.go

@@ -354,15 +354,15 @@ func FileUploadStream(c *gin.Context) {
 	})
 }
 
-// GetUploadCredential 获取上传凭证
+// GetUploadCredential 创建上传会话
 func GetUploadCredential(c *gin.Context) {
 	// 创建上下文
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	var service explorer.UploadCredentialService
+	var service explorer.UploadSessionService
-	if err := c.ShouldBindQuery(&service); err == nil {
+	if err := c.ShouldBindJSON(&service); err == nil {
-		res := service.Get(ctx, c)
+		res := service.Create(ctx, c)
 		c.JSON(200, res)
 	} else {
 		c.JSON(200, ErrorResponse(err))

routers/router.go

@@ -506,8 +506,8 @@ func InitMasterRouter() *gin.Engine {
 			{
 				// 文件上传
 				file.POST("upload", controllers.FileUploadStream)
-				// 获取上传凭证
+				// 创建上传会话
-				file.GET("upload/credential", controllers.GetUploadCredential)
+				file.PUT("upload/session", controllers.GetUploadCredential)
 				// 更新文件
 				file.PUT("update/:id", controllers.PutContent)
 				// 创建空白文件

service/explorer/upload.go

@@ -4,29 +4,31 @@ import (
 	"context"
 
 	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
-	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
 	"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
 	"github.com/gin-gonic/gin"
 )
 
-// UploadCredentialService 获取上传凭证服务
+// UploadSessionService 获取上传凭证服务
-type UploadCredentialService struct {
+type UploadSessionService struct {
-	Path string `form:"path" binding:"required"`
+	Path     string `json:"path" binding:"required"`
-	Size uint64 `form:"size" binding:"min=0"`
+	Size     uint64 `json:"size" binding:"min=0"`
-	Name string `form:"name"`
+	Name     string `json:"name" binding:"required"`
-	Type string `form:"type"`
+	PolicyID uint   `json:"policy_id" binding:"required"`
 }
 
-// Get 获取新的上传凭证
+// Create 创建新的上传会话
-func (service *UploadCredentialService) Get(ctx context.Context, c *gin.Context) serializer.Response {
+func (service *UploadSessionService) Create(ctx context.Context, c *gin.Context) serializer.Response {
 	// 创建文件系统
 	fs, err := filesystem.NewFileSystemFromContext(c)
 	if err != nil {
 		return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
 	}
 
-	ctx = context.WithValue(ctx, fsctx.GinCtx, c)
+	if fs.Policy.ID != service.PolicyID {
-	credential, err := fs.GetUploadToken(ctx, service.Path, service.Size, service.Name)
+		return serializer.Err(serializer.CodePolicyNotAllowed, "存储策略发生变化，请刷新文件列表并重新添加此任务", nil)
+	}
+
+	credential, err := fs.CreateUploadSession(ctx, service.Path, service.Size, service.Name)
 	if err != nil {
 		return serializer.Err(serializer.CodeNotSet, err.Error(), err)
 	}