msb-public
/
cloudreve
mirror of https://github.com/cloudreve/Cloudreve
1
0
Code Issues Projects Releases Wiki Activity

Refactor: create placeholder file and record upload session id in it

Browse Source
pull/1107/head
HFO4 3 years ago
parent 6fdf77e00e
commit 72173bf894
18 changed files with 178 additions and 159 deletions
Show Stats Download Patch File Download Diff File

23
models/file.go
Unescape View File

@ -14,15 +14,15 @@ import (
type File struct { type File struct {
// 表字段 // 表字段
gorm.Model gorm.Model
Name string `gorm:"unique_index:idx_only_one"` Name string `gorm:"unique_index:idx_only_one"`
SourceName string `gorm:"type:text"` SourceName string `gorm:"type:text"`
UserID uint `gorm:"index:user_id;unique_index:idx_only_one"` UserID uint `gorm:"index:user_id;unique_index:idx_only_one"`
Size uint64 Size uint64
PicInfo string PicInfo string
FolderID uint `gorm:"index:folder_id;unique_index:idx_only_one"` FolderID uint `gorm:"index:folder_id;unique_index:idx_only_one"`
PolicyID uint PolicyID uint
Hidden bool UploadSessionID *string `gorm:"index:session_id;unique_index:session_only_one"`
Metadata string `gorm:"type:text"` Metadata string `gorm:"type:text"`
// 关联模型 // 关联模型
Policy Policy `gorm:"PRELOAD:false,association_autoupdate:false"` Policy Policy `gorm:"PRELOAD:false,association_autoupdate:false"`
@ -220,6 +220,11 @@ func (file *File) UpdateSourceName(value string) error {
return DB.Model(&file).Set("gorm:association_autoupdate", false).Update("source_name", value).Error return DB.Model(&file).Set("gorm:association_autoupdate", false).Update("source_name", value).Error
} }
// CanCopy 返回文件是否可被复制
func (file *File) CanCopy() bool {
return file.UploadSessionID == nil
}
/* /*
实现 webdav.FileInfo 接口 实现 webdav.FileInfo 接口
*/ */

10
models/folder.go
Unescape View File

@ -158,6 +158,11 @@ func (folder *Folder) MoveOrCopyFileTo(files []uint, dstFolder *Folder, isCopy b
// 复制文件记录 // 复制文件记录
for _, oldFile := range originFiles { for _, oldFile := range originFiles {
if !oldFile.CanCopy() {
util.Log().Warning("无法复制正在上传中的文件 [%s]， 跳过...", oldFile.Name)
continue
}
oldFile.Model = gorm.Model{} oldFile.Model = gorm.Model{}
oldFile.FolderID = dstFolder.ID oldFile.FolderID = dstFolder.ID
oldFile.UserID = dstFolder.OwnerID oldFile.UserID = dstFolder.OwnerID
@ -246,6 +251,11 @@ func (folder *Folder) CopyFolderTo(folderID uint, dstFolder *Folder) (size uint6
// 复制文件记录 // 复制文件记录
for _, oldFile := range originFiles { for _, oldFile := range originFiles {
if !oldFile.CanCopy() {
util.Log().Warning("无法复制正在上传中的文件 [%s]， 跳过...", oldFile.Name)
continue
}
oldFile.Model = gorm.Model{} oldFile.Model = gorm.Model{}
oldFile.FolderID = newIDCache[oldFile.FolderID] oldFile.FolderID = newIDCache[oldFile.FolderID]
oldFile.UserID = dstFolder.OwnerID oldFile.UserID = dstFolder.OwnerID

7
pkg/filesystem/driver/cos/handler.go
Unescape View File

@ -185,7 +185,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
// Put 将文件流保存到指定目录 // Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
opt := &cossdk.ObjectPutOptions{} opt := &cossdk.ObjectPutOptions{}
_, err := handler.Client.Object.Put(ctx, file.GetSavePath(), file, opt) _, err := handler.Client.Object.Put(ctx, file.Info().SavePath, file, opt)
return err return err
} }
@ -331,6 +331,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
apiURL := siteURL.ResolveReference(apiBaseURI).String() apiURL := siteURL.ResolveReference(apiBaseURI).String()
// 上传策略 // 上传策略
savePath := file.Info().SavePath
startTime := time.Now() startTime := time.Now()
endTime := startTime.Add(time.Duration(ttl) * time.Second) endTime := startTime.Add(time.Duration(ttl) * time.Second)
keyTime := fmt.Sprintf("%d;%d", startTime.Unix(), endTime.Unix()) keyTime := fmt.Sprintf("%d;%d", startTime.Unix(), endTime.Unix())
@ -338,7 +339,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
Expiration: endTime.UTC().Format(time.RFC3339), Expiration: endTime.UTC().Format(time.RFC3339),
Conditions: []interface{}{ Conditions: []interface{}{
map[string]string{"bucket": handler.Policy.BucketName}, map[string]string{"bucket": handler.Policy.BucketName},
map[string]string{"$key": file.GetSavePath()}, map[string]string{"$key": savePath},
map[string]string{"x-cos-meta-callback": apiURL}, map[string]string{"x-cos-meta-callback": apiURL},
map[string]string{"x-cos-meta-key": uploadSession.Key}, map[string]string{"x-cos-meta-key": uploadSession.Key},
map[string]string{"q-sign-algorithm": "sha1"}, map[string]string{"q-sign-algorithm": "sha1"},
@ -352,7 +353,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
[]interface{}{"content-length-range", 0, handler.Policy.MaxSize}) []interface{}{"content-length-range", 0, handler.Policy.MaxSize})
} }
res, err := handler.getUploadCredential(ctx, postPolicy, keyTime, file.GetSavePath()) res, err := handler.getUploadCredential(ctx, postPolicy, keyTime, savePath)
if err == nil { if err == nil {
res.Callback = apiURL res.Callback = apiURL
res.Key = uploadSession.Key res.Key = uploadSession.Key

5
pkg/filesystem/driver/local/handler.go
Unescape View File

@ -85,10 +85,11 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
// Put 将文件流保存到指定目录 // Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close() defer file.Close()
dst := util.RelativePath(filepath.FromSlash(file.GetSavePath())) fileInfo := file.Info()
dst := util.RelativePath(filepath.FromSlash(fileInfo.SavePath))
// 如果非 Overwrite，则检查是否有重名冲突 // 如果非 Overwrite，则检查是否有重名冲突
if file.GetMode() == fsctx.Create { if fileInfo.Mode == fsctx.Create {
if util.Exists(dst) { if util.Exists(dst) {
util.Log().Warning("物理同名文件已存在或不可用: %s", dst) util.Log().Warning("物理同名文件已存在或不可用: %s", dst)
return errors.New("物理同名文件已存在或不可用") return errors.New("物理同名文件已存在或不可用")

7
pkg/filesystem/driver/onedrive/api.go
Unescape View File

@ -258,14 +258,15 @@ func (client *Client) UploadChunk(ctx context.Context, uploadURL string, chunk *
// Upload 上传文件 // Upload 上传文件
func (client *Client) Upload(ctx context.Context, file fsctx.FileHeader) error { func (client *Client) Upload(ctx context.Context, file fsctx.FileHeader) error {
fileInfo := file.Info()
// 决定是否覆盖文件 // 决定是否覆盖文件
overwrite := "replace" overwrite := "replace"
if file.GetMode() != fsctx.Create { if fileInfo.Mode != fsctx.Create {
overwrite = "fail" overwrite = "fail"
} }
size := int(file.GetSize()) size := int(fileInfo.Size)
dst := file.GetSavePath() dst := fileInfo.SavePath
// 小文件，使用简单上传接口上传 // 小文件，使用简单上传接口上传
if size <= int(SmallFileSize) { if size <= int(SmallFileSize) {

7
pkg/filesystem/driver/onedrive/handler.go
Unescape View File

@ -223,9 +223,10 @@ func (handler Driver) replaceSourceHost(origin string) (string, error) {
// Token 获取上传会话URL // Token 获取上传会话URL
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (serializer.UploadCredential, error) { func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (serializer.UploadCredential, error) {
fileInfo := file.Info()
// 如果小于4MB，则由服务端中转 // 如果小于4MB，则由服务端中转
if file.GetSize() <= SmallFileSize { if fileInfo.Size <= SmallFileSize {
return serializer.UploadCredential{}, nil return serializer.UploadCredential{}, nil
} }
@ -234,13 +235,13 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
apiBaseURI, _ := url.Parse("/api/v3/callback/onedrive/finish/" + uploadSession.Key) apiBaseURI, _ := url.Parse("/api/v3/callback/onedrive/finish/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI) apiURL := siteURL.ResolveReference(apiBaseURI)
uploadURL, err := handler.Client.CreateUploadSession(ctx, file.GetSavePath(), WithConflictBehavior("fail")) uploadURL, err := handler.Client.CreateUploadSession(ctx, fileInfo.SavePath, WithConflictBehavior("fail"))
if err != nil { if err != nil {
return serializer.UploadCredential{}, err return serializer.UploadCredential{}, err
} }
// 监控回调及上传 // 监控回调及上传
go handler.Client.MonitorUpload(uploadURL, uploadSession.Key, file.GetSavePath(), file.GetSize(), ttl) go handler.Client.MonitorUpload(uploadURL, uploadSession.Key, fileInfo.SavePath, fileInfo.Size, ttl)
return serializer.UploadCredential{ return serializer.UploadCredential{
Policy: uploadURL, Policy: uploadURL,

10
pkg/filesystem/driver/oss/handler.go
Unescape View File

@ -226,6 +226,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
// Put 将文件流保存到指定目录 // Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close() defer file.Close()
fileInfo := file.Info()
// 初始化客户端 // 初始化客户端
if err := handler.InitOSSClient(false); err != nil { if err := handler.InitOSSClient(false); err != nil {
@ -237,7 +238,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
// 是否允许覆盖 // 是否允许覆盖
overwrite := true overwrite := true
if file.GetMode() == fsctx.Create { if fileInfo.Mode == fsctx.Create {
overwrite = false overwrite = false
} }
@ -247,7 +248,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
} }
// 上传文件 // 上传文件
err := handler.bucket.PutObject(file.GetSavePath(), file, options...) err := handler.bucket.PutObject(fileInfo.SavePath, file, options...)
if err != nil { if err != nil {
return err return err
} }
@ -411,11 +412,12 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
} }
// 上传策略 // 上传策略
savePath := file.Info().SavePath
postPolicy := UploadPolicy{ postPolicy := UploadPolicy{
Expiration: time.Now().UTC().Add(time.Duration(ttl) * time.Second).Format(time.RFC3339), Expiration: time.Now().UTC().Add(time.Duration(ttl) * time.Second).Format(time.RFC3339),
Conditions: []interface{}{ Conditions: []interface{}{
map[string]string{"bucket": handler.Policy.BucketName}, map[string]string{"bucket": handler.Policy.BucketName},
[]string{"starts-with", "$key", path.Dir(file.GetSavePath())}, []string{"starts-with", "$key", path.Dir(savePath)},
}, },
} }
@ -424,7 +426,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
[]interface{}{"content-length-range", 0, handler.Policy.MaxSize}) []interface{}{"content-length-range", 0, handler.Policy.MaxSize})
} }
return handler.getUploadCredential(ctx, postPolicy, callbackPolicy, ttl, file.GetSavePath()) return handler.getUploadCredential(ctx, postPolicy, callbackPolicy, ttl, savePath)
} }
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback CallbackPolicy, TTL int64, savePath string) (serializer.UploadCredential, error) { func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback CallbackPolicy, TTL int64, savePath string) (serializer.UploadCredential, error) {

11
pkg/filesystem/driver/qiniu/handler.go
Unescape View File

@ -150,12 +150,13 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600) credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
// 生成上传策略 // 生成上传策略
fileInfo := file.Info()
putPolicy := storage.PutPolicy{ putPolicy := storage.PutPolicy{
// 指定为覆盖策略 // 指定为覆盖策略
Scope: fmt.Sprintf("%s:%s", handler.Policy.BucketName, file.GetSavePath()), Scope: fmt.Sprintf("%s:%s", handler.Policy.BucketName, fileInfo.SavePath),
SaveKey: file.GetSavePath(), SaveKey: fileInfo.SavePath,
ForceSaveKey: true, ForceSaveKey: true,
FsizeLimit: int64(file.GetSize()), FsizeLimit: int64(fileInfo.Size),
} }
// 是否开启了MIMEType限制 // 是否开启了MIMEType限制
if handler.Policy.OptionsSerialized.MimeType != "" { if handler.Policy.OptionsSerialized.MimeType != "" {
@ -177,7 +178,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
} }
// 开始上传 // 开始上传
err = formUploader.Put(ctx, &ret, token.Token, file.GetSavePath(), file, int64(file.GetSize()), &putExtra) err = formUploader.Put(ctx, &ret, token.Token, fileInfo.SavePath, file, int64(fileInfo.Size), &putExtra)
if err != nil { if err != nil {
return err return err
} }
@ -285,7 +286,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
CallbackURL: apiURL.String(), CallbackURL: apiURL.String(),
CallbackBody: `{"name":"$(fname)","source_name":"$(key)","size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`, CallbackBody: `{"name":"$(fname)","source_name":"$(key)","size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`,
CallbackBodyType: "application/json", CallbackBodyType: "application/json",
SaveKey: file.GetSavePath(), SaveKey: file.Info().SavePath,
ForceSaveKey: true, ForceSaveKey: true,
FsizeLimit: int64(handler.Policy.MaxSize), FsizeLimit: int64(handler.Policy.MaxSize),
} }

13
pkg/filesystem/driver/remote/handler.go
Unescape View File

@ -140,11 +140,12 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600) credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
// 生成上传策略 // 生成上传策略
fileInfo := file.Info()
policy := serializer.UploadPolicy{ policy := serializer.UploadPolicy{
SavePath: path.Dir(file.GetSavePath()), SavePath: path.Dir(fileInfo.SavePath),
FileName: path.Base(file.GetSavePath()), FileName: path.Base(fileInfo.FileName),
AutoRename: false, AutoRename: false,
MaxSize: file.GetSize(), MaxSize: fileInfo.Size,
} }
credential, err := handler.getUploadCredential(ctx, policy, int64(credentialTTL)) credential, err := handler.getUploadCredential(ctx, policy, int64(credentialTTL))
if err != nil { if err != nil {
@ -152,11 +153,11 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
} }
// 对文件名进行URLEncode // 对文件名进行URLEncode
fileName := url.QueryEscape(path.Base(file.GetSavePath())) fileName := url.QueryEscape(path.Base(fileInfo.SavePath))
// 决定是否要禁用文件覆盖 // 决定是否要禁用文件覆盖
overwrite := "true" overwrite := "true"
if file.GetMode() != fsctx.Create { if fileInfo.Mode != fsctx.Create {
overwrite = "false" overwrite = "false"
} }
@ -170,7 +171,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
"X-Cr-FileName": {fileName}, "X-Cr-FileName": {fileName},
"X-Cr-Overwrite": {overwrite}, "X-Cr-Overwrite": {overwrite},
}), }),
request.WithContentLength(int64(file.GetSize())), request.WithContentLength(int64(fileInfo.Size)),
request.WithTimeout(time.Duration(0)), request.WithTimeout(time.Duration(0)),
request.WithMasterMeta(), request.WithMasterMeta(),
request.WithSlaveMeta(handler.Policy.AccessKey), request.WithSlaveMeta(handler.Policy.AccessKey),

7
pkg/filesystem/driver/s3/handler.go
Unescape View File

@ -206,7 +206,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
uploader := s3manager.NewUploader(handler.sess) uploader := s3manager.NewUploader(handler.sess)
dst := file.GetSavePath() dst := file.Info().SavePath
_, err := uploader.Upload(&s3manager.UploadInput{ _, err := uploader.Upload(&s3manager.UploadInput{
Bucket: &handler.Policy.BucketName, Bucket: &handler.Policy.BucketName,
Key: &dst, Key: &dst,
@ -331,11 +331,12 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
apiURL := siteURL.ResolveReference(apiBaseURI) apiURL := siteURL.ResolveReference(apiBaseURI)
// 上传策略 // 上传策略
savePath := file.Info().SavePath
putPolicy := UploadPolicy{ putPolicy := UploadPolicy{
Expiration: time.Now().UTC().Add(time.Duration(ttl) * time.Second).Format(time.RFC3339), Expiration: time.Now().UTC().Add(time.Duration(ttl) * time.Second).Format(time.RFC3339),
Conditions: []interface{}{ Conditions: []interface{}{
map[string]string{"bucket": handler.Policy.BucketName}, map[string]string{"bucket": handler.Policy.BucketName},
[]string{"starts-with", "$key", file.GetSavePath()}, []string{"starts-with", "$key", savePath},
[]string{"starts-with", "$success_action_redirect", apiURL.String()}, []string{"starts-with", "$success_action_redirect", apiURL.String()},
[]string{"starts-with", "$name", ""}, []string{"starts-with", "$name", ""},
[]string{"starts-with", "$Content-Type", ""}, []string{"starts-with", "$Content-Type", ""},
@ -349,7 +350,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
} }
// 生成上传凭证 // 生成上传凭证
return handler.getUploadCredential(ctx, putPolicy, apiURL, file.GetSavePath()) return handler.getUploadCredential(ctx, putPolicy, apiURL, savePath)
} }
// Meta 获取文件信息 // Meta 获取文件信息

2
pkg/filesystem/driver/shadow/slaveinmaster/handler.go
Unescape View File

@ -57,7 +57,7 @@ func (d *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
req := serializer.SlaveTransferReq{ req := serializer.SlaveTransferReq{
Src: src, Src: src,
Dst: file.GetSavePath(), Dst: file.Info().SavePath,
Policy: d.policy, Policy: d.policy,
} }

9
pkg/filesystem/driver/upyun/handler.go
Unescape View File

@ -154,7 +154,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
Password: handler.Policy.SecretKey, Password: handler.Policy.SecretKey,
}) })
err := up.Put(&upyun.PutObjectConfig{ err := up.Put(&upyun.PutObjectConfig{
Path: file.GetSavePath(), Path: file.Info().SavePath,
Reader: file, Reader: file,
}) })
@ -319,14 +319,15 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
apiURL := siteURL.ResolveReference(apiBaseURI) apiURL := siteURL.ResolveReference(apiBaseURI)
// 上传策略 // 上传策略
fileInfo := file.Info()
putPolicy := UploadPolicy{ putPolicy := UploadPolicy{
Bucket: handler.Policy.BucketName, Bucket: handler.Policy.BucketName,
// TODO escape // TODO escape
SaveKey: file.GetSavePath(), SaveKey: fileInfo.SavePath,
Expiration: time.Now().Add(time.Duration(ttl) * time.Second).Unix(), Expiration: time.Now().Add(time.Duration(ttl) * time.Second).Unix(),
CallbackURL: apiURL.String(), CallbackURL: apiURL.String(),
ContentLength: file.GetSize(), ContentLength: fileInfo.Size,
ContentLengthRange: fmt.Sprintf("0,%d", file.GetSize()), ContentLengthRange: fmt.Sprintf("0,%d", fileInfo.Size),
AllowFileType: strings.Join(handler.Policy.OptionsSerialized.FileType, ","), AllowFileType: strings.Join(handler.Policy.OptionsSerialized.FileType, ","),
} }

31
pkg/filesystem/errors.go
Unescape View File

@ -7,19 +7,20 @@ import (
) )
var ( var (
ErrUnknownPolicyType = errors.New("未知存储策略类型") ErrUnknownPolicyType = errors.New("未知存储策略类型")
ErrFileSizeTooBig = errors.New("单个文件尺寸太大") ErrFileSizeTooBig = errors.New("单个文件尺寸太大")
ErrFileExtensionNotAllowed = errors.New("不允许上传此类型的文件") ErrFileExtensionNotAllowed = errors.New("不允许上传此类型的文件")
ErrInsufficientCapacity = errors.New("容量空间不足") ErrInsufficientCapacity = errors.New("容量空间不足")
ErrIllegalObjectName = errors.New("目标名称非法") ErrIllegalObjectName = errors.New("目标名称非法")
ErrClientCanceled = errors.New("客户端取消操作") ErrClientCanceled = errors.New("客户端取消操作")
ErrRootProtected = errors.New("无法对根目录进行操作") ErrRootProtected = errors.New("无法对根目录进行操作")
ErrInsertFileRecord = serializer.NewError(serializer.CodeDBError, "无法插入文件记录", nil) ErrInsertFileRecord = serializer.NewError(serializer.CodeDBError, "无法插入文件记录", nil)
ErrFileExisted = serializer.NewError(serializer.CodeObjectExist, "同名文件或目录已存在", nil) ErrFileExisted = serializer.NewError(serializer.CodeObjectExist, "同名文件或目录已存在", nil)
ErrFolderExisted = serializer.NewError(serializer.CodeObjectExist, "同名目录已存在", nil) ErrFileUploadSessionExisted = serializer.NewError(serializer.CodeObjectExist, "当前目录下已经有同名文件正在上传中", nil)
ErrPathNotExist = serializer.NewError(404, "路径不存在", nil) ErrFolderExisted = serializer.NewError(serializer.CodeObjectExist, "同名目录已存在", nil)
ErrObjectNotExist = serializer.NewError(404, "文件不存在", nil) ErrPathNotExist = serializer.NewError(404, "路径不存在", nil)
ErrIO = serializer.NewError(serializer.CodeIOFailed, "无法读取文件数据", nil) ErrObjectNotExist = serializer.NewError(404, "文件不存在", nil)
ErrDBListObjects = serializer.NewError(serializer.CodeDBError, "无法列取对象记录", nil) ErrIO = serializer.NewError(serializer.CodeIOFailed, "无法读取文件数据", nil)
ErrDBDeleteObjects = serializer.NewError(serializer.CodeDBError, "无法删除对象记录", nil) ErrDBListObjects = serializer.NewError(serializer.CodeDBError, "无法列取对象记录", nil)
ErrDBDeleteObjects = serializer.NewError(serializer.CodeDBError, "无法删除对象记录", nil)
) )

13
pkg/filesystem/file.go
Unescape View File

@ -53,18 +53,19 @@ func (fs *FileSystem) AddFile(ctx context.Context, parent *model.Folder, file fs
return nil, err return nil, err
} }
uploadInfo := file.Info()
newFile := model.File{ newFile := model.File{
Name: file.GetFileName(), Name: uploadInfo.FileName,
SourceName: file.GetSavePath(), SourceName: uploadInfo.SavePath,
UserID: fs.User.ID, UserID: fs.User.ID,
Size: file.GetSize(), Size: uploadInfo.Size,
FolderID: parent.ID, FolderID: parent.ID,
PolicyID: fs.Policy.ID, PolicyID: fs.Policy.ID,
Hidden: file.IsHidden(), MetadataSerialized: uploadInfo.Metadata,
MetadataSerialized: file.GetMetadata(), UploadSessionID: uploadInfo.UploadSessionID,
} }
if fs.Policy.IsThumbExist(file.GetFileName()) { if fs.Policy.IsThumbExist(uploadInfo.FileName) {
newFile.PicInfo = "1,1" newFile.PicInfo = "1,1"
} }

102
pkg/filesystem/fsctx/stream.go
Unescape View File

@ -15,80 +15,62 @@ const (
Nop Nop
) )
// FileStream 用户传来的文件 // FileHeader 上传来的文件数据处理器
type FileStream struct { type FileHeader interface {
Mode WriteMode io.Reader
Hidden bool io.Closer
LastModified *time.Time Info() *UploadTaskInfo
Metadata map[string]string SetSize(uint64)
File io.ReadCloser
Size uint64
VirtualPath string
Name string
MIMEType string
SavePath string
} }
func (file *FileStream) Read(p []byte) (n int, err error) { type UploadTaskInfo struct {
return file.File.Read(p) Size uint64
MIMEType string
FileName string
VirtualPath string
Mode WriteMode
Metadata map[string]string
LastModified *time.Time
SavePath string
UploadSessionID *string
} }
func (file *FileStream) GetMIMEType() string { // FileStream 用户传来的文件
return file.MIMEType type FileStream struct {
Mode WriteMode
LastModified *time.Time
Metadata map[string]string
File io.ReadCloser
Size uint64
VirtualPath string
Name string
MIMEType string
SavePath string
UploadSessionID *string
} }
func (file *FileStream) GetSize() uint64 { func (file *FileStream) Read(p []byte) (n int, err error) {
return file.Size return file.File.Read(p)
} }
func (file *FileStream) Close() error { func (file *FileStream) Close() error {
return file.File.Close() return file.File.Close()
} }
func (file *FileStream) GetFileName() string { func (file *FileStream) Info() *UploadTaskInfo {
return file.Name return &UploadTaskInfo{
} Size: file.Size,
MIMEType: file.MIMEType,
func (file *FileStream) GetVirtualPath() string { FileName: file.Name,
return file.VirtualPath VirtualPath: file.VirtualPath,
} Mode: file.Mode,
Metadata: file.Metadata,
func (file *FileStream) GetMode() WriteMode { LastModified: file.LastModified,
return file.Mode SavePath: file.SavePath,
} UploadSessionID: file.UploadSessionID,
}
func (file *FileStream) GetMetadata() map[string]string {
return file.Metadata
}
func (file *FileStream) GetLastModified() *time.Time {
return file.LastModified
}
func (file *FileStream) IsHidden() bool {
return file.Hidden
}
func (file *FileStream) GetSavePath() string {
return file.SavePath
} }
func (file *FileStream) SetSize(size uint64) { func (file *FileStream) SetSize(size uint64) {
file.Size = size file.Size = size
} }
// FileHeader 上传来的文件数据处理器
type FileHeader interface {
io.Reader
io.Closer
GetSize() uint64
GetMIMEType() string
GetFileName() string
GetVirtualPath() string
GetMode() WriteMode
GetMetadata() map[string]string
GetLastModified() *time.Time
IsHidden() bool
GetSavePath() string
SetSize(uint64)
}

56
pkg/filesystem/hooks.go
Unescape View File

@ -57,21 +57,22 @@ func (fs *FileSystem) Trigger(ctx context.Context, name string, file fsctx.FileH
// HookSlaveUploadValidate Slave模式下对文件上传的一系列验证 // HookSlaveUploadValidate Slave模式下对文件上传的一系列验证
func HookSlaveUploadValidate(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error { func HookSlaveUploadValidate(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
policy := ctx.Value(fsctx.UploadPolicyCtx).(serializer.UploadPolicy) policy := ctx.Value(fsctx.UploadPolicyCtx).(serializer.UploadPolicy)
fileInfo := file.Info()
// 验证单文件尺寸 // 验证单文件尺寸
if policy.MaxSize > 0 { if policy.MaxSize > 0 {
if file.GetSize() > policy.MaxSize { if fileInfo.Size > policy.MaxSize {
return ErrFileSizeTooBig return ErrFileSizeTooBig
} }
} }
// 验证文件名 // 验证文件名
if !fs.ValidateLegalName(ctx, file.GetFileName()) { if !fs.ValidateLegalName(ctx, fileInfo.FileName) {
return ErrIllegalObjectName return ErrIllegalObjectName
} }
// 验证扩展名 // 验证扩展名
if len(policy.AllowedExtension) > 0 && !IsInExtensionList(policy.AllowedExtension, file.GetFileName()) { if len(policy.AllowedExtension) > 0 && !IsInExtensionList(policy.AllowedExtension, fileInfo.FileName) {
return ErrFileExtensionNotAllowed return ErrFileExtensionNotAllowed
} }
@ -80,18 +81,20 @@ func HookSlaveUploadValidate(ctx context.Context, fs *FileSystem, file fsctx.Fil
// HookValidateFile 一系列对文件检验的集合 // HookValidateFile 一系列对文件检验的集合
func HookValidateFile(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error { func HookValidateFile(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
fileInfo := file.Info()
// 验证单文件尺寸 // 验证单文件尺寸
if !fs.ValidateFileSize(ctx, file.GetSize()) { if !fs.ValidateFileSize(ctx, fileInfo.Size) {
return ErrFileSizeTooBig return ErrFileSizeTooBig
} }
// 验证文件名 // 验证文件名
if !fs.ValidateLegalName(ctx, file.GetFileName()) { if !fs.ValidateLegalName(ctx, fileInfo.FileName) {
return ErrIllegalObjectName return ErrIllegalObjectName
} }
// 验证扩展名 // 验证扩展名
if !fs.ValidateExtension(ctx, file.GetFileName()) { if !fs.ValidateExtension(ctx, fileInfo.FileName) {
return ErrFileExtensionNotAllowed return ErrFileExtensionNotAllowed
} }
@ -113,7 +116,7 @@ func HookResetPolicy(ctx context.Context, fs *FileSystem, file fsctx.FileHeader)
// HookValidateCapacity 验证并扣除用户容量，包含数据库操作 // HookValidateCapacity 验证并扣除用户容量，包含数据库操作
func HookValidateCapacity(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error { func HookValidateCapacity(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
// 验证并扣除容量 // 验证并扣除容量
if !fs.ValidateCapacity(ctx, file.GetSize()) { if !fs.ValidateCapacity(ctx, file.Info().Size) {
return ErrInsufficientCapacity return ErrInsufficientCapacity
} }
return nil return nil
@ -122,7 +125,7 @@ func HookValidateCapacity(ctx context.Context, fs *FileSystem, file fsctx.FileHe
// HookValidateCapacityWithoutIncrease 验证用户容量，不扣除 // HookValidateCapacityWithoutIncrease 验证用户容量，不扣除
func HookValidateCapacityWithoutIncrease(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error { func HookValidateCapacityWithoutIncrease(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
// 验证并扣除容量 // 验证并扣除容量
if fs.User.GetRemainingCapacity() < file.GetSize() { if fs.User.GetRemainingCapacity() < file.Info().Size {
return ErrInsufficientCapacity return ErrInsufficientCapacity
} }
return nil return nil
@ -131,22 +134,23 @@ func HookValidateCapacityWithoutIncrease(ctx context.Context, fs *FileSystem, fi
// HookChangeCapacity 根据原有文件和新文件的大小更新用户容量 // HookChangeCapacity 根据原有文件和新文件的大小更新用户容量
func HookChangeCapacity(ctx context.Context, fs *FileSystem, newFile fsctx.FileHeader) error { func HookChangeCapacity(ctx context.Context, fs *FileSystem, newFile fsctx.FileHeader) error {
originFile := ctx.Value(fsctx.FileModelCtx).(model.File) originFile := ctx.Value(fsctx.FileModelCtx).(model.File)
newFileSize := newFile.Info().Size
if newFile.GetSize() > originFile.Size { if newFileSize > originFile.Size {
if !fs.ValidateCapacity(ctx, newFile.GetSize()-originFile.Size) { if !fs.ValidateCapacity(ctx, newFileSize-originFile.Size) {
return ErrInsufficientCapacity return ErrInsufficientCapacity
} }
return nil return nil
} }
fs.User.DeductionStorage(originFile.Size - newFile.GetSize()) fs.User.DeductionStorage(originFile.Size - newFileSize)
return nil return nil
} }
// HookDeleteTempFile 删除已保存的临时文件 // HookDeleteTempFile 删除已保存的临时文件
func HookDeleteTempFile(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error { func HookDeleteTempFile(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
// 删除临时文件 // 删除临时文件
_, err := fs.Handler.Delete(ctx, []string{file.GetSavePath()}) _, err := fs.Handler.Delete(ctx, []string{file.Info().SavePath})
if err != nil { if err != nil {
util.Log().Warning("无法清理上传临时文件，%s", err) util.Log().Warning("无法清理上传临时文件，%s", err)
} }
@ -159,7 +163,7 @@ func HookCleanFileContent(ctx context.Context, fs *FileSystem, file fsctx.FileHe
// 清空内容 // 清空内容
return fs.Handler.Put(ctx, &fsctx.FileStream{ return fs.Handler.Put(ctx, &fsctx.FileStream{
File: ioutil.NopCloser(strings.NewReader("")), File: ioutil.NopCloser(strings.NewReader("")),
SavePath: file.GetSavePath(), SavePath: file.Info().SavePath,
Size: 0, Size: 0,
}) })
} }
@ -192,7 +196,7 @@ func HookGiveBackCapacity(ctx context.Context, fs *FileSystem, file fsctx.FileHe
// 归还用户容量 // 归还用户容量
res := true res := true
once.Do(func() { once.Do(func() {
res = fs.User.DeductionStorage(file.GetSize()) res = fs.User.DeductionStorage(file.Info().Size)
}) })
if !res { if !res {
@ -221,7 +225,7 @@ func GenericAfterUpdate(ctx context.Context, fs *FileSystem, newFile fsctx.FileH
fs.SetTargetFile(&[]model.File{originFile}) fs.SetTargetFile(&[]model.File{originFile})
err := originFile.UpdateSize(newFile.GetSize()) err := originFile.UpdateSize(newFile.Info().Size)
if err != nil { if err != nil {
return err return err
} }
@ -244,11 +248,12 @@ func GenericAfterUpdate(ctx context.Context, fs *FileSystem, newFile fsctx.FileH
// SlaveAfterUpload Slave模式下上传完成钩子 // SlaveAfterUpload Slave模式下上传完成钩子
func SlaveAfterUpload(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error { func SlaveAfterUpload(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
policy := ctx.Value(fsctx.UploadPolicyCtx).(serializer.UploadPolicy) policy := ctx.Value(fsctx.UploadPolicyCtx).(serializer.UploadPolicy)
fileInfo := fileHeader.Info()
// 构造一个model.File，用于生成缩略图 // 构造一个model.File，用于生成缩略图
file := model.File{ file := model.File{
Name: fileHeader.GetFileName(), Name: fileInfo.FileName,
SourceName: fileHeader.GetSavePath(), SourceName: fileInfo.SavePath,
} }
fs.GenerateThumbnail(ctx, &file) fs.GenerateThumbnail(ctx, &file)
@ -261,20 +266,19 @@ func SlaveAfterUpload(ctx context.Context, fs *FileSystem, fileHeader fsctx.File
Name: file.Name, Name: file.Name,
SourceName: file.SourceName, SourceName: file.SourceName,
PicInfo: file.PicInfo, PicInfo: file.PicInfo,
Size: fileHeader.GetSize(), Size: fileInfo.Size,
} }
return request.RemoteCallback(policy.CallbackURL, callbackBody) return request.RemoteCallback(policy.CallbackURL, callbackBody)
} }
// GenericAfterUpload 文件上传完成后，包含数据库操作 // GenericAfterUpload 文件上传完成后，包含数据库操作
func GenericAfterUpload(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error { func GenericAfterUpload(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
// 文件存放的虚拟路径 fileInfo := fileHeader.Info()
virtualPath := fileHeader.GetVirtualPath()
// 检查路径是否存在，不存在就创建 // 检查路径是否存在，不存在就创建
isExist, folder := fs.IsPathExist(virtualPath) isExist, folder := fs.IsPathExist(fileInfo.VirtualPath)
if !isExist { if !isExist {
newFolder, err := fs.CreateDirectory(ctx, virtualPath) newFolder, err := fs.CreateDirectory(ctx, fileInfo.VirtualPath)
if err != nil { if err != nil {
return err return err
} }
@ -282,10 +286,14 @@ func GenericAfterUpload(ctx context.Context, fs *FileSystem, fileHeader fsctx.Fi
} }
// 检查文件是否存在 // 检查文件是否存在
if ok, _ := fs.IsChildFileExist( if ok, file := fs.IsChildFileExist(
folder, folder,
fileHeader.GetFileName(), fileInfo.FileName,
); ok { ); ok {
if file.UploadSessionID != nil {
return ErrFileUploadSessionExisted
}
return ErrFileExisted return ErrFileExisted
} }

2
pkg/filesystem/manage.go
Unescape View File

@ -350,7 +350,7 @@ func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []mo
} }
} }
if !file.Hidden { if file.UploadSessionID == nil {
newFile := serializer.Object{ newFile := serializer.Object{
ID: hashid.HashID(file.ID, hashid.FileID), ID: hashid.HashID(file.ID, hashid.FileID),
Name: file.Name, Name: file.Name,

22
pkg/filesystem/upload.go
Unescape View File

@ -69,10 +69,11 @@ func (fs *FileSystem) Upload(ctx context.Context, file *fsctx.FileStream) (err e
return err return err
} }
fileInfo := file.Info()
util.Log().Info( util.Log().Info(
"新文件PUT:%s , 大小:%d, 上传者:%s", "新文件PUT:%s , 大小:%d, 上传者:%s",
file.GetFileName(), fileInfo.FileName,
file.GetSize(), fileInfo.Size,
fs.User.Nick, fs.User.Nick,
) )
@ -82,15 +83,17 @@ func (fs *FileSystem) Upload(ctx context.Context, file *fsctx.FileStream) (err e
// GenerateSavePath 生成要存放文件的路径 // GenerateSavePath 生成要存放文件的路径
// TODO 完善测试 // TODO 完善测试
func (fs *FileSystem) GenerateSavePath(ctx context.Context, file fsctx.FileHeader) string { func (fs *FileSystem) GenerateSavePath(ctx context.Context, file fsctx.FileHeader) string {
fileInfo := file.Info()
if fs.User.Model.ID != 0 { if fs.User.Model.ID != 0 {
return path.Join( return path.Join(
fs.Policy.GeneratePath( fs.Policy.GeneratePath(
fs.User.Model.ID, fs.User.Model.ID,
file.GetVirtualPath(), fileInfo.VirtualPath,
), ),
fs.Policy.GenerateFileName( fs.Policy.GenerateFileName(
fs.User.Model.ID, fs.User.Model.ID,
file.GetFileName(), fileInfo.FileName,
), ),
) )
} }
@ -112,7 +115,7 @@ func (fs *FileSystem) GenerateSavePath(ctx context.Context, file fsctx.FileHeade
), ),
anonymousPolicy.GenerateFileName( anonymousPolicy.GenerateFileName(
0, 0,
file.GetFileName(), fileInfo.FileName,
), ),
) )
} }
@ -156,11 +159,10 @@ func (fs *FileSystem) CreateUploadSession(ctx context.Context, file *fsctx.FileS
callbackKey := uuid.Must(uuid.NewV4()).String() callbackKey := uuid.Must(uuid.NewV4()).String()
// 创建隐藏的文件，同时校验文件信息 // 创建占位的文件，同时校验文件信息
file.Mode = fsctx.Nop file.Mode = fsctx.Nop
file.Hidden = true if callbackKey != "" {
file.Metadata = map[string]string{ file.UploadSessionID = &callbackKey
UploadSessionMetaKey: callbackKey,
} }
fs.Use("BeforeUpload", HookValidateFile) fs.Use("BeforeUpload", HookValidateFile)
@ -179,7 +181,7 @@ func (fs *FileSystem) CreateUploadSession(ctx context.Context, file *fsctx.FileS
Size: file.Size, Size: file.Size,
SavePath: file.SavePath, SavePath: file.SavePath,
ChunkSize: fs.Policy.OptionsSerialized.ChunkSize, ChunkSize: fs.Policy.OptionsSerialized.ChunkSize,
LastModified: file.GetLastModified(), LastModified: file.LastModified,
} }
// 获取上传凭证 // 获取上传凭证

Write Preview
Loading…
Cancel
Save