From 118d7387976656d8c738227bff7f3122d50f30e6 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Mon, 28 Feb 2022 17:49:00 +0800 Subject: [PATCH] Feat: support apply append mode and overwrite mode for FileStream --- assets | 2 +- pkg/filesystem/archive.go | 2 +- pkg/filesystem/driver/local/handler.go | 16 ++++++++-------- pkg/filesystem/driver/onedrive/api.go | 6 +++--- pkg/filesystem/driver/oss/handler.go | 6 +----- pkg/filesystem/driver/remote/handler.go | 6 +++--- pkg/filesystem/fsctx/stream.go | 7 +++---- pkg/filesystem/hooks.go | 1 + pkg/filesystem/upload.go | 4 ++-- pkg/task/compress.go | 3 +-- pkg/task/slavetask/transfer.go | 1 - pkg/task/tranfer.go | 3 +-- pkg/webdav/webdav.go | 4 +--- routers/controllers/slave.go | 8 ++++---- service/explorer/file.go | 2 +- service/explorer/upload.go | 2 +- 16 files changed, 32 insertions(+), 41 deletions(-) diff --git a/assets b/assets index b90a49b..eb3f329 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit b90a49ba5d809e3042bce801ac28fb7cde74c878 +Subproject commit eb3f32922ab9cd2f9fbef4860b93fec759a7054d diff --git a/pkg/filesystem/archive.go b/pkg/filesystem/archive.go index 8d4dcfd..4899e63 100644 --- a/pkg/filesystem/archive.go +++ b/pkg/filesystem/archive.go @@ -308,7 +308,7 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst string) error { Size: uint64(size), Name: path.Base(dst), VirtualPath: path.Dir(dst), - Mode: fsctx.Create, + Mode: 0, }) fileStream.Close() if err != nil { diff --git a/pkg/filesystem/driver/local/handler.go b/pkg/filesystem/driver/local/handler.go index e91f838..92432d5 100644 --- a/pkg/filesystem/driver/local/handler.go +++ b/pkg/filesystem/driver/local/handler.go @@ -93,7 +93,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { dst := util.RelativePath(filepath.FromSlash(fileInfo.SavePath)) // 如果非 Overwrite,则检查是否有重名冲突 - if fileInfo.Mode == fsctx.Create { + if fileInfo.Mode&fsctx.Overwrite != fsctx.Overwrite { if util.Exists(dst) { util.Log().Warning("物理同名文件已存在或不可用: %s", dst) return errors.New("物理同名文件已存在或不可用") @@ -115,21 +115,21 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { err error ) - if fileInfo.Mode == fsctx.Append { - // 如果是追加模式,则直接打开文件 - out, err = os.OpenFile(dst, os.O_APPEND|os.O_CREATE|os.O_WRONLY, Perm) + openMode := os.O_CREATE | os.O_RDWR + if fileInfo.Mode&fsctx.Append == fsctx.Append { + openMode |= os.O_APPEND } else { - // 创建或覆盖目标文件 - out, err = os.Create(dst) + openMode |= os.O_TRUNC } + out, err = os.OpenFile(dst, openMode, Perm) if err != nil { util.Log().Warning("无法打开或创建文件,%s", err) return err } defer out.Close() - if fileInfo.Mode == fsctx.Append { + if fileInfo.Mode&fsctx.Append == fsctx.Append { stat, err := out.Stat() if err != nil { util.Log().Warning("无法读取文件信息,%s", err) @@ -144,7 +144,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { return fmt.Errorf("覆盖分片时发生错误: %w", err) } - out, err = os.OpenFile(dst, os.O_APPEND|os.O_CREATE|os.O_WRONLY, Perm) + out, err = os.OpenFile(dst, openMode, Perm) defer out.Close() if err != nil { util.Log().Warning("无法打开或创建文件,%s", err) diff --git a/pkg/filesystem/driver/onedrive/api.go b/pkg/filesystem/driver/onedrive/api.go index 06dc96b..c9e7ff2 100644 --- a/pkg/filesystem/driver/onedrive/api.go +++ b/pkg/filesystem/driver/onedrive/api.go @@ -260,9 +260,9 @@ func (client *Client) UploadChunk(ctx context.Context, uploadURL string, chunk * func (client *Client) Upload(ctx context.Context, file fsctx.FileHeader) error { fileInfo := file.Info() // 决定是否覆盖文件 - overwrite := "replace" - if fileInfo.Mode != fsctx.Create { - overwrite = "fail" + overwrite := "fail" + if fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite { + overwrite = "replace" } size := int(fileInfo.Size) diff --git a/pkg/filesystem/driver/oss/handler.go b/pkg/filesystem/driver/oss/handler.go index bd4c736..bbe3fc1 100644 --- a/pkg/filesystem/driver/oss/handler.go +++ b/pkg/filesystem/driver/oss/handler.go @@ -237,11 +237,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600) // 是否允许覆盖 - overwrite := true - if fileInfo.Mode == fsctx.Create { - overwrite = false - } - + overwrite := fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite options := []oss.Option{ oss.Expires(time.Now().Add(time.Duration(credentialTTL) * time.Second)), oss.ForbidOverWrite(!overwrite), diff --git a/pkg/filesystem/driver/remote/handler.go b/pkg/filesystem/driver/remote/handler.go index c68e8bd..2793a70 100644 --- a/pkg/filesystem/driver/remote/handler.go +++ b/pkg/filesystem/driver/remote/handler.go @@ -173,9 +173,9 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { fileName := url.QueryEscape(path.Base(fileInfo.SavePath)) // 决定是否要禁用文件覆盖 - overwrite := "true" - if fileInfo.Mode != fsctx.Create { - overwrite = "false" + overwrite := "false" + if fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite { + overwrite = "true" } // 上传文件 diff --git a/pkg/filesystem/fsctx/stream.go b/pkg/filesystem/fsctx/stream.go index 53ff7a4..0c13ef9 100644 --- a/pkg/filesystem/fsctx/stream.go +++ b/pkg/filesystem/fsctx/stream.go @@ -8,11 +8,10 @@ import ( type WriteMode int const ( - Overwrite WriteMode = iota + Overwrite WriteMode = 0x00001 // Append 只适用于本地策略 - Append - Create - Nop + Append = 0x00002 + Nop = 0x00004 ) type UploadTaskInfo struct { diff --git a/pkg/filesystem/hooks.go b/pkg/filesystem/hooks.go index f07b370..9d2eaec 100644 --- a/pkg/filesystem/hooks.go +++ b/pkg/filesystem/hooks.go @@ -151,6 +151,7 @@ func HookCleanFileContent(ctx context.Context, fs *FileSystem, file fsctx.FileHe File: ioutil.NopCloser(strings.NewReader("")), SavePath: file.Info().SavePath, Size: 0, + Model: fsctx.Overwrite, }) } diff --git a/pkg/filesystem/upload.go b/pkg/filesystem/upload.go index e63731b..f8bec75 100644 --- a/pkg/filesystem/upload.go +++ b/pkg/filesystem/upload.go @@ -51,7 +51,7 @@ func (fs *FileSystem) Upload(ctx context.Context, file *fsctx.FileStream) (err e go fs.CancelUpload(ctx, savePath, file) // 保存文件 - if file.Mode != fsctx.Nop { + if file.Mode&fsctx.Nop != fsctx.Nop { err = fs.Handler.Put(ctx, file) if err != nil { fs.Trigger(ctx, "AfterUploadFailed", file) @@ -73,7 +73,7 @@ func (fs *FileSystem) Upload(ctx context.Context, file *fsctx.FileStream) (err e return err } - if file.Mode == fsctx.Create { + if file.Mode&fsctx.Overwrite == 0 { fileInfo := file.Info() util.Log().Info( "新文件PUT:%s , 大小:%d, 上传者:%s", diff --git a/pkg/task/compress.go b/pkg/task/compress.go index cf45ace..d4a3400 100644 --- a/pkg/task/compress.go +++ b/pkg/task/compress.go @@ -7,7 +7,6 @@ import ( model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem" - "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" "github.com/cloudreve/Cloudreve/v3/pkg/util" ) @@ -107,7 +106,7 @@ func (job *CompressTask) Do() { job.TaskModel.SetProgress(TransferringProgress) // 上传文件 - err = fs.UploadFromPath(ctx, zipFile, job.TaskProps.Dst, true, fsctx.Create) + err = fs.UploadFromPath(ctx, zipFile, job.TaskProps.Dst, true, 0) if err != nil { job.SetErrorMsg(err.Error()) return diff --git a/pkg/task/slavetask/transfer.go b/pkg/task/slavetask/transfer.go index c9653cf..20c5fcc 100644 --- a/pkg/task/slavetask/transfer.go +++ b/pkg/task/slavetask/transfer.go @@ -119,7 +119,6 @@ func (job *TransferTask) Do() { err = fs.Handler.Put(context.Background(), &fsctx.FileStream{ File: file, - Mode: fsctx.Create, SavePath: job.Req.Dst, Size: uint64(size), }) diff --git a/pkg/task/tranfer.go b/pkg/task/tranfer.go index 33ed4e2..4983c4f 100644 --- a/pkg/task/tranfer.go +++ b/pkg/task/tranfer.go @@ -123,11 +123,10 @@ func (job *TransferTask) Do() { Size: job.TaskProps.SrcSizes[file], Name: path.Base(dst), VirtualPath: path.Dir(dst), - Mode: fsctx.Create, }) } else { // 主机节点中转 - err = fs.UploadFromPath(context.Background(), file, dst, true, fsctx.Create) + err = fs.UploadFromPath(context.Background(), file, dst, true, 0) } if err != nil { diff --git a/pkg/webdav/webdav.go b/pkg/webdav/webdav.go index 7f98dfa..241ffd9 100644 --- a/pkg/webdav/webdav.go +++ b/pkg/webdav/webdav.go @@ -355,6 +355,7 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request, fs *filesyst fs.Use("AfterValidateFailed", filesystem.HookCleanFileContent) fs.Use("AfterValidateFailed", filesystem.HookClearFileSize) ctx = context.WithValue(ctx, fsctx.FileModelCtx, *originFile) + fileData.Mode |= fsctx.Overwrite } else { // 给文件系统分配钩子 fs.Use("BeforeUpload", filesystem.HookValidateFile) @@ -364,9 +365,6 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request, fs *filesyst fs.Use("AfterUpload", filesystem.GenericAfterUpload) fs.Use("AfterUpload", filesystem.HookGenerateThumb) fs.Use("AfterValidateFailed", filesystem.HookDeleteTempFile) - - // 禁止覆盖 - fileData.Mode = fsctx.Create } // 执行上传 diff --git a/routers/controllers/slave.go b/routers/controllers/slave.go index 66d712d..73a27c4 100644 --- a/routers/controllers/slave.go +++ b/routers/controllers/slave.go @@ -73,10 +73,10 @@ func SlaveUpload(c *gin.Context) { fs.Use("AfterUpload", filesystem.SlaveAfterUpload) fs.Use("AfterValidateFailed", filesystem.HookDeleteTempFile) - // 是否允许覆盖 - if c.Request.Header.Get("X-Cr-Overwrite") == "false" { - fileData.Mode = fsctx.Create - } + //// 是否允许覆盖 + //if c.Request.Header.Get("X-Cr-Overwrite") == "false" { + // fileData.Mode = fsctx.Create + //} // 执行上传 err = fs.Upload(ctx, &fileData) diff --git a/service/explorer/file.go b/service/explorer/file.go index 622257b..19f84e3 100644 --- a/service/explorer/file.go +++ b/service/explorer/file.go @@ -65,7 +65,6 @@ func (service *SingleFileService) Create(c *gin.Context) serializer.Response { Size: 0, VirtualPath: path.Dir(service.Path), Name: path.Base(service.Path), - Mode: fsctx.Create, }) if err != nil { return serializer.Err(serializer.CodeUploadFailed, err.Error(), err) @@ -375,6 +374,7 @@ func (service *FileIDService) PutContent(ctx context.Context, c *gin.Context) se MIMEType: c.Request.Header.Get("Content-Type"), File: c.Request.Body, Size: fileSize, + Model: fsctx.Overwrite, } // 创建文件系统 diff --git a/service/explorer/upload.go b/service/explorer/upload.go index 15cf426..c8c989e 100644 --- a/service/explorer/upload.go +++ b/service/explorer/upload.go @@ -148,7 +148,7 @@ func processChunkUpload(ctx context.Context, c *gin.Context, fs *filesystem.File Name: session.Name, VirtualPath: session.VirtualPath, SavePath: session.SavePath, - Mode: fsctx.Append, + Mode: fsctx.Append | fsctx.Overwrite, AppendStart: chunkSize * uint64(index), Model: file, LastModified: session.LastModified,