From 15e3e3db5c4ee2d176aa28d821c66a5498c724fc Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Wed, 16 Mar 2022 11:44:40 +0800 Subject: [PATCH 01/21] Fix: unused import and Ping router return wrong version --- routers/controllers/site.go | 2 +- routers/controllers/slave.go | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/routers/controllers/site.go b/routers/controllers/site.go index b066397c..5598605e 100644 --- a/routers/controllers/site.go +++ b/routers/controllers/site.go @@ -49,7 +49,7 @@ func Ping(c *gin.Context) { c.JSON(200, serializer.Response{ Code: 0, - Data: conf.BackendVersion + conf.IsPro, + Data: version, }) } diff --git a/routers/controllers/slave.go b/routers/controllers/slave.go index 5a0d2774..e1e7de22 100644 --- a/routers/controllers/slave.go +++ b/routers/controllers/slave.go @@ -2,8 +2,6 @@ package controllers import ( "context" - "net/url" - "strconv" "github.com/cloudreve/Cloudreve/v3/pkg/request" "github.com/cloudreve/Cloudreve/v3/pkg/serializer" From b6efca187873a0b500ff8b8d10cf486bde21b8a2 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Sun, 20 Mar 2022 11:16:25 +0800 Subject: [PATCH 02/21] Feat: uploading OneDrive files in client side --- middleware/auth.go | 14 +++---------- models/migration.go | 4 ++-- pkg/filesystem/driver/onedrive/api.go | 24 ++++++++--------------- pkg/filesystem/driver/onedrive/handler.go | 18 +++++------------ pkg/filesystem/driver/onedrive/types.go | 3 --- pkg/filesystem/driver/remote/client.go | 2 +- pkg/filesystem/upload.go | 6 +++--- pkg/serializer/upload.go | 23 +++++++++++----------- routers/router.go | 5 +++-- service/callback/upload.go | 14 ++++++------- 10 files changed, 43 insertions(+), 70 deletions(-) diff --git a/middleware/auth.go b/middleware/auth.go index 83f972b7..34237d54 100644 --- a/middleware/auth.go +++ b/middleware/auth.go @@ -2,6 +2,7 @@ package middleware import ( "github.com/cloudreve/Cloudreve/v3/pkg/filesystem" + "github.com/cloudreve/Cloudreve/v3/pkg/mq" "net/http" model "github.com/cloudreve/Cloudreve/v3/models" @@ -284,19 +285,10 @@ func UpyunCallbackAuth() gin.HandlerFunc { } // OneDriveCallbackAuth OneDrive回调签名验证 -// TODO 解耦 func OneDriveCallbackAuth() gin.HandlerFunc { return func(c *gin.Context) { - //// 验证key并查找用户 - //resp, _ := uploadCallbackCheck(c) - //if resp.Code != 0 { - // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg}) - // c.Abort() - // return - //} - // - //// 发送回调结束信号 - //onedrive.FinishCallback(c.Param("key")) + // 发送回调结束信号 + mq.GlobalMQ.Publish(c.Param("sessionID"), mq.Message{}) c.Next() } diff --git a/models/migration.go b/models/migration.go index 1053e654..6b557fc5 100644 --- a/models/migration.go +++ b/models/migration.go @@ -124,8 +124,8 @@ func addDefaultSettings() { {Name: "share_download_session_timeout", Value: `2073600`, Type: "timeout"}, {Name: "onedrive_callback_check", Value: `20`, Type: "timeout"}, {Name: "folder_props_timeout", Value: `300`, Type: "timeout"}, - {Name: "onedrive_chunk_retries", Value: `1`, Type: "retry"}, - {Name: "slave_chunk_retries", Value: `1`, Type: "retry"}, + {Name: "onedrive_chunk_retries", Value: `5`, Type: "retry"}, + {Name: "slave_chunk_retries", Value: `5`, Type: "retry"}, {Name: "onedrive_source_timeout", Value: `1800`, Type: "timeout"}, {Name: "reset_after_upload_failed", Value: `0`, Type: "upload"}, {Name: "login_captcha", Value: `0`, Type: "login"}, diff --git a/pkg/filesystem/driver/onedrive/api.go b/pkg/filesystem/driver/onedrive/api.go index c9e7ff27..1b6d9e16 100644 --- a/pkg/filesystem/driver/onedrive/api.go +++ b/pkg/filesystem/driver/onedrive/api.go @@ -19,6 +19,7 @@ import ( model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/pkg/cache" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" + "github.com/cloudreve/Cloudreve/v3/pkg/mq" "github.com/cloudreve/Cloudreve/v3/pkg/request" "github.com/cloudreve/Cloudreve/v3/pkg/util" ) @@ -487,9 +488,9 @@ func (client *Client) GetThumbURL(ctx context.Context, dst string, w, h uint) (s // MonitorUpload 监控客户端分片上传进度 func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size uint64, ttl int64) { // 回调完成通知chan - callbackChan := make(chan bool) - callbackSignal.Store(callbackKey, callbackChan) - defer callbackSignal.Delete(callbackKey) + callbackChan := mq.GlobalMQ.Subscribe(callbackKey, 1) + defer mq.GlobalMQ.Unsubscribe(callbackKey, callbackChan) + timeout := model.GetIntSetting("onedrive_monitor_timeout", 600) interval := model.GetIntSetting("onedrive_callback_check", 20) @@ -514,16 +515,16 @@ func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size ui if resErr, ok := err.(*RespError); ok { if resErr.APIError.Code == "itemNotFound" { util.Log().Debug("上传会话已完成，稍后检查回调") - time.Sleep(time.Duration(interval) * time.Second) - util.Log().Debug("开始检查回调") - _, ok := cache.Get("callback_" + callbackKey) - if ok { + select { + case <-time.After(time.Duration(interval) * time.Second): util.Log().Warning("未发送回调，删除文件") cache.Deletes([]string{callbackKey}, "callback_") _, err = client.Delete(context.Background(), []string{path}) if err != nil { util.Log().Warning("无法删除未回调的文件，%s", err) } + case <-callbackChan: + util.Log().Debug("客户端完成回调") } return } @@ -560,15 +561,6 @@ func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size ui } } -// FinishCallback 向Monitor发送回调结束信号 -func FinishCallback(key string) { - if signal, ok := callbackSignal.Load(key); ok { - if signalChan, ok := signal.(chan bool); ok { - close(signalChan) - } - } -} - func sysError(err error) *RespError { return &RespError{APIError: APIError{ Code: "system", diff --git a/pkg/filesystem/driver/onedrive/handler.go b/pkg/filesystem/driver/onedrive/handler.go index 2fa85b3a..4b76bfc5 100644 --- a/pkg/filesystem/driver/onedrive/handler.go +++ b/pkg/filesystem/driver/onedrive/handler.go @@ -226,16 +226,6 @@ func (handler Driver) replaceSourceHost(origin string) (string, error) { func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) { fileInfo := file.Info() - // 如果小于4MB，则由服务端中转 - if fileInfo.Size <= SmallFileSize { - return nil, nil - } - - // 生成回调地址 - siteURL := model.GetSiteURL() - apiBaseURI, _ := url.Parse("/api/v3/callback/onedrive/finish/" + uploadSession.Key) - apiURL := siteURL.ResolveReference(apiBaseURI) - uploadURL, err := handler.Client.CreateUploadSession(ctx, fileInfo.SavePath, WithConflictBehavior("fail")) if err != nil { return nil, err @@ -244,13 +234,15 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria // 监控回调及上传 go handler.Client.MonitorUpload(uploadURL, uploadSession.Key, fileInfo.SavePath, fileInfo.Size, ttl) + uploadSession.OneDriveUploadURL = uploadURL return &serializer.UploadCredential{ - Policy: uploadURL, - Token: apiURL.String(), + SessionID: uploadSession.Key, + ChunkSize: handler.Policy.OptionsSerialized.ChunkSize, + UploadURLs: []string{uploadURL}, }, nil } // 取消上传凭证 func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error { - return nil + return handler.Client.DeleteUploadSession(ctx, uploadSession.OneDriveUploadURL) } diff --git a/pkg/filesystem/driver/onedrive/types.go b/pkg/filesystem/driver/onedrive/types.go index 9dc14fa4..aefa6385 100644 --- a/pkg/filesystem/driver/onedrive/types.go +++ b/pkg/filesystem/driver/onedrive/types.go @@ -3,7 +3,6 @@ package onedrive import ( "encoding/gob" "net/url" - "sync" ) // RespError 接口返回错误 @@ -148,5 +147,3 @@ func init() { func (chunk *Chunk) IsLast() bool { return chunk.Total-chunk.Offset == chunk.ChunkSize } - -var callbackSignal sync.Map diff --git a/pkg/filesystem/driver/remote/client.go b/pkg/filesystem/driver/remote/client.go index ab764f51..17bbb929 100644 --- a/pkg/filesystem/driver/remote/client.go +++ b/pkg/filesystem/driver/remote/client.go @@ -90,7 +90,7 @@ func (c *remoteClient) Upload(ctx context.Context, file fsctx.FileHeader) error // Initial chunk groups chunks := chunk.NewChunkGroup(file, c.policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{ - Max: model.GetIntSetting("onedrive_chunk_retries", 1), + Max: model.GetIntSetting("slave_chunk_retries", 5), Sleep: chunkRetrySleep, }) diff --git a/pkg/filesystem/upload.go b/pkg/filesystem/upload.go index 38d42eff..71876ae6 100644 --- a/pkg/filesystem/upload.go +++ b/pkg/filesystem/upload.go @@ -174,9 +174,6 @@ func (fs *FileSystem) CreateUploadSession(ctx context.Context, file *fsctx.FileS fs.Use("BeforeUpload", HookValidateFile) fs.Use("BeforeUpload", HookValidateCapacity) - if !fs.Policy.IsUploadPlaceholderWithSize() { - fs.Use("AfterUpload", HookClearFileHeaderSize) - } // 验证文件规格 if err := fs.Upload(ctx, file); err != nil { @@ -202,6 +199,9 @@ func (fs *FileSystem) CreateUploadSession(ctx context.Context, file *fsctx.FileS } // 创建占位符 + if !fs.Policy.IsUploadPlaceholderWithSize() { + fs.Use("AfterUpload", HookClearFileHeaderSize) + } fs.Use("AfterUpload", GenericAfterUpload) if err := fs.Upload(ctx, file); err != nil { return nil, err diff --git a/pkg/serializer/upload.go b/pkg/serializer/upload.go index 225c4934..485b8430 100644 --- a/pkg/serializer/upload.go +++ b/pkg/serializer/upload.go @@ -31,22 +31,23 @@ type UploadCredential struct { Path string `json:"path"` // 存储路径 AccessKey string `json:"ak"` KeyTime string `json:"key_time,omitempty"` // COS用有效期 - Callback string `json:"callback,omitempty"` // 回调地址 Key string `json:"key,omitempty"` // 文件标识符，通常为回调key + Callback string `json:"callback,omitempty"` // 回调地址 } // UploadSession 上传会话 type UploadSession struct { - Key string // 上传会话 GUID - UID uint // 发起者 - VirtualPath string // 用户文件路径，不含文件名 - Name string // 文件名 - Size uint64 // 文件大小 - SavePath string // 物理存储路径，包含物理文件名 - LastModified *time.Time // 可选的文件最后修改日期 - Policy model.Policy - Callback string // 回调 URL 地址 - CallbackSecret string // 回调 URL + Key string // 上传会话 GUID + UID uint // 发起者 + VirtualPath string // 用户文件路径，不含文件名 + Name string // 文件名 + Size uint64 // 文件大小 + SavePath string // 物理存储路径，包含物理文件名 + LastModified *time.Time // 可选的文件最后修改日期 + Policy model.Policy + Callback string // 回调 URL 地址 + CallbackSecret string // 回调 URL + OneDriveUploadURL string } // UploadCallback 上传回调正文 diff --git a/routers/router.go b/routers/router.go index 21f42ae8..9f61a335 100644 --- a/routers/router.go +++ b/routers/router.go @@ -264,11 +264,12 @@ func InitMasterRouter() *gin.Engine { { // 文件上传完成 onedrive.POST( - "finish/:key", + "finish/:sessionID", + middleware.UseUploadSession("onedrive"), middleware.OneDriveCallbackAuth(), controllers.OneDriveCallback, ) - // 文件上传完成 + // OAuth 完成 onedrive.GET( "auth", controllers.OneDriveOAuth, diff --git a/service/callback/upload.go b/service/callback/upload.go index df5d1807..4f0495e3 100644 --- a/service/callback/upload.go +++ b/service/callback/upload.go @@ -50,7 +50,6 @@ type UpyunCallbackService struct { // OneDriveCallback OneDrive 客户端回调正文 type OneDriveCallback struct { - ID string `json:"id" binding:"required"` Meta *onedrive.FileInfo } @@ -165,22 +164,21 @@ func (service *OneDriveCallback) PreProcess(c *gin.Context) serializer.Response defer fs.Recycle() // 获取回调会话 - callbackSessionRaw, _ := c.Get("callbackSession") - callbackSession := callbackSessionRaw.(*serializer.UploadSession) + uploadSession := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession) // 获取文件信息 - info, err := fs.Handler.(onedrive.Driver).Client.Meta(context.Background(), service.ID, "") + info, err := fs.Handler.(onedrive.Driver).Client.Meta(context.Background(), "", uploadSession.SavePath) if err != nil { return serializer.Err(serializer.CodeUploadFailed, "文件元信息查询失败", err) } // 验证与回调会话中是否一致 - actualPath := strings.TrimPrefix(callbackSession.SavePath, "/") - isSizeCheckFailed := callbackSession.Size != info.Size + actualPath := strings.TrimPrefix(uploadSession.SavePath, "/") + isSizeCheckFailed := uploadSession.Size != info.Size - // SharePoint 会对 Office 文档增加 meta data 导致文件大小不一致，这里增加 10 KB 宽容 + // SharePoint 会对 Office 文档增加 meta data 导致文件大小不一致，这里增加 100 KB 宽容 // See: https://github.com/OneDrive/onedrive-api-docs/issues/935 - if strings.Contains(fs.Policy.OptionsSerialized.OdDriver, "sharepoint.com") && isSizeCheckFailed && (info.Size > callbackSession.Size) && (info.Size-callbackSession.Size <= 10240) { + if strings.Contains(fs.Policy.OptionsSerialized.OdDriver, "sharepoint.com") && isSizeCheckFailed && (info.Size > uploadSession.Size) && (info.Size-uploadSession.Size <= 102400) { isSizeCheckFailed = false } From 58021611027f9a3f755f8e086bfdcc8347b17cf6 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Sun, 20 Mar 2022 11:17:04 +0800 Subject: [PATCH 03/21] Fix: inherited policy ID didn't pass through second layer in Folder / version verification in Ping router --- assets | 2 +- service/admin/policy.go | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/assets b/assets index e0da8f48..8a94d68f 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit e0da8f48856e3fb6e3e9cc920a32390ca132935e +Subproject commit 8a94d68f17c170a1055e2d57c2a2a55d18868a9e diff --git a/service/admin/policy.go b/service/admin/policy.go index 5c648ecd..6d44b6b3 100644 --- a/service/admin/policy.go +++ b/service/admin/policy.go @@ -207,8 +207,12 @@ func (service *SlavePingService) Test() serializer.Response { return serializer.ParamErr("从机无法向主机发送回调请求，请检查主机端 参数设置 - 站点信息 - 站点URL设置，并确保从机可以连接到此地址，"+err.Error(), nil) } - if res.Data.(string) != conf.BackendVersion { - return serializer.ParamErr("Cloudreve版本不一致，主机："+res.Data.(string)+"，从机："+conf.BackendVersion, nil) + version := conf.BackendVersion + if conf.IsPro == "true" { + version += "-pro" + } + if res.Data.(string) != version { + return serializer.ParamErr("Cloudreve版本不一致，主机："+res.Data.(string)+"，从机："+version, nil) } return serializer.Response{} From 015ccd502606c1ba22a8a3178cd9896b3a7458e4 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Sun, 20 Mar 2022 11:20:09 +0800 Subject: [PATCH 04/21] Feat: use new ChunkManager for OneDrive API client --- pkg/filesystem/chunk/chunk.go | 16 ++++++ pkg/filesystem/driver/onedrive/api.go | 77 +++++++++------------------ 2 files changed, 41 insertions(+), 52 deletions(-) diff --git a/pkg/filesystem/chunk/chunk.go b/pkg/filesystem/chunk/chunk.go index e8a63f7d..5eb28932 100644 --- a/pkg/filesystem/chunk/chunk.go +++ b/pkg/filesystem/chunk/chunk.go @@ -60,6 +60,7 @@ func (c *ChunkGroup) Process(processor ChunkProcessFunc) error { return err } + util.Log().Debug("Chunk %d processed", c.currentIndex) return nil } @@ -68,6 +69,16 @@ func (c *ChunkGroup) Start() int64 { return int64(uint64(c.Index()) * c.chunkSize) } +// Total returns the total length current chunk +func (c *ChunkGroup) Total() int64 { + return int64(c.fileInfo.Size) +} + +// RangeHeader returns header value of Content-Range +func (c *ChunkGroup) RangeHeader() string { + return fmt.Sprintf("bytes %d-%d/%d", c.Start(), c.Start()+c.Length()-1, c.Total()) +} + // Index returns current chunk index, starts from 0 func (c *ChunkGroup) Index() int { return c.currentIndex @@ -89,3 +100,8 @@ func (c *ChunkGroup) Length() int64 { return int64(contentLength) } + +// IsLast returns if current chunk is the last one +func (c *ChunkGroup) IsLast() bool { + return c.Index() == int(c.chunkNum-1) +} diff --git a/pkg/filesystem/driver/onedrive/api.go b/pkg/filesystem/driver/onedrive/api.go index 1b6d9e16..2aa49de5 100644 --- a/pkg/filesystem/driver/onedrive/api.go +++ b/pkg/filesystem/driver/onedrive/api.go @@ -1,7 +1,6 @@ package onedrive import ( - "bytes" "context" "encoding/json" "errors" @@ -18,6 +17,8 @@ import ( model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/pkg/cache" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" "github.com/cloudreve/Cloudreve/v3/pkg/mq" "github.com/cloudreve/Cloudreve/v3/pkg/request" @@ -30,7 +31,8 @@ const ( // ChunkSize 服务端中转分片上传分片大小 ChunkSize uint64 = 10 * 1024 * 1024 // ListRetry 列取请求重试次数 - ListRetry = 1 + ListRetry = 1 + chunkRetrySleep = time.Second * 5 ) // GetSourcePath 获取文件的绝对路径 @@ -220,28 +222,21 @@ func (client *Client) GetUploadSessionStatus(ctx context.Context, uploadURL stri } // UploadChunk 上传分片 -func (client *Client) UploadChunk(ctx context.Context, uploadURL string, chunk *Chunk) (*UploadSessionResponse, error) { +func (client *Client) UploadChunk(ctx context.Context, uploadURL string, content io.Reader, current *chunk.ChunkGroup) (*UploadSessionResponse, error) { res, err := client.request( - ctx, "PUT", uploadURL, bytes.NewReader(chunk.Data[0:chunk.ChunkSize]), - request.WithContentLength(int64(chunk.ChunkSize)), + ctx, "PUT", uploadURL, content, + request.WithContentLength(current.Length()), request.WithHeader(http.Header{ - "Content-Range": {fmt.Sprintf("bytes %d-%d/%d", chunk.Offset, chunk.Offset+chunk.ChunkSize-1, chunk.Total)}, + "Content-Range": {current.RangeHeader()}, }), request.WithoutHeader([]string{"Authorization", "Content-Type"}), request.WithTimeout(time.Duration(300)*time.Second), ) if err != nil { - // 如果重试次数小于限制，5秒后重试 - if chunk.Retried < model.GetIntSetting("onedrive_chunk_retries", 1) { - chunk.Retried++ - util.Log().Debug("分片偏移%d上传失败[%s]，5秒钟后重试", chunk.Offset, err) - time.Sleep(time.Duration(5) * time.Second) - return client.UploadChunk(ctx, uploadURL, chunk) - } - return nil, err + return nil, fmt.Errorf("failed to upload OneDrive chunk #%d: %w", current.Index(), err) } - if chunk.IsLast() { + if current.IsLast() { return nil, nil } @@ -282,46 +277,24 @@ func (client *Client) Upload(ctx context.Context, file fsctx.FileHeader) error { return err } - offset := 0 - chunkNum := size / int(ChunkSize) - if size%int(ChunkSize) != 0 { - chunkNum++ - } - - chunkData := make([]byte, ChunkSize) - - for i := 0; i < chunkNum; i++ { - select { - case <-ctx.Done(): - util.Log().Debug("OneDrive 客户端取消") - return ErrClientCanceled - default: - // 分块 - chunkSize := int(ChunkSize) - if size-offset < chunkSize { - chunkSize = size - offset - } - - // 因为后面需要错误重试，这里要把分片内容读到内存中 - chunkContent := chunkData[:chunkSize] - _, err := io.ReadFull(file, chunkContent) + // Initial chunk groups + chunks := chunk.NewChunkGroup(file, client.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{ + Max: model.GetIntSetting("onedrive_chunk_retries", 5), + Sleep: chunkRetrySleep, + }) - chunk := Chunk{ - Offset: offset, - ChunkSize: chunkSize, - Total: size, - Data: chunkContent, - } + uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error { + _, err := client.UploadChunk(ctx, uploadURL, content, current) + return err + } - // 上传 - _, err = client.UploadChunk(ctx, uploadURL, &chunk) - if err != nil { - return err - } - offset += chunkSize + // upload chunks + for chunks.Next() { + if err := chunks.Process(uploadFunc); err != nil { + return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err) } - } + return nil } @@ -354,7 +327,7 @@ func (client *Client) SimpleUpload(ctx context.Context, dst string, body io.Read if v, ok := ctx.Value(fsctx.RetryCtx).(int); ok { retried = v } - if retried < model.GetIntSetting("onedrive_chunk_retries", 1) { + if retried < model.GetIntSetting("onedrive_chunk_retries", 5) { retried++ util.Log().Debug("文件[%s]上传失败[%s]，5秒钟后重试", dst, err) time.Sleep(time.Duration(5) * time.Second) From 0df9529b32322307a962ab6aa522539586e17702 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Sun, 20 Mar 2022 11:23:55 +0800 Subject: [PATCH 05/21] Feat: generating token and callback url for OSS muiltpart upload, support resume upload in sever-side uploading for OSS --- go.mod | 5 +- go.sum | 13 +-- middleware/auth.go | 24 ++-- models/migration.go | 3 +- models/policy.go | 12 +- pkg/filesystem/chunk/chunk.go | 5 + pkg/filesystem/driver/onedrive/api.go | 4 +- pkg/filesystem/driver/onedrive/api_test.go | 6 +- pkg/filesystem/driver/oss/handler.go | 126 ++++++++++++++------- pkg/filesystem/driver/remote/client.go | 2 +- pkg/serializer/upload.go | 8 +- routers/router.go | 3 +- service/callback/upload.go | 21 ++++ 13 files changed, 148 insertions(+), 84 deletions(-) diff --git a/go.mod b/go.mod index 349fedf7..a7de23c3 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/DATA-DOG/go-sqlmock v1.3.3 - github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible + github.com/HFO4/aliyun-oss-go-sdk v2.2.2+incompatible github.com/aws/aws-sdk-go v1.31.5 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4 @@ -20,12 +20,11 @@ require ( github.com/gomodule/redigo v2.0.0+incompatible github.com/google/go-querystring v1.0.0 github.com/gorilla/websocket v1.4.1 - github.com/hashicorp/go-version v1.2.0 + github.com/hashicorp/go-version v1.3.0 github.com/jinzhu/gorm v1.9.11 github.com/juju/ratelimit v1.0.1 github.com/mattn/go-colorable v0.1.4 // indirect github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2 - github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/pkg/errors v0.9.1 github.com/pquerna/otp v1.2.0 github.com/qiniu/api.v7/v7 v7.4.0 diff --git a/go.sum b/go.sum index 402c072f..b8568f53 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7h github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/HFO4/aliyun-oss-go-sdk v2.2.2+incompatible h1:cWidHgoT2ye1YwzP5sz7xkqOK6I84mqrayJSU23oOUo= +github.com/HFO4/aliyun-oss-go-sdk v2.2.2+incompatible/go.mod h1:8KDiKVrHK/UbXAhj+iQGp1m40rQa+UAvzBi7m22KywI= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -12,9 +14,6 @@ github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7I github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/aliyun/aliyun-oss-go-sdk v2.0.0/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible h1:A3oZlWPD/Poa19FvNbw+Zu4yKAurDBTjlRDilYGBiS4= -github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/aws/aws-sdk-go v1.31.5 h1:DFA7BzTydO4etqsTja+x7UfkOKQUv1xzEluLvNk81L0= github.com/aws/aws-sdk-go v1.31.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= @@ -119,8 +118,8 @@ github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9R github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw= +github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jinzhu/gorm v1.9.11 h1:gaHGvE+UnWGlbWG4Y3FUwY1EcZ5n6S9WtqBA/uySMLE= @@ -179,8 +178,6 @@ github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2/go.mod h1:wAQ github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= -github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -247,7 +244,6 @@ golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/image v0.0.0-20190501045829-6d32002ffd75 h1:TbGuee8sSq15Iguxu4deQ7+Bqq/d2rsQejGcEtADAMQ= golang.org/x/image v0.0.0-20190501045829-6d32002ffd75/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ= golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= @@ -289,7 +285,6 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/middleware/auth.go b/middleware/auth.go index 34237d54..28c00fc1 100644 --- a/middleware/auth.go +++ b/middleware/auth.go @@ -2,7 +2,9 @@ package middleware import ( "github.com/cloudreve/Cloudreve/v3/pkg/filesystem" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/oss" "github.com/cloudreve/Cloudreve/v3/pkg/mq" + "github.com/cloudreve/Cloudreve/v3/pkg/util" "net/http" model "github.com/cloudreve/Cloudreve/v3/models" @@ -209,21 +211,13 @@ func QiniuCallbackAuth() gin.HandlerFunc { // OSSCallbackAuth 阿里云OSS回调签名验证 func OSSCallbackAuth() gin.HandlerFunc { return func(c *gin.Context) { - //// 验证key并查找用户 - //resp, _ := uploadCallbackCheck(c) - //if resp.Code != 0 { - // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg}) - // c.Abort() - // return - //} - // - //err := oss.VerifyCallbackSignature(c.Request) - //if err != nil { - // util.Log().Debug("回调签名验证失败，%s", err) - // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名验证失败"}) - // c.Abort() - // return - //} + err := oss.VerifyCallbackSignature(c.Request) + if err != nil { + util.Log().Debug("回调签名验证失败，%s", err) + c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名验证失败"}) + c.Abort() + return + } c.Next() } diff --git a/models/migration.go b/models/migration.go index 6b557fc5..e383f7e3 100644 --- a/models/migration.go +++ b/models/migration.go @@ -124,8 +124,7 @@ func addDefaultSettings() { {Name: "share_download_session_timeout", Value: `2073600`, Type: "timeout"}, {Name: "onedrive_callback_check", Value: `20`, Type: "timeout"}, {Name: "folder_props_timeout", Value: `300`, Type: "timeout"}, - {Name: "onedrive_chunk_retries", Value: `5`, Type: "retry"}, - {Name: "slave_chunk_retries", Value: `5`, Type: "retry"}, + {Name: "chunk_retries", Value: `5`, Type: "retry"}, {Name: "onedrive_source_timeout", Value: `1800`, Type: "timeout"}, {Name: "reset_after_upload_failed", Value: `0`, Type: "upload"}, {Name: "login_captcha", Value: `0`, Type: "login"}, diff --git a/models/policy.go b/models/policy.go index 66716925..0f7f2999 100644 --- a/models/policy.go +++ b/models/policy.go @@ -60,6 +60,8 @@ type PolicyOption struct { ServerSideEndpoint string `json:"server_side_endpoint,omitempty"` // 分片上传的分片大小 ChunkSize uint64 `json:"chunk_size,omitempty"` + // 分片上传时是否需要预留空间 + PlaceholderWithSize bool `json:"placeholder_with_size,omitempty"` } var thumbSuffix = map[string][]string{ @@ -226,7 +228,15 @@ func (policy *Policy) IsThumbGenerateNeeded() bool { // IsUploadPlaceholderWithSize 返回此策略创建上传会话时是否需要预留空间 func (policy *Policy) IsUploadPlaceholderWithSize() bool { - return policy.Type == "remote" + if policy.Type == "remote" { + return true + } + + if policy.Type == "onedrive" || policy.Type == "oss" { + return policy.OptionsSerialized.PlaceholderWithSize + } + + return false } // CanStructureBeListed 返回存储策略是否能被前台列物理目录 diff --git a/pkg/filesystem/chunk/chunk.go b/pkg/filesystem/chunk/chunk.go index 5eb28932..82e6cac7 100644 --- a/pkg/filesystem/chunk/chunk.go +++ b/pkg/filesystem/chunk/chunk.go @@ -74,6 +74,11 @@ func (c *ChunkGroup) Total() int64 { return int64(c.fileInfo.Size) } +// Num returns the total chunk number +func (c *ChunkGroup) Num() int { + return int(c.chunkNum) +} + // RangeHeader returns header value of Content-Range func (c *ChunkGroup) RangeHeader() string { return fmt.Sprintf("bytes %d-%d/%d", c.Start(), c.Start()+c.Length()-1, c.Total()) diff --git a/pkg/filesystem/driver/onedrive/api.go b/pkg/filesystem/driver/onedrive/api.go index 2aa49de5..14c12789 100644 --- a/pkg/filesystem/driver/onedrive/api.go +++ b/pkg/filesystem/driver/onedrive/api.go @@ -279,7 +279,7 @@ func (client *Client) Upload(ctx context.Context, file fsctx.FileHeader) error { // Initial chunk groups chunks := chunk.NewChunkGroup(file, client.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{ - Max: model.GetIntSetting("onedrive_chunk_retries", 5), + Max: model.GetIntSetting("chunk_retries", 5), Sleep: chunkRetrySleep, }) @@ -327,7 +327,7 @@ func (client *Client) SimpleUpload(ctx context.Context, dst string, body io.Read if v, ok := ctx.Value(fsctx.RetryCtx).(int); ok { retried = v } - if retried < model.GetIntSetting("onedrive_chunk_retries", 5) { + if retried < model.GetIntSetting("chunk_retries", 5) { retried++ util.Log().Debug("文件[%s]上传失败[%s]，5秒钟后重试", dst, err) time.Sleep(time.Duration(5) * time.Second) diff --git a/pkg/filesystem/driver/onedrive/api_test.go b/pkg/filesystem/driver/onedrive/api_test.go index 02fb2941..8acc6dbe 100644 --- a/pkg/filesystem/driver/onedrive/api_test.go +++ b/pkg/filesystem/driver/onedrive/api_test.go @@ -535,7 +535,7 @@ func TestClient_UploadChunk(t *testing.T) { // 最后分片，第一次失败，重试后成功 { - cache.Set("setting_onedrive_chunk_retries", "1", 0) + cache.Set("setting_chunk_retries", "1", 0) client.Credential.ExpiresIn = 0 go func() { time.Sleep(time.Duration(2) * time.Second) @@ -641,7 +641,7 @@ func TestClient_SimpleUpload(t *testing.T) { client, _ := NewClient(&model.Policy{}) client.Credential.AccessToken = "AccessToken" client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix() - cache.Set("setting_onedrive_chunk_retries", "1", 0) + cache.Set("setting_chunk_retries", "1", 0) // 请求失败，并重试 { @@ -651,7 +651,7 @@ func TestClient_SimpleUpload(t *testing.T) { asserts.Nil(res) } - cache.Set("setting_onedrive_chunk_retries", "0", 0) + cache.Set("setting_chunk_retries", "0", 0) // 返回未知响应 { client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix() diff --git a/pkg/filesystem/driver/oss/handler.go b/pkg/filesystem/driver/oss/handler.go index bbe3fc18..10e7fdd1 100644 --- a/pkg/filesystem/driver/oss/handler.go +++ b/pkg/filesystem/driver/oss/handler.go @@ -2,8 +2,6 @@ package oss import ( "context" - "crypto/hmac" - "crypto/sha1" "encoding/base64" "encoding/json" "errors" @@ -15,8 +13,10 @@ import ( "strings" "time" - "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/HFO4/aliyun-oss-go-sdk/oss" model "github.com/cloudreve/Cloudreve/v3/models" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response" "github.com/cloudreve/Cloudreve/v3/pkg/request" @@ -48,6 +48,10 @@ type Driver struct { type key int const ( + chunkRetrySleep = time.Duration(5) * time.Second + + // MultiPartUploadThreshold 服务端使用分片上传的阈值 + MultiPartUploadThreshold uint64 = 5 * (1 << 30) // 5GB // VersionID 文件版本标识 VersionID key = iota ) @@ -244,12 +248,34 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { } // 上传文件 - err := handler.bucket.PutObject(fileInfo.SavePath, file, options...) + //err := handler.bucket.PutObject(fileInfo.SavePath, file, options...) + //if err != nil { + // return err + //} + + imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...) if err != nil { + return fmt.Errorf("failed to initiate multipart upload: %w", err) + } + + chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{ + Max: model.GetIntSetting("chunk_retries", 5), + Sleep: chunkRetrySleep, + }) + + uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error { + _, err := handler.bucket.UploadPart(imur, content, current.Length(), current.Index()+1) return err } - return nil + for chunks.Next() { + if err := chunks.Process(uploadFunc); err != nil { + return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err) + } + } + + _, err = handler.bucket.CompleteMultipartUpload(imur, oss.CompleteAll("yes"), oss.ForbidOverWrite(!overwrite)) + return err } // Delete 删除一个或多个文件， @@ -395,6 +421,10 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64, // Token 获取上传策略和认证Token func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) { + if err := handler.InitOSSClient(false); err != nil { + return nil, err + } + // 生成回调地址 siteURL := model.GetSiteURL() apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + uploadSession.Key) @@ -406,61 +436,69 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria CallbackBody: `{"name":${x:fname},"source_name":${object},"size":${size},"pic_info":"${imageInfo.width},${imageInfo.height}"}`, CallbackBodyType: "application/json", } - - // 上传策略 - savePath := file.Info().SavePath - postPolicy := UploadPolicy{ - Expiration: time.Now().UTC().Add(time.Duration(ttl) * time.Second).Format(time.RFC3339), - Conditions: []interface{}{ - map[string]string{"bucket": handler.Policy.BucketName}, - []string{"starts-with", "$key", path.Dir(savePath)}, - }, + callbackPolicyJSON, err := json.Marshal(callbackPolicy) + if err != nil { + return nil, fmt.Errorf("failed to encode callback policy: %w", err) } + callbackPolicyEncoded := base64.StdEncoding.EncodeToString(callbackPolicyJSON) - if handler.Policy.MaxSize > 0 { - postPolicy.Conditions = append(postPolicy.Conditions, - []interface{}{"content-length-range", 0, handler.Policy.MaxSize}) + // 初始化分片上传 + fileInfo := file.Info() + options := []oss.Option{ + oss.Expires(time.Now().Add(time.Duration(ttl) * time.Second)), + oss.ForbidOverWrite(true), } - - return handler.getUploadCredential(ctx, postPolicy, callbackPolicy, ttl, savePath) -} - -func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback CallbackPolicy, TTL int64, savePath string) (*serializer.UploadCredential, error) { - // 处理回调策略 - callbackPolicyEncoded := "" - if callback.CallbackURL != "" { - callbackPolicyJSON, err := json.Marshal(callback) + imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...) + if err != nil { + return nil, fmt.Errorf("failed to initialize multipart upload: %w", err) + } + uploadSession.OSSUploadID = imur.UploadID + + // 为每个分片签名上传 URL + chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}) + urls := make([]string, chunks.Num()) + for chunks.Next() { + err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error { + signedURL, err := handler.bucket.SignURL(fileInfo.SavePath, oss.HTTPPut, ttl, + oss.PartNumber(c.Index()+1), + oss.UploadID(imur.UploadID), + oss.ContentType("application/octet-stream")) + if err != nil { + return err + } + + urls[c.Index()] = signedURL + return nil + }) if err != nil { return nil, err } - callbackPolicyEncoded = base64.StdEncoding.EncodeToString(callbackPolicyJSON) - policy.Conditions = append(policy.Conditions, map[string]string{"callback": callbackPolicyEncoded}) } - // 编码上传策略 - policyJSON, err := json.Marshal(policy) + // 签名完成分片上传的URL + completeURL, err := handler.bucket.SignURL(fileInfo.SavePath, oss.HTTPPost, ttl, + oss.UploadID(imur.UploadID), + oss.Expires(time.Now().Add(time.Duration(ttl)*time.Second)), + oss.CompleteAll("yes"), + oss.ForbidOverWrite(true), + oss.CallbackParam(callbackPolicyEncoded)) if err != nil { return nil, err } - policyEncoded := base64.StdEncoding.EncodeToString(policyJSON) - - // 签名上传策略 - hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey)) - _, err = io.WriteString(hmacSign, policyEncoded) - if err != nil { - return nil, err - } - signature := base64.StdEncoding.EncodeToString(hmacSign.Sum(nil)) return &serializer.UploadCredential{ - Policy: fmt.Sprintf("%s:%s", callbackPolicyEncoded, policyEncoded), - Path: savePath, - AccessKey: handler.Policy.AccessKey, - Token: signature, + SessionID: uploadSession.Key, + ChunkSize: handler.Policy.OptionsSerialized.ChunkSize, + UploadID: imur.UploadID, + UploadURLs: urls, + Callback: completeURL, }, nil } // 取消上传凭证 func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error { - return nil + if err := handler.InitOSSClient(false); err != nil { + return err + } + return handler.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{UploadID: uploadSession.OSSUploadID, Key: uploadSession.SavePath}, nil) } diff --git a/pkg/filesystem/driver/remote/client.go b/pkg/filesystem/driver/remote/client.go index 17bbb929..b6759f1f 100644 --- a/pkg/filesystem/driver/remote/client.go +++ b/pkg/filesystem/driver/remote/client.go @@ -90,7 +90,7 @@ func (c *remoteClient) Upload(ctx context.Context, file fsctx.FileHeader) error // Initial chunk groups chunks := chunk.NewChunkGroup(file, c.policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{ - Max: model.GetIntSetting("slave_chunk_retries", 5), + Max: model.GetIntSetting("chunk_retries", 5), Sleep: chunkRetrySleep, }) diff --git a/pkg/serializer/upload.go b/pkg/serializer/upload.go index 485b8430..713641b3 100644 --- a/pkg/serializer/upload.go +++ b/pkg/serializer/upload.go @@ -23,8 +23,10 @@ type UploadCredential struct { SessionID string `json:"sessionID"` ChunkSize uint64 `json:"chunkSize"` // 分块大小，0 为部分快 Expires int64 `json:"expires"` // 上传凭证过期时间， Unix 时间戳 - UploadURLs []string `json:"uploadURLs"` - Credential string `json:"credential"` + UploadURLs []string `json:"uploadURLs,omitempty"` + Credential string `json:"credential,omitempty"` + UploadID string `json:"uploadID,omitempty"` + Callback string `json:"callback,omitempty"` // 回调地址 Token string `json:"token"` Policy string `json:"policy"` @@ -32,7 +34,6 @@ type UploadCredential struct { AccessKey string `json:"ak"` KeyTime string `json:"key_time,omitempty"` // COS用有效期 Key string `json:"key,omitempty"` // 文件标识符，通常为回调key - Callback string `json:"callback,omitempty"` // 回调地址 } // UploadSession 上传会话 @@ -48,6 +49,7 @@ type UploadSession struct { Callback string // 回调 URL 地址 CallbackSecret string // 回调 URL OneDriveUploadURL string + OSSUploadID string } // UploadCallback 上传回调正文 diff --git a/routers/router.go b/routers/router.go index 9f61a335..05b700a8 100644 --- a/routers/router.go +++ b/routers/router.go @@ -250,7 +250,8 @@ func InitMasterRouter() *gin.Engine { ) // 阿里云OSS策略上传回调 callback.POST( - "oss/:key", + "oss/:sessionID", + middleware.UseUploadSession("oss"), middleware.OSSCallbackAuth(), controllers.OSSCallback, ) diff --git a/service/callback/upload.go b/service/callback/upload.go index 4f0495e3..da0d67ed 100644 --- a/service/callback/upload.go +++ b/service/callback/upload.go @@ -243,3 +243,24 @@ func (service *S3Callback) PreProcess(c *gin.Context) serializer.Response { return ProcessCallback(service, c) } + +// PreProcess 对OneDrive客户端回调进行预处理验证 +func (service *UploadCallbackService) PreProcess(c *gin.Context) serializer.Response { + // 创建文件系统 + fs, err := filesystem.NewFileSystemFromCallback(c) + if err != nil { + return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err) + } + defer fs.Recycle() + + // 获取回调会话 + uploadSession := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession) + + // 验证文件大小 + if uploadSession.Size != service.Size { + fs.Handler.Delete(context.Background(), []string{uploadSession.SavePath}) + return serializer.Err(serializer.CodeUploadFailed, "文件大小不一致", nil) + } + + return ProcessCallback(service, c) +} From 07f13cc350e3bf3304db54a5707be653b6280f90 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Sun, 20 Mar 2022 11:26:26 +0800 Subject: [PATCH 06/21] Refactor: factory method for OSS client Fix: use HTTPS schema by default in OSS client Feat: new handler for Qiniu policy --- go.mod | 8 +- go.sum | 65 ++++++++++--- middleware/auth.go | 40 ++++---- middleware/auth_test.go | 2 +- models/policy.go | 7 +- pkg/filesystem/driver/onedrive/handler.go | 4 +- pkg/filesystem/driver/oss/handler.go | 105 ++++++++------------- pkg/filesystem/driver/qiniu/handler.go | 110 ++++++++++++++-------- pkg/filesystem/filesystem.go | 12 +-- pkg/filesystem/upload.go | 35 ++----- pkg/serializer/upload.go | 25 ++--- routers/router.go | 3 +- service/admin/policy.go | 7 +- 13 files changed, 219 insertions(+), 204 deletions(-) diff --git a/go.mod b/go.mod index a7de23c3..51890bfc 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/DATA-DOG/go-sqlmock v1.3.3 - github.com/HFO4/aliyun-oss-go-sdk v2.2.2+incompatible + github.com/HFO4/aliyun-oss-go-sdk v2.2.3+incompatible github.com/aws/aws-sdk-go v1.31.5 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4 @@ -27,18 +27,18 @@ require ( github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2 github.com/pkg/errors v0.9.1 github.com/pquerna/otp v1.2.0 - github.com/qiniu/api.v7/v7 v7.4.0 + github.com/qiniu/go-sdk/v7 v7.11.1 github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1 github.com/rakyll/statik v0.1.7 github.com/robfig/cron/v3 v3.0.1 github.com/smartystreets/goconvey v1.6.4 // indirect github.com/speps/go-hashids v2.0.0+incompatible - github.com/stretchr/testify v1.5.1 + github.com/stretchr/testify v1.6.1 github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac github.com/upyun/go-sdk v2.1.0+incompatible golang.org/x/image v0.0.0-20211028202545-6944b10bf410 - golang.org/x/text v0.3.6 + golang.org/x/text v0.3.7 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/go-playground/validator.v9 v9.29.1 gopkg.in/ini.v1 v1.51.0 // indirect diff --git a/go.sum b/go.sum index b8568f53..bdbc9209 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7h github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/HFO4/aliyun-oss-go-sdk v2.2.2+incompatible h1:cWidHgoT2ye1YwzP5sz7xkqOK6I84mqrayJSU23oOUo= -github.com/HFO4/aliyun-oss-go-sdk v2.2.2+incompatible/go.mod h1:8KDiKVrHK/UbXAhj+iQGp1m40rQa+UAvzBi7m22KywI= +github.com/HFO4/aliyun-oss-go-sdk v2.2.3+incompatible h1:aX/+gJM2dAMDDy3JqWS0DJn3JfOUchf4k37P5TbBKU8= +github.com/HFO4/aliyun-oss-go-sdk v2.2.3+incompatible/go.mod h1:8KDiKVrHK/UbXAhj+iQGp1m40rQa+UAvzBi7m22KywI= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -30,6 +30,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7 h1:Puu1hUwfps3+1CUzYdAZXijuvLuRMirgiXdf3zsM2Ig= github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -69,10 +70,16 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-mail/mail v2.3.1+incompatible h1:UzNOn0k5lpfVtO31cK3hn6I4VEVGhe3lX8AJBAxXExM= github.com/go-mail/mail v2.3.1+incompatible/go.mod h1:VPWjmmNyRsWXQZHVHT3g0YbIINUkSmuKOiLIDkWbL6M= -github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= -github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= @@ -145,14 +152,18 @@ github.com/kidstuff/mongostore v0.0.0-20181113001930-e650cd85ee4b/go.mod h1:g2nV github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= @@ -183,6 +194,7 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -197,8 +209,10 @@ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1: github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/qiniu/api.v7/v7 v7.4.0 h1:9dZMVQifh31QGFLVaHls6akCaS2rlj3du8MnEFd7XjQ= -github.com/qiniu/api.v7/v7 v7.4.0/go.mod h1:VE5oC5rkE1xul0u1S2N0b2Uxq9/6hZzhyqjgK25XDcM= +github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= +github.com/qiniu/go-sdk/v7 v7.11.1 h1:/LZ9rvFS4p6SnszhGv11FNB1+n4OZvBCwFg7opH5Ovs= +github.com/qiniu/go-sdk/v7 v7.11.1/go.mod h1:btsaOc8CA3hdVloULfFdDgDc+g4f3TDZEFsDY0BLE+w= +github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438 h1:jnz/4VenymvySjE+Ez511s0pqVzkUOmr1fwCVytNNWk= github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438/go.mod h1:wTPjTepVu7uJBYgZ0SdWHQlIas582j6cn2jgk4DDdlg= github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1 h1:leEwA4MD1ew0lNgzz6Q4G76G3AEfeci+TMggN6WuFRs= @@ -208,6 +222,9 @@ github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Ung github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -224,8 +241,9 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible h1:dqpmYaez7VBT7PCRBcBxkzlDOiTk7Td8ATiia1b1GuE= github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac h1:PSBhZblOjdwH7SIVgcue+7OlnLHkM45KuScLZ+PiVbQ= @@ -241,8 +259,10 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/image v0.0.0-20190501045829-6d32002ffd75/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ= @@ -262,8 +282,9 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -271,6 +292,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -281,13 +304,20 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211020174200-9d6173849985 h1:LOlKVhfDyahgmqa97awczplwkjzNaELFg3zRIJ13RYo= +golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -309,8 +339,10 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= @@ -325,6 +357,9 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/middleware/auth.go b/middleware/auth.go index 28c00fc1..09bc434c 100644 --- a/middleware/auth.go +++ b/middleware/auth.go @@ -5,6 +5,7 @@ import ( "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/oss" "github.com/cloudreve/Cloudreve/v3/pkg/mq" "github.com/cloudreve/Cloudreve/v3/pkg/util" + "github.com/qiniu/go-sdk/v7/auth/qbox" "net/http" model "github.com/cloudreve/Cloudreve/v3/models" @@ -181,28 +182,22 @@ func RemoteCallbackAuth() gin.HandlerFunc { // QiniuCallbackAuth 七牛回调签名验证 func QiniuCallbackAuth() gin.HandlerFunc { return func(c *gin.Context) { - //// 验证key并查找用户 - //resp, user := uploadCallbackCheck(c) - //if resp.Code != 0 { - // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg}) - // c.Abort() - // return - //} - // - //// 验证回调是否来自qiniu - //mac := qbox.NewMac(user.Policy.AccessKey, user.Policy.SecretKey) - //ok, err := mac.VerifyCallback(c.Request) - //if err != nil { - // util.Log().Debug("无法验证回调请求，%s", err) - // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "无法验证回调请求"}) - // c.Abort() - // return - //} - //if !ok { - // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名无效"}) - // c.Abort() - // return - //} + session := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession) + + // 验证回调是否来自qiniu + mac := qbox.NewMac(session.Policy.AccessKey, session.Policy.SecretKey) + ok, err := mac.VerifyCallback(c.Request) + if err != nil { + util.Log().Debug("无法验证回调请求，%s", err) + c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "无法验证回调请求"}) + c.Abort() + return + } + if !ok { + c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名无效"}) + c.Abort() + return + } c.Next() } @@ -289,7 +284,6 @@ func OneDriveCallbackAuth() gin.HandlerFunc { } // COSCallbackAuth 腾讯云COS回调签名验证 -// TODO 解耦 测试 func COSCallbackAuth() gin.HandlerFunc { return func(c *gin.Context) { //// 验证key并查找用户 diff --git a/middleware/auth_test.go b/middleware/auth_test.go index 84d229e2..ab206a09 100644 --- a/middleware/auth_test.go +++ b/middleware/auth_test.go @@ -17,7 +17,7 @@ import ( "github.com/cloudreve/Cloudreve/v3/pkg/util" "github.com/gin-gonic/gin" "github.com/jinzhu/gorm" - "github.com/qiniu/api.v7/v7/auth/qbox" + "github.com/qiniu/go-sdk/v7/auth/qbox" "github.com/stretchr/testify/assert" ) diff --git a/models/policy.go b/models/policy.go index 0f7f2999..5d2717ef 100644 --- a/models/policy.go +++ b/models/policy.go @@ -152,7 +152,7 @@ func (policy *Policy) GeneratePath(uid uint, origin string) string { func (policy *Policy) GenerateFileName(uid uint, origin string) string { // 未开启自动重命名时，直接返回原始文件名 if !policy.AutoRename { - return policy.getOriginNameRule(origin) + return origin } fileRule := policy.FileNameRule @@ -171,10 +171,9 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string { "{hour}": time.Now().Format("15"), "{minute}": time.Now().Format("04"), "{second}": time.Now().Format("05"), + "{originname}": origin, } - replaceTable["{originname}"] = policy.getOriginNameRule(origin) - fileRule = util.Replace(replaceTable, fileRule) return fileRule } @@ -232,7 +231,7 @@ func (policy *Policy) IsUploadPlaceholderWithSize() bool { return true } - if policy.Type == "onedrive" || policy.Type == "oss" { + if util.ContainsString([]string{"onedrive", "oss", "qiniu"}, policy.Type) { return policy.OptionsSerialized.PlaceholderWithSize } diff --git a/pkg/filesystem/driver/onedrive/handler.go b/pkg/filesystem/driver/onedrive/handler.go index 4b76bfc5..8e8dacb5 100644 --- a/pkg/filesystem/driver/onedrive/handler.go +++ b/pkg/filesystem/driver/onedrive/handler.go @@ -234,7 +234,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria // 监控回调及上传 go handler.Client.MonitorUpload(uploadURL, uploadSession.Key, fileInfo.SavePath, fileInfo.Size, ttl) - uploadSession.OneDriveUploadURL = uploadURL + uploadSession.UploadURL = uploadURL return &serializer.UploadCredential{ SessionID: uploadSession.Key, ChunkSize: handler.Policy.OptionsSerialized.ChunkSize, @@ -244,5 +244,5 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria // 取消上传凭证 func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error { - return handler.Client.DeleteUploadSession(ctx, uploadSession.OneDriveUploadURL) + return handler.Client.DeleteUploadSession(ctx, uploadSession.UploadURL) } diff --git a/pkg/filesystem/driver/oss/handler.go b/pkg/filesystem/driver/oss/handler.go index 10e7fdd1..b04784be 100644 --- a/pkg/filesystem/driver/oss/handler.go +++ b/pkg/filesystem/driver/oss/handler.go @@ -56,13 +56,17 @@ const ( VersionID key = iota ) -// CORS 创建跨域策略 -func (handler *Driver) CORS() error { - // 初始化客户端 - if err := handler.InitOSSClient(false); err != nil { - return err +func NewDriver(policy *model.Policy) (*Driver, error) { + driver := &Driver{ + Policy: policy, + HTTPClient: request.NewClient(), } + return driver, driver.InitOSSClient(false) +} + +// CORS 创建跨域策略 +func (handler *Driver) CORS() error { return handler.client.SetBucketCORS(handler.Policy.BucketName, []oss.CORSRule{ { AllowedOrigin: []string{"*"}, @@ -86,39 +90,31 @@ func (handler *Driver) InitOSSClient(forceUsePublicEndpoint bool) error { return errors.New("存储策略为空") } - if handler.client == nil { - // 决定是否使用内网 Endpoint - endpoint := handler.Policy.Server - if handler.Policy.OptionsSerialized.ServerSideEndpoint != "" && !forceUsePublicEndpoint { - endpoint = handler.Policy.OptionsSerialized.ServerSideEndpoint - } - - // 初始化客户端 - client, err := oss.New(endpoint, handler.Policy.AccessKey, handler.Policy.SecretKey) - if err != nil { - return err - } - handler.client = client + // 决定是否使用内网 Endpoint + endpoint := handler.Policy.Server + if handler.Policy.OptionsSerialized.ServerSideEndpoint != "" && !forceUsePublicEndpoint { + endpoint = handler.Policy.OptionsSerialized.ServerSideEndpoint + } - // 初始化存储桶 - bucket, err := client.Bucket(handler.Policy.BucketName) - if err != nil { - return err - } - handler.bucket = bucket + // 初始化客户端 + client, err := oss.New(endpoint, handler.Policy.AccessKey, handler.Policy.SecretKey) + if err != nil { + return err + } + handler.client = client + // 初始化存储桶 + bucket, err := client.Bucket(handler.Policy.BucketName) + if err != nil { + return err } + handler.bucket = bucket return nil } // List 列出OSS上的文件 -func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) { - // 初始化客户端 - if err := handler.InitOSSClient(false); err != nil { - return nil, err - } - +func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) { // 列取文件 base = strings.TrimPrefix(base, "/") if base != "" { @@ -185,7 +181,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([] } // Get 获取文件 -func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) { +func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) { // 通过VersionID禁止缓存 ctx = context.WithValue(ctx, VersionID, time.Now().UnixNano()) @@ -228,15 +224,10 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, } // Put 将文件流保存到指定目录 -func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { +func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error { defer file.Close() fileInfo := file.Info() - // 初始化客户端 - if err := handler.InitOSSClient(false); err != nil { - return err - } - // 凭证有效期 credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600) @@ -247,12 +238,12 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { oss.ForbidOverWrite(!overwrite), } - // 上传文件 - //err := handler.bucket.PutObject(fileInfo.SavePath, file, options...) - //if err != nil { - // return err - //} + // 小文件直接上传 + if fileInfo.Size < MultiPartUploadThreshold { + return handler.bucket.PutObject(fileInfo.SavePath, file, options...) + } + // 超过阈值时使用分片上传 imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...) if err != nil { return fmt.Errorf("failed to initiate multipart upload: %w", err) @@ -280,12 +271,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { // Delete 删除一个或多个文件， // 返回未删除的文件 -func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) { - // 初始化客户端 - if err := handler.InitOSSClient(false); err != nil { - return files, err - } - +func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) { // 删除文件 delRes, err := handler.bucket.DeleteObjects(files) @@ -303,7 +289,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err } // Thumb 获取文件缩略图 -func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) { +func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) { // 初始化客户端 if err := handler.InitOSSClient(true); err != nil { return nil, err @@ -337,7 +323,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content } // Source 获取外链URL -func (handler Driver) Source( +func (handler *Driver) Source( ctx context.Context, path string, baseURL url.URL, @@ -382,7 +368,7 @@ func (handler Driver) Source( return handler.signSourceURL(ctx, path, ttl, signOptions) } -func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64, options []oss.Option) (string, error) { +func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64, options []oss.Option) (string, error) { signedURL, err := handler.bucket.SignURL(path, oss.HTTPGet, ttl, options...) if err != nil { return "", err @@ -394,9 +380,6 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64, return "", err } - // 优先使用https - finalURL.Scheme = "https" - // 公有空间替换掉Key及不支持的头 if !handler.Policy.IsPrivate { query := finalURL.Query() @@ -420,10 +403,7 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64, } // Token 获取上传策略和认证Token -func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) { - if err := handler.InitOSSClient(false); err != nil { - return nil, err - } +func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) { // 生成回调地址 siteURL := model.GetSiteURL() @@ -452,7 +432,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria if err != nil { return nil, fmt.Errorf("failed to initialize multipart upload: %w", err) } - uploadSession.OSSUploadID = imur.UploadID + uploadSession.UploadID = imur.UploadID // 为每个分片签名上传 URL chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}) @@ -496,9 +476,6 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria } // 取消上传凭证 -func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error { - if err := handler.InitOSSClient(false); err != nil { - return err - } - return handler.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{UploadID: uploadSession.OSSUploadID, Key: uploadSession.SavePath}, nil) +func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error { + return handler.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{UploadID: uploadSession.UploadID, Key: uploadSession.SavePath}, nil) } diff --git a/pkg/filesystem/driver/qiniu/handler.go b/pkg/filesystem/driver/qiniu/handler.go index f453dd7b..a47b8da4 100644 --- a/pkg/filesystem/driver/qiniu/handler.go +++ b/pkg/filesystem/driver/qiniu/handler.go @@ -2,6 +2,7 @@ package qiniu import ( "context" + "encoding/base64" "errors" "fmt" "net/http" @@ -16,17 +17,31 @@ import ( "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response" "github.com/cloudreve/Cloudreve/v3/pkg/request" "github.com/cloudreve/Cloudreve/v3/pkg/serializer" - "github.com/qiniu/api.v7/v7/auth/qbox" - "github.com/qiniu/api.v7/v7/storage" + "github.com/qiniu/go-sdk/v7/auth/qbox" + "github.com/qiniu/go-sdk/v7/storage" ) // Driver 本地策略适配器 type Driver struct { Policy *model.Policy + mac *qbox.Mac + cfg *storage.Config + bucket *storage.BucketManager +} + +func NewDriver(policy *model.Policy) *Driver { + mac := qbox.NewMac(policy.AccessKey, policy.SecretKey) + cfg := &storage.Config{UseHTTPS: true} + return &Driver{ + Policy: policy, + mac: mac, + cfg: cfg, + bucket: storage.NewBucketManager(mac, cfg), + } } // List 列出给定路径下的文件 -func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) { +func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) { base = strings.TrimPrefix(base, "/") if base != "" { base += "/" @@ -42,14 +57,8 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([] delimiter = "/" } - mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey) - cfg := storage.Config{ - UseHTTPS: true, - } - bucketManager := storage.NewBucketManager(mac, &cfg) - for { - entries, folders, nextMarker, hashNext, err := bucketManager.ListFiles( + entries, folders, nextMarker, hashNext, err := handler.bucket.ListFiles( handler.Policy.BucketName, base, delimiter, marker, 1000) if err != nil { @@ -99,7 +108,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([] } // Get 获取文件 -func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) { +func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) { // 给文件名加上随机参数以强制拉取 path = fmt.Sprintf("%s?v=%d", path, time.Now().UnixNano()) @@ -143,7 +152,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, } // Put 将文件流保存到指定目录 -func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { +func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error { defer file.Close() // 凭证有效期 @@ -151,9 +160,14 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { // 生成上传策略 fileInfo := file.Info() + scope := handler.Policy.BucketName + if fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite { + scope = fmt.Sprintf("%s:%s", handler.Policy.BucketName, fileInfo.SavePath) + } + putPolicy := storage.PutPolicy{ // 指定为覆盖策略 - Scope: fmt.Sprintf("%s:%s", handler.Policy.BucketName, fileInfo.SavePath), + Scope: scope, SaveKey: fileInfo.SavePath, ForceSaveKey: true, FsizeLimit: int64(fileInfo.Size), @@ -164,7 +178,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { } // 生成上传凭证 - token, err := handler.getUploadCredential(ctx, putPolicy, int64(credentialTTL)) + token, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, int64(credentialTTL), false) if err != nil { return err } @@ -178,7 +192,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { } // 开始上传 - err = formUploader.Put(ctx, &ret, token.Token, fileInfo.SavePath, file, int64(fileInfo.Size), &putExtra) + err = formUploader.Put(ctx, &ret, token.Credential, fileInfo.SavePath, file, int64(fileInfo.Size), &putExtra) if err != nil { return err } @@ -188,19 +202,14 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { // Delete 删除一个或多个文件， // 返回未删除的文件 -func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) { +func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) { // TODO 大于一千个文件需要分批发送 deleteOps := make([]string, 0, len(files)) for _, key := range files { deleteOps = append(deleteOps, storage.URIDelete(handler.Policy.BucketName, key)) } - mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey) - cfg := storage.Config{ - UseHTTPS: true, - } - bucketManager := storage.NewBucketManager(mac, &cfg) - rets, err := bucketManager.Batch(deleteOps) + rets, err := handler.bucket.Batch(deleteOps) // 处理删除结果 if err != nil { @@ -217,7 +226,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err } // Thumb 获取文件缩略图 -func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) { +func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) { var ( thumbSize = [2]uint{400, 300} ok = false @@ -238,7 +247,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content } // Source 获取外链URL -func (handler Driver) Source( +func (handler *Driver) Source( ctx context.Context, path string, baseURL url.URL, @@ -261,12 +270,11 @@ func (handler Driver) Source( return handler.signSourceURL(ctx, path, ttl), nil } -func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64) string { +func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64) string { var sourceURL string if handler.Policy.IsPrivate { - mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey) deadline := time.Now().Add(time.Second * time.Duration(ttl)).Unix() - sourceURL = storage.MakePrivateURL(mac, handler.Policy.BaseURL, path, deadline) + sourceURL = storage.MakePrivateURL(handler.mac, handler.Policy.BaseURL, path, deadline) } else { sourceURL = storage.MakePublicURL(handler.Policy.BaseURL, path) } @@ -274,19 +282,20 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64) } // Token 获取上传策略和认证Token -func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) { +func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) { // 生成回调地址 siteURL := model.GetSiteURL() apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + uploadSession.Key) apiURL := siteURL.ResolveReference(apiBaseURI) // 创建上传策略 + fileInfo := file.Info() putPolicy := storage.PutPolicy{ Scope: handler.Policy.BucketName, CallbackURL: apiURL.String(), - CallbackBody: `{"name":"$(fname)","source_name":"$(key)","size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`, + CallbackBody: `{"size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`, CallbackBodyType: "application/json", - SaveKey: file.Info().SavePath, + SaveKey: fileInfo.SavePath, ForceSaveKey: true, FsizeLimit: int64(handler.Policy.MaxSize), } @@ -295,21 +304,46 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType } - return handler.getUploadCredential(ctx, putPolicy, ttl) + credential, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, ttl, true) + if err != nil { + return nil, fmt.Errorf("failed to init parts: %w", err) + } + + credential.SessionID = uploadSession.Key + credential.ChunkSize = handler.Policy.OptionsSerialized.ChunkSize + + uploadSession.UploadURL = credential.UploadURLs[0] + uploadSession.Credential = credential.Credential + + return credential, nil } -// getUploadCredential 签名上传策略 -func (handler Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, TTL int64) (*serializer.UploadCredential, error) { +// getUploadCredential 签名上传策略并创建上传会话 +func (handler *Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, file *fsctx.UploadTaskInfo, TTL int64, resume bool) (*serializer.UploadCredential, error) { + // 上传凭证 policy.Expires = uint64(TTL) - mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey) - upToken := policy.UploadToken(mac) + upToken := policy.UploadToken(handler.mac) + + // 初始化分片上传 + resumeUploader := storage.NewResumeUploaderV2(handler.cfg) + upHost, err := resumeUploader.UpHost(handler.Policy.AccessKey, handler.Policy.BucketName) + if err != nil { + return nil, err + } + + ret := &storage.InitPartsRet{} + if resume { + err = resumeUploader.InitParts(ctx, upToken, upHost, handler.Policy.BucketName, file.SavePath, true, ret) + } return &serializer.UploadCredential{ - Token: upToken, - }, nil + UploadURLs: []string{upHost + "/buckets/" + handler.Policy.BucketName + "/objects/" + base64.URLEncoding.EncodeToString([]byte(file.SavePath)) + "/uploads/" + ret.UploadID}, + Credential: upToken, + }, err } // 取消上传凭证 func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error { - return nil + resumeUploader := storage.NewResumeUploaderV2(handler.cfg) + return resumeUploader.Client.CallWith(ctx, nil, "DELETE", uploadSession.UploadURL, http.Header{"Authorization": {"UpToken " + uploadSession.Credential}}, nil, 0) } diff --git a/pkg/filesystem/filesystem.go b/pkg/filesystem/filesystem.go index 48e03784..19dd01c7 100644 --- a/pkg/filesystem/filesystem.go +++ b/pkg/filesystem/filesystem.go @@ -144,16 +144,12 @@ func (fs *FileSystem) DispatchHandler() error { fs.Handler = handler case "qiniu": - fs.Handler = qiniu.Driver{ - Policy: currentPolicy, - } + fs.Handler = qiniu.NewDriver(currentPolicy) return nil case "oss": - fs.Handler = oss.Driver{ - Policy: currentPolicy, - HTTPClient: request.NewClient(), - } - return nil + handler, err := oss.NewDriver(currentPolicy) + fs.Handler = handler + return err case "upyun": fs.Handler = upyun.Driver{ Policy: currentPolicy, diff --git a/pkg/filesystem/upload.go b/pkg/filesystem/upload.go index 71876ae6..5ff90a18 100644 --- a/pkg/filesystem/upload.go +++ b/pkg/filesystem/upload.go @@ -91,40 +91,17 @@ func (fs *FileSystem) Upload(ctx context.Context, file *fsctx.FileStream) (err e // TODO 完善测试 func (fs *FileSystem) GenerateSavePath(ctx context.Context, file fsctx.FileHeader) string { fileInfo := file.Info() - - if fs.User.Model.ID != 0 { - return path.Join( - fs.Policy.GeneratePath( - fs.User.Model.ID, - fileInfo.VirtualPath, - ), - fs.Policy.GenerateFileName( - fs.User.Model.ID, - fileInfo.FileName, - ), - ) - } - - // 匿名文件系统尝试根据上下文中的上传策略生成路径 - var anonymousPolicy model.Policy - if policy, ok := ctx.Value(fsctx.UploadPolicyCtx).(serializer.UploadPolicy); ok { - anonymousPolicy = model.Policy{ - Type: "remote", - AutoRename: policy.AutoRename, - DirNameRule: policy.SavePath, - FileNameRule: policy.FileName, - } - } return path.Join( - anonymousPolicy.GeneratePath( - 0, - "", + fs.Policy.GeneratePath( + fs.User.Model.ID, + fileInfo.VirtualPath, ), - anonymousPolicy.GenerateFileName( - 0, + fs.Policy.GenerateFileName( + fs.User.Model.ID, fileInfo.FileName, ), ) + } // CancelUpload 监测客户端取消上传 diff --git a/pkg/serializer/upload.go b/pkg/serializer/upload.go index 713641b3..b82ccb63 100644 --- a/pkg/serializer/upload.go +++ b/pkg/serializer/upload.go @@ -38,18 +38,19 @@ type UploadCredential struct { // UploadSession 上传会话 type UploadSession struct { - Key string // 上传会话 GUID - UID uint // 发起者 - VirtualPath string // 用户文件路径，不含文件名 - Name string // 文件名 - Size uint64 // 文件大小 - SavePath string // 物理存储路径，包含物理文件名 - LastModified *time.Time // 可选的文件最后修改日期 - Policy model.Policy - Callback string // 回调 URL 地址 - CallbackSecret string // 回调 URL - OneDriveUploadURL string - OSSUploadID string + Key string // 上传会话 GUID + UID uint // 发起者 + VirtualPath string // 用户文件路径，不含文件名 + Name string // 文件名 + Size uint64 // 文件大小 + SavePath string // 物理存储路径，包含物理文件名 + LastModified *time.Time // 可选的文件最后修改日期 + Policy model.Policy + Callback string // 回调 URL 地址 + CallbackSecret string // 回调 URL + UploadURL string + UploadID string + Credential string } // UploadCallback 上传回调正文 diff --git a/routers/router.go b/routers/router.go index 05b700a8..255c9b35 100644 --- a/routers/router.go +++ b/routers/router.go @@ -244,7 +244,8 @@ func InitMasterRouter() *gin.Engine { ) // 七牛策略上传回调 callback.POST( - "qiniu/:key", + "qiniu/:sessionID", + middleware.UseUploadSession("qiniu"), middleware.QiniuCallbackAuth(), controllers.QiniuCallback, ) diff --git a/service/admin/policy.go b/service/admin/policy.go index 6d44b6b3..c5b771ec 100644 --- a/service/admin/policy.go +++ b/service/admin/policy.go @@ -149,9 +149,9 @@ func (service *PolicyService) AddCORS() serializer.Response { switch policy.Type { case "oss": - handler := oss.Driver{ - Policy: &policy, - HTTPClient: request.NewClient(), + handler, err := oss.NewDriver(&policy) + if err != nil { + return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err) } if err := handler.CORS(); err != nil { return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err) @@ -169,6 +169,7 @@ func (service *PolicyService) AddCORS() serializer.Response { }, }), } + if err := handler.CORS(); err != nil { return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err) } From 9e5713b139a2e98d1bfd7d8b3a84dcb16d04ace8 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Sun, 20 Mar 2022 11:27:17 +0800 Subject: [PATCH 07/21] Feat: adapt new uploader for COS policy --- middleware/auth.go | 15 ----------- models/policy.go | 37 +--------------------------- pkg/filesystem/driver/cos/handler.go | 13 +++++----- pkg/serializer/upload.go | 11 ++++----- routers/router.go | 4 +-- service/callback/upload.go | 7 +++--- 6 files changed, 18 insertions(+), 69 deletions(-) diff --git a/middleware/auth.go b/middleware/auth.go index 09bc434c..1540d0a0 100644 --- a/middleware/auth.go +++ b/middleware/auth.go @@ -283,21 +283,6 @@ func OneDriveCallbackAuth() gin.HandlerFunc { } } -// COSCallbackAuth 腾讯云COS回调签名验证 -func COSCallbackAuth() gin.HandlerFunc { - return func(c *gin.Context) { - //// 验证key并查找用户 - //resp, _ := uploadCallbackCheck(c) - //if resp.Code != 0 { - // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg}) - // c.Abort() - // return - //} - - c.Next() - } -} - // S3CallbackAuth Amazon S3回调签名验证 func S3CallbackAuth() gin.HandlerFunc { return func(c *gin.Context) { diff --git a/models/policy.go b/models/policy.go index 5d2717ef..c182ebaf 100644 --- a/models/policy.go +++ b/models/policy.go @@ -3,8 +3,6 @@ package model import ( "encoding/gob" "encoding/json" - "fmt" - "net/url" "path" "path/filepath" "strconv" @@ -231,7 +229,7 @@ func (policy *Policy) IsUploadPlaceholderWithSize() bool { return true } - if util.ContainsString([]string{"onedrive", "oss", "qiniu"}, policy.Type) { + if util.ContainsString([]string{"onedrive", "oss", "qiniu", "cos"}, policy.Type) { return policy.OptionsSerialized.PlaceholderWithSize } @@ -243,39 +241,6 @@ func (policy *Policy) CanStructureBeListed() bool { return policy.Type != "local" && policy.Type != "remote" } -// GetUploadURL 获取文件上传服务API地址 -func (policy *Policy) GetUploadURL() string { - server, err := url.Parse(policy.Server) - if err != nil { - return policy.Server - } - - controller, _ := url.Parse("") - switch policy.Type { - case "local", "onedrive": - return "/api/v3/file/upload" - case "remote": - controller, _ = url.Parse("/api/v3/slave/upload") - case "oss": - return "https://" + policy.BucketName + "." + policy.Server - case "cos": - return policy.Server - case "upyun": - return "https://v0.api.upyun.com/" + policy.BucketName - case "s3": - if policy.Server == "" { - return fmt.Sprintf("https://%s.s3.%s.amazonaws.com/", policy.BucketName, - policy.OptionsSerialized.Region) - } - - if !strings.Contains(policy.Server, policy.BucketName) { - controller, _ = url.Parse("/" + policy.BucketName) - } - } - - return server.ResolveReference(controller).String() -} - // SaveAndClearCache 更新并清理缓存 func (policy *Policy) SaveAndClearCache() error { err := DB.Save(policy).Error diff --git a/pkg/filesystem/driver/cos/handler.go b/pkg/filesystem/driver/cos/handler.go index cc55ac4b..53896416 100644 --- a/pkg/filesystem/driver/cos/handler.go +++ b/pkg/filesystem/driver/cos/handler.go @@ -357,8 +357,9 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria res, err := handler.getUploadCredential(ctx, postPolicy, keyTime, savePath) if err == nil { + res.SessionID = uploadSession.Key res.Callback = apiURL - res.Key = uploadSession.Key + res.UploadURLs = []string{handler.Policy.Server} } return res, err @@ -415,10 +416,10 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli signature := hmacFinalSign.Sum(nil) return &serializer.UploadCredential{ - Policy: policyEncoded, - Path: savePath, - AccessKey: handler.Policy.AccessKey, - Token: fmt.Sprintf("%x", signature), - KeyTime: keyTime, + Policy: policyEncoded, + Path: savePath, + AccessKey: handler.Policy.AccessKey, + Credential: fmt.Sprintf("%x", signature), + KeyTime: keyTime, }, nil } diff --git a/pkg/serializer/upload.go b/pkg/serializer/upload.go index b82ccb63..da61c7de 100644 --- a/pkg/serializer/upload.go +++ b/pkg/serializer/upload.go @@ -27,13 +27,12 @@ type UploadCredential struct { Credential string `json:"credential,omitempty"` UploadID string `json:"uploadID,omitempty"` Callback string `json:"callback,omitempty"` // 回调地址 + Path string `json:"path,omitempty"` // 存储路径 + AccessKey string `json:"ak,omitempty"` + KeyTime string `json:"keyTime,omitempty"` // COS用有效期 + Policy string `json:"policy,omitempty"` - Token string `json:"token"` - Policy string `json:"policy"` - Path string `json:"path"` // 存储路径 - AccessKey string `json:"ak"` - KeyTime string `json:"key_time,omitempty"` // COS用有效期 - Key string `json:"key,omitempty"` // 文件标识符，通常为回调key + Token string `json:"token,omitempty"` } // UploadSession 上传会话 diff --git a/routers/router.go b/routers/router.go index 255c9b35..fae395c2 100644 --- a/routers/router.go +++ b/routers/router.go @@ -279,8 +279,8 @@ func InitMasterRouter() *gin.Engine { } // 腾讯云COS策略上传回调 callback.GET( - "cos/:key", - middleware.COSCallbackAuth(), + "cos/:sessionID", + middleware.UseUploadSession("cos"), controllers.COSCallback, ) // AWS S3策略上传回调 diff --git a/service/callback/upload.go b/service/callback/upload.go index da0d67ed..411b571b 100644 --- a/service/callback/upload.go +++ b/service/callback/upload.go @@ -200,17 +200,16 @@ func (service *COSCallback) PreProcess(c *gin.Context) serializer.Response { defer fs.Recycle() // 获取回调会话 - callbackSessionRaw, _ := c.Get("callbackSession") - callbackSession := callbackSessionRaw.(*serializer.UploadSession) + uploadSession := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession) // 获取文件信息 - info, err := fs.Handler.(cos.Driver).Meta(context.Background(), callbackSession.SavePath) + info, err := fs.Handler.(cos.Driver).Meta(context.Background(), uploadSession.SavePath) if err != nil { return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err) } // 验证实际文件信息与回调会话中是否一致 - if callbackSession.Size != info.Size || callbackSession.Key != info.CallbackKey { + if uploadSession.Size != info.Size || uploadSession.Key != info.CallbackKey { return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err) } From d3016b60af6c48d8955c030e0a223a2861ca84b8 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Sun, 20 Mar 2022 11:27:43 +0800 Subject: [PATCH 08/21] Feat: adapt new uploader for upyun policy --- middleware/auth.go | 94 +++++++++++++------------- pkg/filesystem/driver/upyun/handler.go | 25 +++---- routers/router.go | 3 +- 3 files changed, 59 insertions(+), 63 deletions(-) diff --git a/middleware/auth.go b/middleware/auth.go index 1540d0a0..ca674028 100644 --- a/middleware/auth.go +++ b/middleware/auth.go @@ -1,11 +1,17 @@ package middleware import ( + "bytes" + "context" + "crypto/md5" + "fmt" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/oss" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/upyun" "github.com/cloudreve/Cloudreve/v3/pkg/mq" "github.com/cloudreve/Cloudreve/v3/pkg/util" "github.com/qiniu/go-sdk/v7/auth/qbox" + "io/ioutil" "net/http" model "github.com/cloudreve/Cloudreve/v3/models" @@ -221,53 +227,47 @@ func OSSCallbackAuth() gin.HandlerFunc { // UpyunCallbackAuth 又拍云回调签名验证 func UpyunCallbackAuth() gin.HandlerFunc { return func(c *gin.Context) { - //// 验证key并查找用户 - //resp, user := uploadCallbackCheck(c) - //if resp.Code != 0 { - // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg}) - // c.Abort() - // return - //} - // - //// 获取请求正文 - //body, err := ioutil.ReadAll(c.Request.Body) - //c.Request.Body.Close() - //if err != nil { - // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: err.Error()}) - // c.Abort() - // return - //} - // - //c.Request.Body = ioutil.NopCloser(bytes.NewReader(body)) - // - //// 准备验证Upyun回调签名 - //handler := upyun.Driver{Policy: &user.Policy} - //contentMD5 := c.Request.Header.Get("Content-Md5") - //date := c.Request.Header.Get("Date") - //actualSignature := c.Request.Header.Get("Authorization") - // - //// 计算正文MD5 - //actualContentMD5 := fmt.Sprintf("%x", md5.Sum(body)) - //if actualContentMD5 != contentMD5 { - // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "MD5不一致"}) - // c.Abort() - // return - //} - // - //// 计算理论签名 - //signature := handler.Sign(context.Background(), []string{ - // "POST", - // c.Request.URL.Path, - // date, - // contentMD5, - //}) - // - //// 对比签名 - //if signature != actualSignature { - // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "鉴权失败"}) - // c.Abort() - // return - //} + session := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession) + + // 获取请求正文 + body, err := ioutil.ReadAll(c.Request.Body) + c.Request.Body.Close() + if err != nil { + c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: err.Error()}) + c.Abort() + return + } + + c.Request.Body = ioutil.NopCloser(bytes.NewReader(body)) + + // 准备验证Upyun回调签名 + handler := upyun.Driver{Policy: &session.Policy} + contentMD5 := c.Request.Header.Get("Content-Md5") + date := c.Request.Header.Get("Date") + actualSignature := c.Request.Header.Get("Authorization") + + // 计算正文MD5 + actualContentMD5 := fmt.Sprintf("%x", md5.Sum(body)) + if actualContentMD5 != contentMD5 { + c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "MD5不一致"}) + c.Abort() + return + } + + // 计算理论签名 + signature := handler.Sign(context.Background(), []string{ + "POST", + c.Request.URL.Path, + date, + contentMD5, + }) + + // 对比签名 + if signature != actualSignature { + c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "鉴权失败"}) + c.Abort() + return + } c.Next() } diff --git a/pkg/filesystem/driver/upyun/handler.go b/pkg/filesystem/driver/upyun/handler.go index ab033969..0357f720 100644 --- a/pkg/filesystem/driver/upyun/handler.go +++ b/pkg/filesystem/driver/upyun/handler.go @@ -311,8 +311,6 @@ func (handler Driver) signURL(ctx context.Context, path *url.URL, TTL int64) (st // Token 获取上传策略和认证Token func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) { - // 检查文件大小 - // 生成回调地址 siteURL := model.GetSiteURL() apiBaseURI, _ := url.Parse("/api/v3/callback/upyun/" + uploadSession.Key) @@ -332,17 +330,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria } // 生成上传凭证 - return handler.getUploadCredential(ctx, putPolicy) -} - -// 取消上传凭证 -func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error { - return nil -} - -func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy) (*serializer.UploadCredential, error) { - // 生成上传策略 - policyJSON, err := json.Marshal(policy) + policyJSON, err := json.Marshal(putPolicy) if err != nil { return nil, err } @@ -353,11 +341,18 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli signStr := handler.Sign(ctx, elements) return &serializer.UploadCredential{ - Policy: policyEncoded, - Token: signStr, + SessionID: uploadSession.Key, + Policy: policyEncoded, + Credential: signStr, + UploadURLs: []string{"https://v0.api.upyun.com/" + handler.Policy.BucketName}, }, nil } +// 取消上传凭证 +func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error { + return nil +} + // Sign 计算又拍云的签名头 func (handler Driver) Sign(ctx context.Context, elements []string) string { password := fmt.Sprintf("%x", md5.Sum([]byte(handler.Policy.SecretKey))) diff --git a/routers/router.go b/routers/router.go index fae395c2..bfaa37e5 100644 --- a/routers/router.go +++ b/routers/router.go @@ -258,7 +258,8 @@ func InitMasterRouter() *gin.Engine { ) // 又拍云策略上传回调 callback.POST( - "upyun/:key", + "upyun/:sessionID", + middleware.UseUploadSession("upyun"), middleware.UpyunCallbackAuth(), controllers.UpyunCallback, ) From 7eb81731018fa5dd1d20261ad538dc960c4fb36d Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Sun, 20 Mar 2022 11:29:50 +0800 Subject: [PATCH 09/21] Feat: adapt new uploader for s3 like policy This commit also fix #730, #713, #756, #5 --- models/policy.go | 2 +- pkg/filesystem/driver/oss/handler.go | 10 +- pkg/filesystem/driver/s3/handler.go | 193 +++++++++++++-------------- pkg/filesystem/filesystem.go | 7 +- pkg/filesystem/fsctx/stream.go | 7 +- pkg/serializer/upload.go | 25 ++-- routers/controllers/callback.go | 1 - routers/router.go | 4 +- service/admin/policy.go | 6 +- service/callback/upload.go | 10 +- 10 files changed, 128 insertions(+), 137 deletions(-) diff --git a/models/policy.go b/models/policy.go index c182ebaf..0dc06010 100644 --- a/models/policy.go +++ b/models/policy.go @@ -229,7 +229,7 @@ func (policy *Policy) IsUploadPlaceholderWithSize() bool { return true } - if util.ContainsString([]string{"onedrive", "oss", "qiniu", "cos"}, policy.Type) { + if util.ContainsString([]string{"onedrive", "oss", "qiniu", "cos", "s3"}, policy.Type) { return policy.OptionsSerialized.PlaceholderWithSize } diff --git a/pkg/filesystem/driver/oss/handler.go b/pkg/filesystem/driver/oss/handler.go index b04784be..2e156741 100644 --- a/pkg/filesystem/driver/oss/handler.go +++ b/pkg/filesystem/driver/oss/handler.go @@ -467,11 +467,11 @@ func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *seri } return &serializer.UploadCredential{ - SessionID: uploadSession.Key, - ChunkSize: handler.Policy.OptionsSerialized.ChunkSize, - UploadID: imur.UploadID, - UploadURLs: urls, - Callback: completeURL, + SessionID: uploadSession.Key, + ChunkSize: handler.Policy.OptionsSerialized.ChunkSize, + UploadID: imur.UploadID, + UploadURLs: urls, + CompleteURL: completeURL, }, nil } diff --git a/pkg/filesystem/driver/s3/handler.go b/pkg/filesystem/driver/s3/handler.go index 120bcf1e..a84178f4 100644 --- a/pkg/filesystem/driver/s3/handler.go +++ b/pkg/filesystem/driver/s3/handler.go @@ -2,13 +2,12 @@ package s3 import ( "context" - "crypto/hmac" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "encoding/json" "errors" + "fmt" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff" "github.com/cloudreve/Cloudreve/v3/pkg/util" + "io" "net/http" "net/url" "path" @@ -47,6 +46,14 @@ type MetaData struct { Etag string } +func NewDriver(policy *model.Policy) (*Driver, error) { + driver := &Driver{ + Policy: policy, + } + + return driver, driver.InitS3Client() +} + // InitS3Client 初始化S3会话 func (handler *Driver) InitS3Client() error { if handler.Policy == nil { @@ -72,13 +79,7 @@ func (handler *Driver) InitS3Client() error { } // List 列出给定路径下的文件 -func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) { - - // 初始化客户端 - if err := handler.InitS3Client(); err != nil { - return nil, err - } - +func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) { // 初始化列目录参数 base = strings.TrimPrefix(base, "/") if base != "" { @@ -155,8 +156,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([] } // Get 获取文件 -func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) { - +func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) { // 获取文件源地址 downloadURL, err := handler.Source( ctx, @@ -197,7 +197,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, } // Put 将文件流保存到指定目录 -func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { +func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error { defer file.Close() // 初始化客户端 @@ -205,13 +205,15 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { return err } - uploader := s3manager.NewUploader(handler.sess) + uploader := s3manager.NewUploader(handler.sess, func(u *s3manager.Uploader) { + u.PartSize = int64(handler.Policy.OptionsSerialized.ChunkSize) + }) dst := file.Info().SavePath _, err := uploader.Upload(&s3manager.UploadInput{ Bucket: &handler.Policy.BucketName, Key: &dst, - Body: file, + Body: io.LimitReader(file, int64(file.Info().Size)), }) if err != nil { @@ -223,13 +225,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { // Delete 删除一个或多个文件， // 返回未删除的文件，及遇到的最后一个错误 -func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) { - - // 初始化客户端 - if err := handler.InitS3Client(); err != nil { - return files, err - } - +func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) { failed := make([]string, 0, len(files)) deleted := make([]string, 0, len(files)) @@ -263,12 +259,12 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err } // Thumb 获取文件缩略图 -func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) { +func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) { return nil, errors.New("未实现") } // Source 获取外链URL -func (handler Driver) Source( +func (handler *Driver) Source( ctx context.Context, path string, baseURL url.URL, @@ -325,42 +321,75 @@ func (handler Driver) Source( } // Token 获取上传策略和认证Token -func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) { - // 生成回调地址 - siteURL := model.GetSiteURL() - apiBaseURI, _ := url.Parse("/api/v3/callback/s3/" + uploadSession.Key) - apiURL := siteURL.ResolveReference(apiBaseURI) - - // 上传策略 - savePath := file.Info().SavePath - putPolicy := UploadPolicy{ - Expiration: time.Now().UTC().Add(time.Duration(ttl) * time.Second).Format(time.RFC3339), - Conditions: []interface{}{ - map[string]string{"bucket": handler.Policy.BucketName}, - []string{"starts-with", "$key", savePath}, - []string{"starts-with", "$success_action_redirect", apiURL.String()}, - []string{"starts-with", "$name", ""}, - []string{"starts-with", "$Content-Type", ""}, - map[string]string{"x-amz-algorithm": "AWS4-HMAC-SHA256"}, - }, +func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) { + // 检查文件是否存在 + fileInfo := file.Info() + if _, err := handler.Meta(ctx, fileInfo.SavePath); err == nil { + return nil, fmt.Errorf("file already exist") } - if handler.Policy.MaxSize > 0 { - putPolicy.Conditions = append(putPolicy.Conditions, - []interface{}{"content-length-range", 0, handler.Policy.MaxSize}) + // 创建分片上传 + expires := time.Now().Add(time.Duration(ttl) * time.Second) + res, err := handler.svc.CreateMultipartUpload(&s3.CreateMultipartUploadInput{ + Bucket: &handler.Policy.BucketName, + Key: &fileInfo.SavePath, + Expires: &expires, + }) + if err != nil { + return nil, fmt.Errorf("failed to create multipart upload: %w", err) } - // 生成上传凭证 - return handler.getUploadCredential(ctx, putPolicy, apiURL, savePath) -} + uploadSession.UploadID = *res.UploadId + + // 为每个分片签名上传 URL + chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}) + urls := make([]string, chunks.Num()) + for chunks.Next() { + err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error { + signedReq, _ := handler.svc.UploadPartRequest(&s3.UploadPartInput{ + Bucket: &handler.Policy.BucketName, + Key: &fileInfo.SavePath, + PartNumber: aws.Int64(int64(c.Index() + 1)), + UploadId: res.UploadId, + }) + + signedURL, err := signedReq.Presign(time.Duration(ttl) * time.Second) + if err != nil { + return err + } + + urls[c.Index()] = signedURL + return nil + }) + if err != nil { + return nil, err + } + } -// Meta 获取文件信息 -func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) { - // 初始化客户端 - if err := handler.InitS3Client(); err != nil { + // 签名完成分片上传的请求URL + signedReq, _ := handler.svc.CompleteMultipartUploadRequest(&s3.CompleteMultipartUploadInput{ + Bucket: &handler.Policy.BucketName, + Key: &fileInfo.SavePath, + UploadId: res.UploadId, + }) + + signedURL, err := signedReq.Presign(time.Duration(ttl) * time.Second) + if err != nil { return nil, err } + // 生成上传凭证 + return &serializer.UploadCredential{ + SessionID: uploadSession.Key, + ChunkSize: handler.Policy.OptionsSerialized.ChunkSize, + UploadID: *res.UploadId, + UploadURLs: urls, + CompleteURL: signedURL, + }, nil +} + +// Meta 获取文件信息 +func (handler *Driver) Meta(ctx context.Context, path string) (*MetaData, error) { res, err := handler.svc.GetObject( &s3.GetObjectInput{ Bucket: &handler.Policy.BucketName, @@ -378,52 +407,8 @@ func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) } -func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback *url.URL, savePath string) (*serializer.UploadCredential, error) { - - longDate := time.Now().UTC().Format("20060102T150405Z") - shortDate := time.Now().UTC().Format("20060102") - - credential := handler.Policy.AccessKey + "/" + shortDate + "/" + handler.Policy.OptionsSerialized.Region + "/s3/aws4_request" - policy.Conditions = append(policy.Conditions, map[string]string{"x-amz-credential": credential}) - policy.Conditions = append(policy.Conditions, map[string]string{"x-amz-date": longDate}) - - // 编码上传策略 - policyJSON, err := json.Marshal(policy) - if err != nil { - return nil, err - } - policyEncoded := base64.StdEncoding.EncodeToString(policyJSON) - - //签名 - signature := getHMAC([]byte("AWS4"+handler.Policy.SecretKey), []byte(shortDate)) - signature = getHMAC(signature, []byte(handler.Policy.OptionsSerialized.Region)) - signature = getHMAC(signature, []byte("s3")) - signature = getHMAC(signature, []byte("aws4_request")) - signature = getHMAC(signature, []byte(policyEncoded)) - - return &serializer.UploadCredential{ - Policy: policyEncoded, - Callback: callback.String(), - Token: hex.EncodeToString(signature), - AccessKey: credential, - Path: savePath, - KeyTime: longDate, - }, nil -} - -func getHMAC(key []byte, data []byte) []byte { - hash := hmac.New(sha256.New, key) - hash.Write(data) - return hash.Sum(nil) -} - // CORS 创建跨域策略 -func (handler Driver) CORS() error { - // 初始化客户端 - if err := handler.InitS3Client(); err != nil { - return err - } - +func (handler *Driver) CORS() error { rule := s3.CORSRule{ AllowedMethods: aws.StringSlice([]string{ "GET", @@ -434,6 +419,7 @@ func (handler Driver) CORS() error { }), AllowedOrigins: aws.StringSlice([]string{"*"}), AllowedHeaders: aws.StringSlice([]string{"*"}), + ExposeHeaders: aws.StringSlice([]string{"ETag"}), MaxAgeSeconds: aws.Int64(3600), } @@ -448,6 +434,11 @@ func (handler Driver) CORS() error { } // 取消上传凭证 -func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error { - return nil +func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error { + _, err := handler.svc.AbortMultipartUpload(&s3.AbortMultipartUploadInput{ + UploadId: &uploadSession.UploadID, + Bucket: &handler.Policy.BucketName, + Key: &uploadSession.SavePath, + }) + return err } diff --git a/pkg/filesystem/filesystem.go b/pkg/filesystem/filesystem.go index 19dd01c7..51a13a85 100644 --- a/pkg/filesystem/filesystem.go +++ b/pkg/filesystem/filesystem.go @@ -174,10 +174,9 @@ func (fs *FileSystem) DispatchHandler() error { } return nil case "s3": - fs.Handler = s3.Driver{ - Policy: currentPolicy, - } - return nil + handler, err := s3.NewDriver(currentPolicy) + fs.Handler = handler + return err default: return ErrUnknownPolicyType } diff --git a/pkg/filesystem/fsctx/stream.go b/pkg/filesystem/fsctx/stream.go index c51d28c2..4cf48e3d 100644 --- a/pkg/filesystem/fsctx/stream.go +++ b/pkg/filesystem/fsctx/stream.go @@ -1,6 +1,7 @@ package fsctx import ( + "errors" "io" "time" ) @@ -75,7 +76,11 @@ func (file *FileStream) Close() error { } func (file *FileStream) Seek(offset int64, whence int) (int64, error) { - return file.Seeker.Seek(offset, whence) + if file.Seekable() { + return file.Seeker.Seek(offset, whence) + } + + return 0, errors.New("no seeker") } func (file *FileStream) Seekable() bool { diff --git a/pkg/serializer/upload.go b/pkg/serializer/upload.go index da61c7de..b9e90299 100644 --- a/pkg/serializer/upload.go +++ b/pkg/serializer/upload.go @@ -20,19 +20,18 @@ type UploadPolicy struct { // UploadCredential 返回给客户端的上传凭证 type UploadCredential struct { - SessionID string `json:"sessionID"` - ChunkSize uint64 `json:"chunkSize"` // 分块大小，0 为部分快 - Expires int64 `json:"expires"` // 上传凭证过期时间， Unix 时间戳 - UploadURLs []string `json:"uploadURLs,omitempty"` - Credential string `json:"credential,omitempty"` - UploadID string `json:"uploadID,omitempty"` - Callback string `json:"callback,omitempty"` // 回调地址 - Path string `json:"path,omitempty"` // 存储路径 - AccessKey string `json:"ak,omitempty"` - KeyTime string `json:"keyTime,omitempty"` // COS用有效期 - Policy string `json:"policy,omitempty"` - - Token string `json:"token,omitempty"` + SessionID string `json:"sessionID"` + ChunkSize uint64 `json:"chunkSize"` // 分块大小，0 为部分快 + Expires int64 `json:"expires"` // 上传凭证过期时间， Unix 时间戳 + UploadURLs []string `json:"uploadURLs,omitempty"` + Credential string `json:"credential,omitempty"` + UploadID string `json:"uploadID,omitempty"` + Callback string `json:"callback,omitempty"` // 回调地址 + Path string `json:"path,omitempty"` // 存储路径 + AccessKey string `json:"ak,omitempty"` + KeyTime string `json:"keyTime,omitempty"` // COS用有效期 + Policy string `json:"policy,omitempty"` + CompleteURL string `json:"completeURL,omitempty"` } // UploadSession 上传会话 diff --git a/routers/controllers/callback.go b/routers/controllers/callback.go index 9179ab58..dade5662 100644 --- a/routers/controllers/callback.go +++ b/routers/controllers/callback.go @@ -112,7 +112,6 @@ func COSCallback(c *gin.Context) { // S3Callback S3上传完成客户端回调 func S3Callback(c *gin.Context) { - c.Header("Access-Control-Allow-Origin", "*") var callbackBody callback.S3Callback if err := c.ShouldBindQuery(&callbackBody); err == nil { res := callbackBody.PreProcess(c) diff --git a/routers/router.go b/routers/router.go index bfaa37e5..962179f9 100644 --- a/routers/router.go +++ b/routers/router.go @@ -286,8 +286,8 @@ func InitMasterRouter() *gin.Engine { ) // AWS S3策略上传回调 callback.GET( - "s3/:key", - middleware.S3CallbackAuth(), + "s3/:sessionID", + middleware.UseUploadSession("s3"), controllers.S3Callback, ) } diff --git a/service/admin/policy.go b/service/admin/policy.go index c5b771ec..a9151d53 100644 --- a/service/admin/policy.go +++ b/service/admin/policy.go @@ -174,9 +174,11 @@ func (service *PolicyService) AddCORS() serializer.Response { return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err) } case "s3": - handler := s3.Driver{ - Policy: &policy, + handler, err := s3.NewDriver(&policy) + if err != nil { + return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err) } + if err := handler.CORS(); err != nil { return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err) } diff --git a/service/callback/upload.go b/service/callback/upload.go index 411b571b..fb94675a 100644 --- a/service/callback/upload.go +++ b/service/callback/upload.go @@ -61,9 +61,6 @@ type COSCallback struct { // S3Callback S3 客户端回调正文 type S3Callback struct { - Bucket string `form:"bucket"` - Etag string `form:"etag"` - Key string `form:"key"` } // GetBody 返回回调正文 @@ -226,17 +223,16 @@ func (service *S3Callback) PreProcess(c *gin.Context) serializer.Response { defer fs.Recycle() // 获取回调会话 - callbackSessionRaw, _ := c.Get("callbackSession") - callbackSession := callbackSessionRaw.(*serializer.UploadSession) + uploadSession := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession) // 获取文件信息 - info, err := fs.Handler.(s3.Driver).Meta(context.Background(), callbackSession.SavePath) + info, err := fs.Handler.(*s3.Driver).Meta(context.Background(), uploadSession.SavePath) if err != nil { return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err) } // 验证实际文件信息与回调会话中是否一致 - if callbackSession.Size != info.Size || service.Etag != info.Etag { + if uploadSession.Size != info.Size { return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err) } From d54ca151b23885b6e79d99d4f4954741dd99fa74 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Wed, 23 Mar 2022 18:58:18 +0800 Subject: [PATCH 10/21] Feat: overwrite database settings in conf.ini for slave node. --- pkg/cache/driver.go | 10 ++++++++-- pkg/conf/conf.go | 6 +++++- pkg/conf/defaults.go | 19 ++----------------- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/pkg/cache/driver.go b/pkg/cache/driver.go index 35f69228..8df967dc 100644 --- a/pkg/cache/driver.go +++ b/pkg/cache/driver.go @@ -2,6 +2,7 @@ package cache import ( "github.com/cloudreve/Cloudreve/v3/pkg/conf" + "github.com/cloudreve/Cloudreve/v3/pkg/util" "github.com/gin-gonic/gin" ) @@ -10,8 +11,6 @@ var Store Driver = NewMemoStore() // Init 初始化缓存 func Init() { - //Store = NewRedisStore(10, "tcp", "127.0.0.1:6379", "", "0") - //return if conf.RedisConfig.Server != "" && gin.Mode() != gin.TestMode { Store = NewRedisStore( 10, @@ -21,6 +20,13 @@ func Init() { conf.RedisConfig.DB, ) } + + if conf.SystemConfig.Mode == "slave" { + err := Store.Sets(conf.OptionOverwrite, "setting_") + if err != nil { + util.Log().Warning("无法覆盖数据库设置: %s", err) + } + } } // Driver 键值缓存存储容器 diff --git a/pkg/conf/conf.go b/pkg/conf/conf.go index 7258646b..53fd4c10 100644 --- a/pkg/conf/conf.go +++ b/pkg/conf/conf.go @@ -131,7 +131,6 @@ func Init(path string) { "System": SystemConfig, "SSL": SSLConfig, "UnixSocket": UnixConfig, - "Captcha": CaptchaConfig, "Redis": RedisConfig, "Thumbnail": ThumbConfig, "CORS": CORSConfig, @@ -144,6 +143,11 @@ func Init(path string) { } } + // 映射数据库配置覆盖 + for _, key := range cfg.Section("OptionOverwrite").Keys() { + OptionOverwrite[key.Name()] = key.Value() + } + // 重设log等级 if !SystemConfig.Debug { util.Level = util.LevelInformational diff --git a/pkg/conf/defaults.go b/pkg/conf/defaults.go index b39329c9..a9989763 100644 --- a/pkg/conf/defaults.go +++ b/pkg/conf/defaults.go @@ -1,7 +1,5 @@ package conf -import "github.com/mojocn/base64Captcha" - // RedisConfig Redis服务器配置 var RedisConfig = &redis{ Network: "tcp", @@ -25,21 +23,6 @@ var SystemConfig = &system{ Listen: ":5212", } -// CaptchaConfig 验证码配置 -var CaptchaConfig = &captcha{ - Height: 60, - Width: 240, - Mode: 3, - ComplexOfNoiseText: base64Captcha.CaptchaComplexLower, - ComplexOfNoiseDot: base64Captcha.CaptchaComplexLower, - IsShowHollowLine: false, - IsShowNoiseDot: false, - IsShowNoiseText: false, - IsShowSlimeLine: false, - IsShowSineLine: false, - CaptchaLen: 6, -} - // CORSConfig 跨域配置 var CORSConfig = &cors{ AllowOrigins: []string{"UNSET"}, @@ -75,3 +58,5 @@ var SSLConfig = &ssl{ var UnixConfig = &unix{ Listen: "", } + +var OptionOverwrite = map[string]interface{}{} From eaa8c9e12d68ba7f7be2c0ace10e1ec668494cf8 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Wed, 23 Mar 2022 19:02:39 +0800 Subject: [PATCH 11/21] Refactor: move thumbnail config from ini file to database --- assets | 2 +- models/defaults.go | 117 +++++++++++++++++++++++++ models/migration.go | 108 ----------------------- models/setting.go | 9 ++ pkg/conf/conf.go | 27 ------ pkg/conf/defaults.go | 11 --- pkg/filesystem/driver/local/handler.go | 5 +- pkg/filesystem/hooks.go | 3 +- pkg/filesystem/image.go | 17 ++-- pkg/thumb/image.go | 5 +- routers/controllers/file.go | 4 +- 11 files changed, 139 insertions(+), 169 deletions(-) create mode 100644 models/defaults.go diff --git a/assets b/assets index 8a94d68f..1a47ca86 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit 8a94d68f17c170a1055e2d57c2a2a55d18868a9e +Subproject commit 1a47ca8674654a2507fc77764b3ac1f295b04cad diff --git a/models/defaults.go b/models/defaults.go new file mode 100644 index 00000000..bd7d4a4c --- /dev/null +++ b/models/defaults.go @@ -0,0 +1,117 @@ +package model + +import ( + "github.com/cloudreve/Cloudreve/v3/pkg/conf" + "github.com/cloudreve/Cloudreve/v3/pkg/util" + "github.com/gofrs/uuid" +) + +var defaultSettings = []Setting{ + {Name: "siteURL", Value: `http://localhost`, Type: "basic"}, + {Name: "siteName", Value: `Cloudreve`, Type: "basic"}, + {Name: "siteICPId", Value: ``, Type: "basic"}, + {Name: "register_enabled", Value: `1`, Type: "register"}, + {Name: "default_group", Value: `2`, Type: "register"}, + {Name: "siteKeywords", Value: `网盘，网盘`, Type: "basic"}, + {Name: "siteDes", Value: `Cloudreve`, Type: "basic"}, + {Name: "siteTitle", Value: `平步云端`, Type: "basic"}, + {Name: "siteScript", Value: ``, Type: "basic"}, + {Name: "siteID", Value: uuid.Must(uuid.NewV4()).String(), Type: "basic"}, + {Name: "fromName", Value: `Cloudreve`, Type: "mail"}, + {Name: "mail_keepalive", Value: `30`, Type: "mail"}, + {Name: "fromAdress", Value: `no-reply@acg.blue`, Type: "mail"}, + {Name: "smtpHost", Value: `smtp.mxhichina.com`, Type: "mail"}, + {Name: "smtpPort", Value: `25`, Type: "mail"}, + {Name: "replyTo", Value: `abslant@126.com`, Type: "mail"}, + {Name: "smtpUser", Value: `no-reply@acg.blue`, Type: "mail"}, + {Name: "smtpPass", Value: ``, Type: "mail"}, + {Name: "smtpEncryption", Value: `0`, Type: "mail"}, + {Name: "maxEditSize", Value: `4194304`, Type: "file_edit"}, + {Name: "archive_timeout", Value: `60`, Type: "timeout"}, + {Name: "download_timeout", Value: `60`, Type: "timeout"}, + {Name: "preview_timeout", Value: `60`, Type: "timeout"}, + {Name: "doc_preview_timeout", Value: `60`, Type: "timeout"}, + {Name: "upload_credential_timeout", Value: `1800`, Type: "timeout"}, + {Name: "upload_session_timeout", Value: `86400`, Type: "timeout"}, + {Name: "slave_api_timeout", Value: `60`, Type: "timeout"}, + {Name: "slave_node_retry", Value: `3`, Type: "slave"}, + {Name: "slave_ping_interval", Value: `60`, Type: "slave"}, + {Name: "slave_recover_interval", Value: `120`, Type: "slave"}, + {Name: "slave_transfer_timeout", Value: `172800`, Type: "timeout"}, + {Name: "onedrive_monitor_timeout", Value: `600`, Type: "timeout"}, + {Name: "share_download_session_timeout", Value: `2073600`, Type: "timeout"}, + {Name: "onedrive_callback_check", Value: `20`, Type: "timeout"}, + {Name: "folder_props_timeout", Value: `300`, Type: "timeout"}, + {Name: "chunk_retries", Value: `5`, Type: "retry"}, + {Name: "onedrive_source_timeout", Value: `1800`, Type: "timeout"}, + {Name: "reset_after_upload_failed", Value: `0`, Type: "upload"}, + {Name: "login_captcha", Value: `0`, Type: "login"}, + {Name: "reg_captcha", Value: `0`, Type: "login"}, + {Name: "email_active", Value: `0`, Type: "register"}, + {Name: "mail_activation_template", Value: `激活您的账户
激活{siteTitle}账户
亲爱的{userName}：
感谢您注册{siteTitle},请点击下方按钮完成账户激活。
激活账户
感谢您选择{siteTitle}。
此邮件由系统自动发送，请不要直接回复。
`, Type: "mail_template"}, + {Name: "forget_captcha", Value: `0`, Type: "login"}, + {Name: "mail_reset_pwd_template", Value: `重设密码
重设{siteTitle}密码
亲爱的{userName}：
请点击下方按钮完成密码重设。如果非你本人操作，请忽略此邮件。
重设密码
感谢您选择{siteTitle}。
此邮件由系统自动发送，请不要直接回复。
`, Type: "mail_template"}, + {Name: "db_version_" + conf.RequiredDBVersion, Value: `installed`, Type: "version"}, + {Name: "hot_share_num", Value: `10`, Type: "share"}, + {Name: "gravatar_server", Value: `https://www.gravatar.com/`, Type: "avatar"}, + {Name: "defaultTheme", Value: `#3f51b5`, Type: "basic"}, + {Name: "themes", Value: `{"#3f51b5":{"palette":{"primary":{"main":"#3f51b5"},"secondary":{"main":"#f50057"}}},"#2196f3":{"palette":{"primary":{"main":"#2196f3"},"secondary":{"main":"#FFC107"}}},"#673AB7":{"palette":{"primary":{"main":"#673AB7"},"secondary":{"main":"#2196F3"}}},"#E91E63":{"palette":{"primary":{"main":"#E91E63"},"secondary":{"main":"#42A5F5","contrastText":"#fff"}}},"#FF5722":{"palette":{"primary":{"main":"#FF5722"},"secondary":{"main":"#3F51B5"}}},"#FFC107":{"palette":{"primary":{"main":"#FFC107"},"secondary":{"main":"#26C6DA"}}},"#8BC34A":{"palette":{"primary":{"main":"#8BC34A","contrastText":"#fff"},"secondary":{"main":"#FF8A65","contrastText":"#fff"}}},"#009688":{"palette":{"primary":{"main":"#009688"},"secondary":{"main":"#4DD0E1","contrastText":"#fff"}}},"#607D8B":{"palette":{"primary":{"main":"#607D8B"},"secondary":{"main":"#F06292"}}},"#795548":{"palette":{"primary":{"main":"#795548"},"secondary":{"main":"#4CAF50","contrastText":"#fff"}}}}`, Type: "basic"}, + {Name: "max_worker_num", Value: `10`, Type: "task"}, + {Name: "max_parallel_transfer", Value: `4`, Type: "task"}, + {Name: "secret_key", Value: util.RandStringRunes(256), Type: "auth"}, + {Name: "temp_path", Value: "temp", Type: "path"}, + {Name: "avatar_path", Value: "avatar", Type: "path"}, + {Name: "avatar_size", Value: "2097152", Type: "avatar"}, + {Name: "avatar_size_l", Value: "200", Type: "avatar"}, + {Name: "avatar_size_m", Value: "130", Type: "avatar"}, + {Name: "avatar_size_s", Value: "50", Type: "avatar"}, + {Name: "home_view_method", Value: "icon", Type: "view"}, + {Name: "share_view_method", Value: "list", Type: "view"}, + {Name: "cron_garbage_collect", Value: "@hourly", Type: "cron"}, + {Name: "cron_recycle_upload_session", Value: "@every 1h30m", Type: "cron"}, + {Name: "authn_enabled", Value: "0", Type: "authn"}, + {Name: "captcha_type", Value: "normal", Type: "captcha"}, + {Name: "captcha_height", Value: "60", Type: "captcha"}, + {Name: "captcha_width", Value: "240", Type: "captcha"}, + {Name: "captcha_mode", Value: "3", Type: "captcha"}, + {Name: "captcha_ComplexOfNoiseText", Value: "0", Type: "captcha"}, + {Name: "captcha_ComplexOfNoiseDot", Value: "0", Type: "captcha"}, + {Name: "captcha_IsShowHollowLine", Value: "0", Type: "captcha"}, + {Name: "captcha_IsShowNoiseDot", Value: "1", Type: "captcha"}, + {Name: "captcha_IsShowNoiseText", Value: "0", Type: "captcha"}, + {Name: "captcha_IsShowSlimeLine", Value: "1", Type: "captcha"}, + {Name: "captcha_IsShowSineLine", Value: "0", Type: "captcha"}, + {Name: "captcha_CaptchaLen", Value: "6", Type: "captcha"}, + {Name: "captcha_ReCaptchaKey", Value: "defaultKey", Type: "captcha"}, + {Name: "captcha_ReCaptchaSecret", Value: "defaultSecret", Type: "captcha"}, + {Name: "captcha_TCaptcha_CaptchaAppId", Value: "", Type: "captcha"}, + {Name: "captcha_TCaptcha_AppSecretKey", Value: "", Type: "captcha"}, + {Name: "captcha_TCaptcha_SecretId", Value: "", Type: "captcha"}, + {Name: "captcha_TCaptcha_SecretKey", Value: "", Type: "captcha"}, + {Name: "thumb_width", Value: "400", Type: "thumb"}, + {Name: "thumb_height", Value: "300", Type: "thumb"}, + {Name: "thumb_file_suffix", Value: "._thumb", Type: "thumb"}, + {Name: "thumb_max_task_count", Value: "-1", Type: "thumb"}, + {Name: "thumb_encode_method", Value: "jpg", Type: "thumb"}, + {Name: "thumb_gc_after_gen", Value: "false", Type: "thumb"}, + {Name: "thumb_encode_quality", Value: "85", Type: "thumb"}, + {Name: "pwa_small_icon", Value: "/static/img/favicon.ico", Type: "pwa"}, + {Name: "pwa_medium_icon", Value: "/static/img/logo192.png", Type: "pwa"}, + {Name: "pwa_large_icon", Value: "/static/img/logo512.png", Type: "pwa"}, + {Name: "pwa_display", Value: "standalone", Type: "pwa"}, + {Name: "pwa_theme_color", Value: "#000000", Type: "pwa"}, + {Name: "pwa_background_color", Value: "#ffffff", Type: "pwa"}, + {Name: "office_preview_service", Value: "https://view.officeapps.live.com/op/view.aspx?src={$src}", Type: "preview"}, +} diff --git a/models/migration.go b/models/migration.go index e383f7e3..d3ba8dfd 100644 --- a/models/migration.go +++ b/models/migration.go @@ -7,7 +7,6 @@ import ( "github.com/cloudreve/Cloudreve/v3/pkg/conf" "github.com/cloudreve/Cloudreve/v3/pkg/util" "github.com/fatih/color" - "github.com/gofrs/uuid" "github.com/hashicorp/go-version" "github.com/jinzhu/gorm" "sort" @@ -86,113 +85,6 @@ func addDefaultPolicy() { } func addDefaultSettings() { - siteID, _ := uuid.NewV4() - - defaultSettings := []Setting{ - {Name: "siteURL", Value: `http://localhost`, Type: "basic"}, - {Name: "siteName", Value: `Cloudreve`, Type: "basic"}, - {Name: "siteICPId", Value: ``, Type: "basic"}, - {Name: "register_enabled", Value: `1`, Type: "register"}, - {Name: "default_group", Value: `2`, Type: "register"}, - {Name: "siteKeywords", Value: `网盘，网盘`, Type: "basic"}, - {Name: "siteDes", Value: `Cloudreve`, Type: "basic"}, - {Name: "siteTitle", Value: `平步云端`, Type: "basic"}, - {Name: "siteScript", Value: ``, Type: "basic"}, - {Name: "siteID", Value: siteID.String(), Type: "basic"}, - {Name: "fromName", Value: `Cloudreve`, Type: "mail"}, - {Name: "mail_keepalive", Value: `30`, Type: "mail"}, - {Name: "fromAdress", Value: `no-reply@acg.blue`, Type: "mail"}, - {Name: "smtpHost", Value: `smtp.mxhichina.com`, Type: "mail"}, - {Name: "smtpPort", Value: `25`, Type: "mail"}, - {Name: "replyTo", Value: `abslant@126.com`, Type: "mail"}, - {Name: "smtpUser", Value: `no-reply@acg.blue`, Type: "mail"}, - {Name: "smtpPass", Value: ``, Type: "mail"}, - {Name: "smtpEncryption", Value: `0`, Type: "mail"}, - {Name: "maxEditSize", Value: `4194304`, Type: "file_edit"}, - {Name: "archive_timeout", Value: `60`, Type: "timeout"}, - {Name: "download_timeout", Value: `60`, Type: "timeout"}, - {Name: "preview_timeout", Value: `60`, Type: "timeout"}, - {Name: "doc_preview_timeout", Value: `60`, Type: "timeout"}, - {Name: "upload_credential_timeout", Value: `1800`, Type: "timeout"}, - {Name: "upload_session_timeout", Value: `86400`, Type: "timeout"}, - {Name: "slave_api_timeout", Value: `60`, Type: "timeout"}, - {Name: "slave_node_retry", Value: `3`, Type: "slave"}, - {Name: "slave_ping_interval", Value: `60`, Type: "slave"}, - {Name: "slave_recover_interval", Value: `120`, Type: "slave"}, - {Name: "slave_transfer_timeout", Value: `172800`, Type: "timeout"}, - {Name: "onedrive_monitor_timeout", Value: `600`, Type: "timeout"}, - {Name: "share_download_session_timeout", Value: `2073600`, Type: "timeout"}, - {Name: "onedrive_callback_check", Value: `20`, Type: "timeout"}, - {Name: "folder_props_timeout", Value: `300`, Type: "timeout"}, - {Name: "chunk_retries", Value: `5`, Type: "retry"}, - {Name: "onedrive_source_timeout", Value: `1800`, Type: "timeout"}, - {Name: "reset_after_upload_failed", Value: `0`, Type: "upload"}, - {Name: "login_captcha", Value: `0`, Type: "login"}, - {Name: "reg_captcha", Value: `0`, Type: "login"}, - {Name: "email_active", Value: `0`, Type: "register"}, - {Name: "mail_activation_template", Value: `激活您的账户
激活{siteTitle}账户
亲爱的{userName}：
感谢您注册{siteTitle},请点击下方按钮完成账户激活。
激活账户
感谢您选择{siteTitle}。
此邮件由系统自动发送，请不要直接回复。
`, Type: "mail_template"}, - {Name: "forget_captcha", Value: `0`, Type: "login"}, - {Name: "mail_reset_pwd_template", Value: `重设密码
重设{siteTitle}密码
亲爱的{userName}：
请点击下方按钮完成密码重设。如果非你本人操作，请忽略此邮件。
重设密码
感谢您选择{siteTitle}。
此邮件由系统自动发送，请不要直接回复。
`, Type: "mail_template"}, - {Name: "db_version_" + conf.RequiredDBVersion, Value: `installed`, Type: "version"}, - {Name: "hot_share_num", Value: `10`, Type: "share"}, - {Name: "gravatar_server", Value: `https://www.gravatar.com/`, Type: "avatar"}, - {Name: "defaultTheme", Value: `#3f51b5`, Type: "basic"}, - {Name: "themes", Value: `{"#3f51b5":{"palette":{"primary":{"main":"#3f51b5"},"secondary":{"main":"#f50057"}}},"#2196f3":{"palette":{"primary":{"main":"#2196f3"},"secondary":{"main":"#FFC107"}}},"#673AB7":{"palette":{"primary":{"main":"#673AB7"},"secondary":{"main":"#2196F3"}}},"#E91E63":{"palette":{"primary":{"main":"#E91E63"},"secondary":{"main":"#42A5F5","contrastText":"#fff"}}},"#FF5722":{"palette":{"primary":{"main":"#FF5722"},"secondary":{"main":"#3F51B5"}}},"#FFC107":{"palette":{"primary":{"main":"#FFC107"},"secondary":{"main":"#26C6DA"}}},"#8BC34A":{"palette":{"primary":{"main":"#8BC34A","contrastText":"#fff"},"secondary":{"main":"#FF8A65","contrastText":"#fff"}}},"#009688":{"palette":{"primary":{"main":"#009688"},"secondary":{"main":"#4DD0E1","contrastText":"#fff"}}},"#607D8B":{"palette":{"primary":{"main":"#607D8B"},"secondary":{"main":"#F06292"}}},"#795548":{"palette":{"primary":{"main":"#795548"},"secondary":{"main":"#4CAF50","contrastText":"#fff"}}}}`, Type: "basic"}, - {Name: "max_worker_num", Value: `10`, Type: "task"}, - {Name: "max_parallel_transfer", Value: `4`, Type: "task"}, - {Name: "secret_key", Value: util.RandStringRunes(256), Type: "auth"}, - {Name: "temp_path", Value: "temp", Type: "path"}, - {Name: "avatar_path", Value: "avatar", Type: "path"}, - {Name: "avatar_size", Value: "2097152", Type: "avatar"}, - {Name: "avatar_size_l", Value: "200", Type: "avatar"}, - {Name: "avatar_size_m", Value: "130", Type: "avatar"}, - {Name: "avatar_size_s", Value: "50", Type: "avatar"}, - {Name: "home_view_method", Value: "icon", Type: "view"}, - {Name: "share_view_method", Value: "list", Type: "view"}, - {Name: "cron_garbage_collect", Value: "@hourly", Type: "cron"}, - {Name: "cron_recycle_upload_session", Value: "@every 1h30m", Type: "cron"}, - {Name: "authn_enabled", Value: "0", Type: "authn"}, - {Name: "captcha_type", Value: "normal", Type: "captcha"}, - {Name: "captcha_height", Value: "60", Type: "captcha"}, - {Name: "captcha_width", Value: "240", Type: "captcha"}, - {Name: "captcha_mode", Value: "3", Type: "captcha"}, - {Name: "captcha_ComplexOfNoiseText", Value: "0", Type: "captcha"}, - {Name: "captcha_ComplexOfNoiseDot", Value: "0", Type: "captcha"}, - {Name: "captcha_IsShowHollowLine", Value: "0", Type: "captcha"}, - {Name: "captcha_IsShowNoiseDot", Value: "1", Type: "captcha"}, - {Name: "captcha_IsShowNoiseText", Value: "0", Type: "captcha"}, - {Name: "captcha_IsShowSlimeLine", Value: "1", Type: "captcha"}, - {Name: "captcha_IsShowSineLine", Value: "0", Type: "captcha"}, - {Name: "captcha_CaptchaLen", Value: "6", Type: "captcha"}, - {Name: "captcha_ReCaptchaKey", Value: "defaultKey", Type: "captcha"}, - {Name: "captcha_ReCaptchaSecret", Value: "defaultSecret", Type: "captcha"}, - {Name: "captcha_TCaptcha_CaptchaAppId", Value: "", Type: "captcha"}, - {Name: "captcha_TCaptcha_AppSecretKey", Value: "", Type: "captcha"}, - {Name: "captcha_TCaptcha_SecretId", Value: "", Type: "captcha"}, - {Name: "captcha_TCaptcha_SecretKey", Value: "", Type: "captcha"}, - {Name: "thumb_width", Value: "400", Type: "thumb"}, - {Name: "thumb_height", Value: "300", Type: "thumb"}, - {Name: "pwa_small_icon", Value: "/static/img/favicon.ico", Type: "pwa"}, - {Name: "pwa_medium_icon", Value: "/static/img/logo192.png", Type: "pwa"}, - {Name: "pwa_large_icon", Value: "/static/img/logo512.png", Type: "pwa"}, - {Name: "pwa_display", Value: "standalone", Type: "pwa"}, - {Name: "pwa_theme_color", Value: "#000000", Type: "pwa"}, - {Name: "pwa_background_color", Value: "#ffffff", Type: "pwa"}, - {Name: "office_preview_service", Value: "https://view.officeapps.live.com/op/view.aspx?src={$src}", Type: "preview"}, - } - for _, value := range defaultSettings { DB.Where(Setting{Name: value.Name}).Create(&value) } diff --git a/models/setting.go b/models/setting.go index 1738c1d8..643ca848 100644 --- a/models/setting.go +++ b/models/setting.go @@ -43,6 +43,15 @@ func GetSettingByName(name string) string { return "" } +// GetSettingByNameWithDefault 用 Name 获取设置值, 取不到时使用缺省值 +func GetSettingByNameWithDefault(name, fallback string) string { + res := GetSettingByName(name) + if res == "" { + return fallback + } + return res +} + // GetSettingByNames 用多个 Name 获取设置值 func GetSettingByNames(names ...string) map[string]string { var queryRes []Setting diff --git a/pkg/conf/conf.go b/pkg/conf/conf.go index 53fd4c10..4988ef34 100644 --- a/pkg/conf/conf.go +++ b/pkg/conf/conf.go @@ -45,21 +45,6 @@ type slave struct { SignatureTTL int `validate:"omitempty,gte=1"` } -// captcha 验证码配置 -type captcha struct { - Height int `validate:"gte=0"` - Width int `validate:"gte=0"` - Mode int `validate:"gte=0,lte=3"` - ComplexOfNoiseText int `validate:"gte=0,lte=2"` - ComplexOfNoiseDot int `validate:"gte=0,lte=2"` - IsShowHollowLine bool - IsShowNoiseDot bool - IsShowNoiseText bool - IsShowSlimeLine bool - IsShowSineLine bool - CaptchaLen int `validate:"gt=0"` -} - // redis 配置 type redis struct { Network string @@ -68,17 +53,6 @@ type redis struct { DB string } -// 缩略图 配置 -type thumb struct { - MaxWidth uint - MaxHeight uint - FileSuffix string `validate:"min=1"` - MaxTaskCount int - EncodeMethod string `validate:"eq=jpg|eq=png"` - EncodeQuality int `validate:"gte=1,lte=100"` - GCAfterGen bool -} - // 跨域配置 type cors struct { AllowOrigins []string @@ -132,7 +106,6 @@ func Init(path string) { "SSL": SSLConfig, "UnixSocket": UnixConfig, "Redis": RedisConfig, - "Thumbnail": ThumbConfig, "CORS": CORSConfig, "Slave": SlaveConfig, } diff --git a/pkg/conf/defaults.go b/pkg/conf/defaults.go index a9989763..44c5371e 100644 --- a/pkg/conf/defaults.go +++ b/pkg/conf/defaults.go @@ -32,17 +32,6 @@ var CORSConfig = &cors{ ExposeHeaders: nil, } -// ThumbConfig 缩略图配置 -var ThumbConfig = &thumb{ - MaxWidth: 400, - MaxHeight: 300, - FileSuffix: "._thumb", - MaxTaskCount: -1, - EncodeMethod: "jpg", - GCAfterGen: false, - EncodeQuality: 85, -} - // SlaveConfig 从机配置 var SlaveConfig = &slave{ CallbackTimeout: 20, diff --git a/pkg/filesystem/driver/local/handler.go b/pkg/filesystem/driver/local/handler.go index 85bb3eb8..7e665c89 100644 --- a/pkg/filesystem/driver/local/handler.go +++ b/pkg/filesystem/driver/local/handler.go @@ -12,7 +12,6 @@ import ( model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/pkg/auth" "github.com/cloudreve/Cloudreve/v3/pkg/cache" - "github.com/cloudreve/Cloudreve/v3/pkg/conf" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response" "github.com/cloudreve/Cloudreve/v3/pkg/serializer" @@ -188,7 +187,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err } // 尝试删除文件的缩略图（如果有） - _ = os.Remove(util.RelativePath(value + conf.ThumbConfig.FileSuffix)) + _ = os.Remove(util.RelativePath(value + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"))) } return deleteFailed, retErr @@ -196,7 +195,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err // Thumb 获取文件缩略图 func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) { - file, err := handler.Get(ctx, path+conf.ThumbConfig.FileSuffix) + file, err := handler.Get(ctx, path+model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")) if err != nil { return nil, err } diff --git a/pkg/filesystem/hooks.go b/pkg/filesystem/hooks.go index e73fa33a..0cd1cb99 100644 --- a/pkg/filesystem/hooks.go +++ b/pkg/filesystem/hooks.go @@ -5,7 +5,6 @@ import ( model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/pkg/cache" "github.com/cloudreve/Cloudreve/v3/pkg/cluster" - "github.com/cloudreve/Cloudreve/v3/pkg/conf" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" "github.com/cloudreve/Cloudreve/v3/pkg/serializer" @@ -245,7 +244,7 @@ func HookGenerateThumb(ctx context.Context, fs *FileSystem, fileHeader fsctx.Fil fs.recycleLock.Lock() go func() { defer fs.recycleLock.Unlock() - _, _ = fs.Handler.Delete(ctx, []string{fileMode.SourceName + conf.ThumbConfig.FileSuffix}) + _, _ = fs.Handler.Delete(ctx, []string{fileMode.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")}) fs.GenerateThumbnail(ctx, fileMode) }() } diff --git a/pkg/filesystem/image.go b/pkg/filesystem/image.go index 759ebca1..30e80bf4 100644 --- a/pkg/filesystem/image.go +++ b/pkg/filesystem/image.go @@ -3,7 +3,6 @@ package filesystem import ( "context" "fmt" - "strconv" "sync" "runtime" @@ -65,7 +64,7 @@ type Pool struct { // Init 初始化任务池 func getThumbWorker() *Pool { once.Do(func() { - maxWorker := conf.ThumbConfig.MaxTaskCount + maxWorker := model.GetIntSetting("thumb_max_task_count", -1) if maxWorker <= 0 { maxWorker = runtime.GOMAXPROCS(0) } @@ -118,9 +117,9 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) { // 生成缩略图 image.GetThumb(fs.GenerateThumbnailSize(w, h)) // 保存到文件 - err = image.Save(util.RelativePath(file.SourceName + conf.ThumbConfig.FileSuffix)) + err = image.Save(util.RelativePath(file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"))) image = nil - if conf.ThumbConfig.GCAfterGen { + if model.IsTrueVal(model.GetSettingByName("thumb_gc_after_gen")) { util.Log().Debug("GenerateThumbnail runtime.GC") runtime.GC() } @@ -139,17 +138,11 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) { // 失败时删除缩略图文件 if err != nil { - _, _ = fs.Handler.Delete(newCtx, []string{file.SourceName + conf.ThumbConfig.FileSuffix}) + _, _ = fs.Handler.Delete(newCtx, []string{file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")}) } } // GenerateThumbnailSize 获取要生成的缩略图的尺寸 func (fs *FileSystem) GenerateThumbnailSize(w, h int) (uint, uint) { - if conf.SystemConfig.Mode == "master" { - options := model.GetSettingByNames("thumb_width", "thumb_height") - w, _ := strconv.ParseUint(options["thumb_width"], 10, 32) - h, _ := strconv.ParseUint(options["thumb_height"], 10, 32) - return uint(w), uint(h) - } - return conf.ThumbConfig.MaxWidth, conf.ThumbConfig.MaxHeight + return uint(model.GetIntSetting("thumb_width", 400)), uint(model.GetIntSetting("thumb_width", 300)) } diff --git a/pkg/thumb/image.go b/pkg/thumb/image.go index 7b026206..69c73a3b 100644 --- a/pkg/thumb/image.go +++ b/pkg/thumb/image.go @@ -12,7 +12,6 @@ import ( "strings" model "github.com/cloudreve/Cloudreve/v3/models" - "github.com/cloudreve/Cloudreve/v3/pkg/conf" "github.com/cloudreve/Cloudreve/v3/pkg/util" //"github.com/nfnt/resize" @@ -78,11 +77,11 @@ func (image *Thumb) Save(path string) (err error) { return err } defer out.Close() - switch conf.ThumbConfig.EncodeMethod { + switch model.GetSettingByNameWithDefault("thumb_encode_method", "jpg") { case "png": err = png.Encode(out, image.src) default: - err = jpeg.Encode(out, image.src, &jpeg.Options{Quality: conf.ThumbConfig.EncodeQuality}) + err = jpeg.Encode(out, image.src, &jpeg.Options{Quality: model.GetIntSetting("thumb_encode_quality", 85)}) } return err diff --git a/routers/controllers/file.go b/routers/controllers/file.go index 9b0d2486..1141d7ab 100644 --- a/routers/controllers/file.go +++ b/routers/controllers/file.go @@ -5,7 +5,7 @@ import ( "fmt" "net/http" - "github.com/cloudreve/Cloudreve/v3/pkg/conf" + model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem" "github.com/cloudreve/Cloudreve/v3/pkg/request" "github.com/cloudreve/Cloudreve/v3/pkg/serializer" @@ -174,7 +174,7 @@ func Thumb(c *gin.Context) { } defer resp.Content.Close() - http.ServeContent(c.Writer, c.Request, "thumb."+conf.ThumbConfig.EncodeMethod, fs.FileTarget[0].UpdatedAt, resp.Content) + http.ServeContent(c.Writer, c.Request, "thumb."+model.GetSettingByNameWithDefault("thumb_encode_method", "jpg"), fs.FileTarget[0].UpdatedAt, resp.Content) } From 5a3ea898667636b12ea663c67c5d0b38a28900c3 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Wed, 23 Mar 2022 19:26:25 +0800 Subject: [PATCH 12/21] Feat: support `{ext}` and `{uuid}` magic variable --- assets | 2 +- models/policy.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/assets b/assets index 1a47ca86..1f488377 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit 1a47ca8674654a2507fc77764b3ac1f295b04cad +Subproject commit 1f4883774ab9fc24685a4adf8099e38e7daf67c8 diff --git a/models/policy.go b/models/policy.go index 0dc06010..4a827fb2 100644 --- a/models/policy.go +++ b/models/policy.go @@ -3,6 +3,7 @@ package model import ( "encoding/gob" "encoding/json" + "github.com/gofrs/uuid" "path" "path/filepath" "strconv" @@ -170,6 +171,8 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string { "{minute}": time.Now().Format("04"), "{second}": time.Now().Format("05"), "{originname}": origin, + "{ext}": filepath.Ext(origin), + "{uuid}": uuid.Must(uuid.NewV4()).String(), } fileRule = util.Replace(replaceTable, fileRule) From e51c5cd70ddbdc735f962feb85a881b007621272 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Wed, 23 Mar 2022 19:32:31 +0800 Subject: [PATCH 13/21] Fix: root folder should not be deleted --- pkg/filesystem/manage.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/filesystem/manage.go b/pkg/filesystem/manage.go index ddf83e6e..ad327457 100644 --- a/pkg/filesystem/manage.go +++ b/pkg/filesystem/manage.go @@ -221,6 +221,15 @@ func (fs *FileSystem) ListDeleteDirs(ctx context.Context, ids []uint) error { if err != nil { return ErrDBListObjects.WithError(err) } + + // 忽略根目录 + for i := 0; i < len(folders); i++ { + if folders[i].ParentID == nil { + folders = append(folders[:i], folders[i+1:]...) + break + } + } + fs.SetTargetDir(&folders) // 检索目录下的子文件 From a568e5e45a844be7bcc768b1054d583d9c510b09 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Wed, 23 Mar 2022 20:05:10 +0800 Subject: [PATCH 14/21] Test: new changes in middleware pkg --- middleware/auth.go | 16 +- middleware/auth_test.go | 521 +++++++++++++--------------------------- 2 files changed, 162 insertions(+), 375 deletions(-) diff --git a/middleware/auth.go b/middleware/auth.go index ca674028..7d1dd3f9 100644 --- a/middleware/auth.go +++ b/middleware/auth.go @@ -199,6 +199,7 @@ func QiniuCallbackAuth() gin.HandlerFunc { c.Abort() return } + if !ok { c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名无效"}) c.Abort() @@ -283,21 +284,6 @@ func OneDriveCallbackAuth() gin.HandlerFunc { } } -// S3CallbackAuth Amazon S3回调签名验证 -func S3CallbackAuth() gin.HandlerFunc { - return func(c *gin.Context) { - //// 验证key并查找用户 - //resp, _ := uploadCallbackCheck(c) - //if resp.Code != 0 { - // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg}) - // c.Abort() - // return - //} - - c.Next() - } -} - // IsAdmin 必须为管理员用户组 func IsAdmin() gin.HandlerFunc { return func(c *gin.Context) { diff --git a/middleware/auth_test.go b/middleware/auth_test.go index ab206a09..9e8650fe 100644 --- a/middleware/auth_test.go +++ b/middleware/auth_test.go @@ -3,21 +3,24 @@ package middleware import ( "database/sql" "errors" + "github.com/cloudreve/Cloudreve/v3/pkg/cache" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem" + "github.com/cloudreve/Cloudreve/v3/pkg/mq" + "github.com/cloudreve/Cloudreve/v3/pkg/serializer" + "github.com/qiniu/go-sdk/v7/auth/qbox" "io/ioutil" "net/http" "net/http/httptest" "strings" "testing" + "time" "github.com/DATA-DOG/go-sqlmock" model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/pkg/auth" - "github.com/cloudreve/Cloudreve/v3/pkg/cache" - "github.com/cloudreve/Cloudreve/v3/pkg/serializer" "github.com/cloudreve/Cloudreve/v3/pkg/util" "github.com/gin-gonic/gin" "github.com/jinzhu/gorm" - "github.com/qiniu/go-sdk/v7/auth/qbox" "github.com/stretchr/testify/assert" ) @@ -223,19 +226,31 @@ func TestWebDAVAuth(t *testing.T) { } -func TestRemoteCallbackAuth(t *testing.T) { +func TestUseUploadSession(t *testing.T) { asserts := assert.New(t) rec := httptest.NewRecorder() - AuthFunc := RemoteCallbackAuth() + AuthFunc := UseUploadSession("local") + + // sessionID 为空 + { + + c, _ := gin.CreateTestContext(rec) + c.Params = []gin.Param{} + c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/sessionID", nil) + authInstance := auth.HMACAuth{SecretKey: []byte("123")} + auth.SignRequest(authInstance, c.Request, 0) + AuthFunc(c) + asserts.True(c.IsAborted()) + } // 成功 { cache.Set( - "callback_testCallBackRemote", + filesystem.UploadSessionCachePrefix+"testCallBackRemote", serializer.UploadSession{ UID: 1, - PolicyID: 513, VirtualPath: "/", + Policy: model.Policy{Type: "local"}, }, 0, ) @@ -248,7 +263,7 @@ func TestRemoteCallbackAuth(t *testing.T) { WillReturnRows(sqlmock.NewRows([]string{"id", "secret_key"}).AddRow(2, "123")) c, _ := gin.CreateTestContext(rec) c.Params = []gin.Param{ - {"key", "testCallBackRemote"}, + {"sessionID", "testCallBackRemote"}, } c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil) authInstance := auth.HMACAuth{SecretKey: []byte("123")} @@ -257,79 +272,95 @@ func TestRemoteCallbackAuth(t *testing.T) { asserts.NoError(mock.ExpectationsWereMet()) asserts.False(c.IsAborted()) } +} - // Callback Key 不存在 - { +func TestUploadCallbackCheck(t *testing.T) { + a := assert.New(t) + rec := httptest.NewRecorder() + // 上传会话不存在 + { c, _ := gin.CreateTestContext(rec) c.Params = []gin.Param{ - {"key", "testCallBackRemote"}, + {"sessionID", "testSessionNotExist"}, } - c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil) - authInstance := auth.HMACAuth{SecretKey: []byte("123")} - auth.SignRequest(authInstance, c.Request, 0) - AuthFunc(c) - asserts.True(c.IsAborted()) + res := uploadCallbackCheck(c, "local") + a.Contains("上传会话不存在或已过期", res.Msg) } - // 用户不存在 + // 上传策略不一致 { + c, _ := gin.CreateTestContext(rec) + c.Params = []gin.Param{ + {"sessionID", "testPolicyNotMatch"}, + } cache.Set( - "callback_testCallBackRemote", + filesystem.UploadSessionCachePrefix+"testPolicyNotMatch", serializer.UploadSession{ UID: 1, - PolicyID: 550, VirtualPath: "/", + Policy: model.Policy{Type: "remote"}, }, 0, ) - cache.Deletes([]string{"1"}, "policy_") - mock.ExpectQuery("SELECT(.+)users(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"})) - c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testCallBackRemote"}, - } - c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil) - authInstance := auth.HMACAuth{SecretKey: []byte("123")} - auth.SignRequest(authInstance, c.Request, 0) - AuthFunc(c) - asserts.NoError(mock.ExpectationsWereMet()) - asserts.True(c.IsAborted()) + res := uploadCallbackCheck(c, "local") + a.Contains("Policy not supported", res.Msg) } - // 签名错误 + // 用户不存在 { + c, _ := gin.CreateTestContext(rec) + c.Params = []gin.Param{ + {"sessionID", "testUserNotExist"}, + } cache.Set( - "callback_testCallBackRemote", + filesystem.UploadSessionCachePrefix+"testUserNotExist", serializer.UploadSession{ - UID: 1, - PolicyID: 514, + UID: 313, VirtualPath: "/", + Policy: model.Policy{Type: "remote"}, }, 0, ) - cache.Deletes([]string{"1"}, "policy_") mock.ExpectQuery("SELECT(.+)users(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1)) - mock.ExpectQuery("SELECT(.+)groups(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[514]")) - mock.ExpectQuery("SELECT(.+)policies(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "secret_key"}).AddRow(2, "123")) + WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"})) + res := uploadCallbackCheck(c, "remote") + a.Contains("找不到用户", res.Msg) + a.NoError(mock.ExpectationsWereMet()) + _, ok := cache.Get(filesystem.UploadSessionCachePrefix + "testUserNotExist") + a.False(ok) + } +} + +func TestRemoteCallbackAuth(t *testing.T) { + asserts := assert.New(t) + rec := httptest.NewRecorder() + AuthFunc := RemoteCallbackAuth() + + // 成功 + { c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testCallBackRemote"}, - } + c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{ + UID: 1, + VirtualPath: "/", + Policy: model.Policy{SecretKey: "123"}, + }) c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil) + authInstance := auth.HMACAuth{SecretKey: []byte("123")} + auth.SignRequest(authInstance, c.Request, 0) AuthFunc(c) - asserts.NoError(mock.ExpectationsWereMet()) - asserts.True(c.IsAborted()) + asserts.False(c.IsAborted()) } - // Callback Key 为空 + // 签名错误 { c, _ := gin.CreateTestContext(rec) - c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote", nil) + c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{ + UID: 1, + VirtualPath: "/", + Policy: model.Policy{SecretKey: "123"}, + }) + c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil) AuthFunc(c) asserts.True(c.IsAborted()) } @@ -340,39 +371,17 @@ func TestQiniuCallbackAuth(t *testing.T) { rec := httptest.NewRecorder() AuthFunc := QiniuCallbackAuth() - // Callback Key 相关验证失败 - { - c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testQiniuBackRemote"}, - } - c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testQiniuBackRemote", nil) - AuthFunc(c) - asserts.True(c.IsAborted()) - } - // 成功 { - cache.Set( - "callback_testCallBackQiniu", - serializer.UploadSession{ - UID: 1, - PolicyID: 515, - VirtualPath: "/", - }, - 0, - ) - cache.Deletes([]string{"1"}, "policy_") - mock.ExpectQuery("SELECT(.+)users(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1)) - mock.ExpectQuery("SELECT(.+)groups(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[515]")) - mock.ExpectQuery("SELECT(.+)policies(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123")) c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testCallBackQiniu"}, - } + c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{ + UID: 1, + VirtualPath: "/", + Policy: model.Policy{ + SecretKey: "123", + AccessKey: "123", + }, + }) c.Request, _ = http.NewRequest("POST", "/api/v3/callback/qiniu/testCallBackQiniu", nil) mac := qbox.NewMac("123", "123") token, err := mac.SignRequest(c.Request) @@ -385,33 +394,21 @@ func TestQiniuCallbackAuth(t *testing.T) { // 验证失败 { - cache.Set( - "callback_testCallBackQiniu", - serializer.UploadSession{ - UID: 1, - PolicyID: 516, - VirtualPath: "/", - }, - 0, - ) - cache.Deletes([]string{"1"}, "policy_") - mock.ExpectQuery("SELECT(.+)users(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1)) - mock.ExpectQuery("SELECT(.+)groups(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[516]")) - mock.ExpectQuery("SELECT(.+)policies(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123")) c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testCallBackQiniu"}, - } + c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{ + UID: 1, + VirtualPath: "/", + Policy: model.Policy{ + SecretKey: "123", + AccessKey: "123", + }, + }) c.Request, _ = http.NewRequest("POST", "/api/v3/callback/qiniu/testCallBackQiniu", nil) - mac := qbox.NewMac("123", "123") + mac := qbox.NewMac("123", "1213") token, err := mac.SignRequest(c.Request) asserts.NoError(err) - c.Request.Header["Authorization"] = []string{"QBox " + token + " "} + c.Request.Header["Authorization"] = []string{"QBox " + token} AuthFunc(c) - asserts.NoError(mock.ExpectationsWereMet()) asserts.True(c.IsAborted()) } } @@ -421,76 +418,41 @@ func TestOSSCallbackAuth(t *testing.T) { rec := httptest.NewRecorder() AuthFunc := OSSCallbackAuth() - // Callback Key 相关验证失败 - { - c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testOSSBackRemote"}, - } - c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/testQiniuBackRemote", nil) - AuthFunc(c) - asserts.True(c.IsAborted()) - } - // 签名验证失败 { - cache.Set( - "callback_testCallBackOSS", - serializer.UploadSession{ - UID: 1, - PolicyID: 517, - VirtualPath: "/", - }, - 0, - ) - cache.Deletes([]string{"1"}, "policy_") - mock.ExpectQuery("SELECT(.+)users(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1)) - mock.ExpectQuery("SELECT(.+)groups(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[517]")) - mock.ExpectQuery("SELECT(.+)policies(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123")) c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testCallBackOSS"}, - } + c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{ + UID: 1, + VirtualPath: "/", + Policy: model.Policy{ + SecretKey: "123", + AccessKey: "123", + }, + }) c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/testCallBackOSS", nil) mac := qbox.NewMac("123", "123") token, err := mac.SignRequest(c.Request) asserts.NoError(err) c.Request.Header["Authorization"] = []string{"QBox " + token} AuthFunc(c) - asserts.NoError(mock.ExpectationsWereMet()) asserts.True(c.IsAborted()) } // 成功 { - cache.Set( - "callback_TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH", - serializer.UploadSession{ - UID: 1, - PolicyID: 518, - VirtualPath: "/", - }, - 0, - ) - cache.Deletes([]string{"1"}, "policy_") - mock.ExpectQuery("SELECT(.+)users(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1)) - mock.ExpectQuery("SELECT(.+)groups(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[518]")) - mock.ExpectQuery("SELECT(.+)policies(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123")) c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"}, - } + c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{ + UID: 1, + VirtualPath: "/", + Policy: model.Policy{ + SecretKey: "123", + AccessKey: "123", + }, + }) c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH", ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`))) c.Request.Header["Authorization"] = []string{"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="} c.Request.Header["X-Oss-Pub-Key-Url"] = []string{"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="} AuthFunc(c) - asserts.NoError(mock.ExpectationsWereMet()) asserts.False(c.IsAborted()) } @@ -507,130 +469,71 @@ func TestUpyunCallbackAuth(t *testing.T) { rec := httptest.NewRecorder() AuthFunc := UpyunCallbackAuth() - // Callback Key 相关验证失败 - { - c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testUpyunBackRemote"}, - } - c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil) - AuthFunc(c) - asserts.True(c.IsAborted()) - } - // 无法获取请求正文 { - cache.Set( - "callback_testCallBackUpyun", - serializer.UploadSession{ - UID: 1, - PolicyID: 509, - VirtualPath: "/", - }, - 0, - ) - cache.Deletes([]string{"1"}, "policy_") - mock.ExpectQuery("SELECT(.+)users(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1)) - mock.ExpectQuery("SELECT(.+)groups(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[519]")) - mock.ExpectQuery("SELECT(.+)policies(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123")) c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testCallBackUpyun"}, - } + c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{ + UID: 1, + VirtualPath: "/", + Policy: model.Policy{ + SecretKey: "123", + AccessKey: "123", + }, + }) c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(fakeRead(""))) AuthFunc(c) - asserts.NoError(mock.ExpectationsWereMet()) asserts.True(c.IsAborted()) } // 正文MD5不一致 { - cache.Set( - "callback_testCallBackUpyun", - serializer.UploadSession{ - UID: 1, - PolicyID: 510, - VirtualPath: "/", - }, - 0, - ) - cache.Deletes([]string{"1"}, "policy_") - mock.ExpectQuery("SELECT(.+)users(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1)) - mock.ExpectQuery("SELECT(.+)groups(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[520]")) - mock.ExpectQuery("SELECT(.+)policies(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123")) c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testCallBackUpyun"}, - } + c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{ + UID: 1, + VirtualPath: "/", + Policy: model.Policy{ + SecretKey: "123", + AccessKey: "123", + }, + }) c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1"))) c.Request.Header["Content-Md5"] = []string{"123"} AuthFunc(c) - asserts.NoError(mock.ExpectationsWereMet()) asserts.True(c.IsAborted()) } // 签名不一致 { - cache.Set( - "callback_testCallBackUpyun", - serializer.UploadSession{ - UID: 1, - PolicyID: 511, - VirtualPath: "/", - }, - 0, - ) - cache.Deletes([]string{"1"}, "policy_") - mock.ExpectQuery("SELECT(.+)users(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1)) - mock.ExpectQuery("SELECT(.+)groups(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[521]")) - mock.ExpectQuery("SELECT(.+)policies(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123")) c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testCallBackUpyun"}, - } + c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{ + UID: 1, + VirtualPath: "/", + Policy: model.Policy{ + SecretKey: "123", + AccessKey: "123", + }, + }) c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1"))) c.Request.Header["Content-Md5"] = []string{"c4ca4238a0b923820dcc509a6f75849b"} AuthFunc(c) - asserts.NoError(mock.ExpectationsWereMet()) asserts.True(c.IsAborted()) } // 成功 { - cache.Set( - "callback_testCallBackUpyun", - serializer.UploadSession{ - UID: 1, - PolicyID: 512, - VirtualPath: "/", - }, - 0, - ) - cache.Deletes([]string{"1"}, "policy_") - mock.ExpectQuery("SELECT(.+)users(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1)) - mock.ExpectQuery("SELECT(.+)groups(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[522]")) - mock.ExpectQuery("SELECT(.+)policies(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123")) c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testCallBackUpyun"}, - } + c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{ + UID: 1, + VirtualPath: "/", + Policy: model.Policy{ + SecretKey: "123", + AccessKey: "123", + }, + }) c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1"))) c.Request.Header["Content-Md5"] = []string{"c4ca4238a0b923820dcc509a6f75849b"} c.Request.Header["Authorization"] = []string{"UPYUN 123:GWueK9x493BKFFk5gmfdO2Mn6EM="} AuthFunc(c) - asserts.NoError(mock.ExpectationsWereMet()) asserts.False(c.IsAborted()) } } @@ -640,87 +543,28 @@ func TestOneDriveCallbackAuth(t *testing.T) { rec := httptest.NewRecorder() AuthFunc := OneDriveCallbackAuth() - // Callback Key 相关验证失败 - { - c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testUpyunBackRemote"}, - } - c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil) - AuthFunc(c) - asserts.True(c.IsAborted()) - } - // 成功 - { - cache.Set( - "callback_testCallBackUpyun", - serializer.UploadSession{ - UID: 1, - PolicyID: 512, - VirtualPath: "/", - }, - 0, - ) - cache.Deletes([]string{"1"}, "policy_") - mock.ExpectQuery("SELECT(.+)users(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1)) - mock.ExpectQuery("SELECT(.+)groups(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[657]")) - mock.ExpectQuery("SELECT(.+)policies(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123")) - c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testCallBackUpyun"}, - } - c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1"))) - AuthFunc(c) - asserts.NoError(mock.ExpectationsWereMet()) - asserts.False(c.IsAborted()) - } -} - -func TestCOSCallbackAuth(t *testing.T) { - asserts := assert.New(t) - rec := httptest.NewRecorder() - AuthFunc := COSCallbackAuth() - - // Callback Key 相关验证失败 { c, _ := gin.CreateTestContext(rec) c.Params = []gin.Param{ - {"key", "testUpyunBackRemote"}, + {"sessionID", "TestOneDriveCallbackAuth"}, } - c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil) - AuthFunc(c) - asserts.True(c.IsAborted()) - } - - // 成功 - { - cache.Set( - "callback_testCallBackUpyun", - serializer.UploadSession{ - UID: 1, - PolicyID: 512, - VirtualPath: "/", + c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{ + UID: 1, + VirtualPath: "/", + Policy: model.Policy{ + SecretKey: "123", + AccessKey: "123", }, - 0, - ) - cache.Deletes([]string{"1"}, "policy_") - mock.ExpectQuery("SELECT(.+)users(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1)) - mock.ExpectQuery("SELECT(.+)groups(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[702]")) - mock.ExpectQuery("SELECT(.+)policies(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123")) - c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testCallBackUpyun"}, - } - c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1"))) + }) + c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/TestOneDriveCallbackAuth", ioutil.NopCloser(strings.NewReader("1"))) + res := mq.GlobalMQ.Subscribe("TestOneDriveCallbackAuth", 1) AuthFunc(c) - asserts.NoError(mock.ExpectationsWereMet()) + select { + case <-res: + case <-time.After(time.Millisecond * 500): + asserts.Fail("mq message should be published") + } asserts.False(c.IsAborted()) } } @@ -759,46 +603,3 @@ func TestIsAdmin(t *testing.T) { asserts.False(c.IsAborted()) } } - -func TestS3CallbackAuth(t *testing.T) { - asserts := assert.New(t) - rec := httptest.NewRecorder() - AuthFunc := S3CallbackAuth() - - // Callback Key 相关验证失败 - { - c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testUpyunBackRemote"}, - } - c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil) - AuthFunc(c) - asserts.True(c.IsAborted()) - } - - // 成功 - { - cache.Set( - "callback_testCallBackUpyun", - serializer.UploadSession{ - UID: 1, - PolicyID: 512, - VirtualPath: "/", - }, - 0, - ) - cache.Deletes([]string{"1"}, "policy_") - mock.ExpectQuery("SELECT(.+)users(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1)) - mock.ExpectQuery("SELECT(.+)groups(.+)"). - WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[702]")) - c, _ := gin.CreateTestContext(rec) - c.Params = []gin.Param{ - {"key", "testCallBackUpyun"}, - } - c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1"))) - AuthFunc(c) - asserts.False(c.IsAborted()) - asserts.NoError(mock.ExpectationsWereMet()) - } -} From 1821923b74f65d495ab64cb51eca9a9d414b3440 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Thu, 24 Mar 2022 20:07:56 +0800 Subject: [PATCH 15/21] Test: new changes in model pkg --- models/file_test.go | 215 +++++++++++++++++++++++++++++++++++------ models/folder_test.go | 19 ++-- models/policy.go | 21 ---- models/policy_test.go | 93 +++--------------- models/setting_test.go | 9 ++ models/tag_test.go | 4 +- 6 files changed, 221 insertions(+), 140 deletions(-) diff --git a/models/file_test.go b/models/file_test.go index 7e53c499..0f971848 100644 --- a/models/file_test.go +++ b/models/file_test.go @@ -15,22 +15,62 @@ func TestFile_Create(t *testing.T) { Name: "123", } - mock.ExpectBegin() - mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1)) - mock.ExpectCommit() - fileID, err := file.Create() - asserts.NoError(err) - asserts.Equal(uint(5), fileID) - asserts.Equal(uint(5), file.ID) - asserts.NoError(mock.ExpectationsWereMet()) + // 无法插入文件记录 + { + mock.ExpectBegin() + mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error")) + mock.ExpectRollback() + err := file.Create() + asserts.Error(err) + asserts.NoError(mock.ExpectationsWereMet()) + } - mock.ExpectBegin() - mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error")) - mock.ExpectRollback() - fileID, err = file.Create() - asserts.Error(err) - asserts.NoError(mock.ExpectationsWereMet()) + // 无法更新用户容量 + { + mock.ExpectBegin() + mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1)) + mock.ExpectExec("UPDATE(.+)").WillReturnError(errors.New("error")) + mock.ExpectRollback() + err := file.Create() + asserts.Error(err) + asserts.NoError(mock.ExpectationsWereMet()) + } + + // 成功 + { + mock.ExpectBegin() + mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1)) + mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + err := file.Create() + asserts.NoError(err) + asserts.Equal(uint(5), file.ID) + asserts.NoError(mock.ExpectationsWereMet()) + } +} + +func TestFile_AfterFind(t *testing.T) { + a := assert.New(t) + file := File{ + Name: "123", + Metadata: "{\"name\":\"123\"}", + } + + a.NoError(file.AfterFind()) + a.Equal("123", file.MetadataSerialized["name"]) +} + +func TestFile_BeforeSave(t *testing.T) { + a := assert.New(t) + file := File{ + Name: "123", + MetadataSerialized: map[string]string{ + "name": "123", + }, + } + a.NoError(file.BeforeSave()) + a.Equal("{\"name\":\"123\"}", file.Metadata) } func TestFolder_GetChildFile(t *testing.T) { @@ -175,6 +215,17 @@ func TestGetChildFilesOfFolders(t *testing.T) { } } +func TestGetUploadPlaceholderFiles(t *testing.T) { + a := assert.New(t) + + mock.ExpectQuery("SELECT(.+)upload_session_id(.+)"). + WithArgs(1). + WillReturnRows(sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "1")) + files := GetUploadPlaceholderFiles(1) + a.NoError(mock.ExpectationsWereMet()) + a.Len(files, 1) +} + func TestFile_GetPolicy(t *testing.T) { asserts := assert.New(t) @@ -282,28 +333,50 @@ func TestRemoveFilesWithSoftLinks(t *testing.T) { } } -func TestDeleteFileByIDs(t *testing.T) { - asserts := assert.New(t) +func TestDeleteFiles(t *testing.T) { + a := assert.New(t) - // 出错 + // uid 不一致 + { + err := DeleteFiles([]*File{{}}, 1) + a.Contains("User id not consistent", err.Error()) + } + + // 删除失败 { mock.ExpectBegin() mock.ExpectExec("DELETE(.+)"). WillReturnError(errors.New("error")) mock.ExpectRollback() - err := DeleteFileByIDs([]uint{1, 2, 3}) - asserts.NoError(mock.ExpectationsWereMet()) - asserts.Error(err) + err := DeleteFiles([]*File{{}}, 0) + a.NoError(mock.ExpectationsWereMet()) + a.Error(err) } - // 成功 + + // 无法变更用户容量 { mock.ExpectBegin() mock.ExpectExec("DELETE(.+)"). - WillReturnResult(sqlmock.NewResult(0, 3)) + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnError(errors.New("error")) + mock.ExpectRollback() + err := DeleteFiles([]*File{{}}, 0) + a.NoError(mock.ExpectationsWereMet()) + a.Error(err) + } + + // 成功，其中一个文件已经不存在 + { + mock.ExpectBegin() + mock.ExpectExec("DELETE(.+)"). + WillReturnResult(sqlmock.NewResult(1, 0)) + mock.ExpectExec("DELETE(.+)"). + WillReturnResult(sqlmock.NewResult(2, 1)) + mock.ExpectExec("UPDATE(.+)storage(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() - err := DeleteFileByIDs([]uint{1, 2, 3}) - asserts.NoError(mock.ExpectationsWereMet()) - asserts.NoError(err) + err := DeleteFiles([]*File{{Size: 1}, {Size: 2}}, 0) + a.NoError(mock.ExpectationsWereMet()) + a.NoError(err) } } @@ -324,6 +397,19 @@ func TestGetFilesByParentIDs(t *testing.T) { asserts.Len(files, 3) } +func TestGetFilesByUploadSession(t *testing.T) { + a := assert.New(t) + + mock.ExpectQuery("SELECT(.+)"). + WithArgs(1, "sessionID"). + WillReturnRows( + sqlmock.NewRows([]string{"id", "name"}).AddRow(4, "4.txt")) + files, err := GetFilesByUploadSession("sessionID", 1) + a.NoError(err) + a.NoError(mock.ExpectationsWereMet()) + a.Equal("4.txt", files.Name) +} + func TestFile_Updates(t *testing.T) { asserts := assert.New(t) file := File{Model: gorm.Model{ID: 1}} @@ -340,24 +426,93 @@ func TestFile_Updates(t *testing.T) { // UpdatePicInfo { mock.ExpectBegin() - mock.ExpectExec("UPDATE(.+)").WithArgs(10, sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE(.+)").WithArgs("1,1", 1).WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() - err := file.UpdateSize(10) + err := file.UpdatePicInfo("1,1") asserts.NoError(mock.ExpectationsWereMet()) asserts.NoError(err) } - // UpdatePicInfo + // UpdateSourceName { mock.ExpectBegin() - mock.ExpectExec("UPDATE(.+)").WithArgs("1,1", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE(.+)").WithArgs("newName", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() - err := file.UpdatePicInfo("1,1") + err := file.UpdateSourceName("newName") asserts.NoError(mock.ExpectationsWereMet()) asserts.NoError(err) } } +func TestFile_UpdateSize(t *testing.T) { + a := assert.New(t) + + // 增加成功 + { + file := File{Size: 10} + mock.ExpectBegin() + mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(11, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE(.+)storage(.+)+(.+)").WithArgs(uint64(1), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + + a.NoError(file.UpdateSize(11)) + a.NoError(mock.ExpectationsWereMet()) + } + + // 减少成功 + { + file := File{Size: 10} + mock.ExpectBegin() + mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE(.+)storage(.+)-(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + + a.NoError(file.UpdateSize(8)) + a.NoError(mock.ExpectationsWereMet()) + } + + // 文件更新失败 + { + file := File{Size: 10} + mock.ExpectBegin() + mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnError(errors.New("error")) + mock.ExpectRollback() + + a.Error(file.UpdateSize(8)) + a.NoError(mock.ExpectationsWereMet()) + } + + // 用户容量更新失败 + { + file := File{Size: 10} + mock.ExpectBegin() + mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE(.+)storage(.+)-(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnError(errors.New("error")) + mock.ExpectRollback() + + a.Error(file.UpdateSize(8)) + a.NoError(mock.ExpectationsWereMet()) + } +} + +func TestFile_PopChunkToFile(t *testing.T) { + a := assert.New(t) + timeNow := time.Now() + file := File{} + mock.ExpectBegin() + mock.ExpectExec("UPDATE(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + a.NoError(file.PopChunkToFile(&timeNow, "1,1")) +} + +func TestFile_CanCopy(t *testing.T) { + a := assert.New(t) + file := File{} + a.True(file.CanCopy()) + file.UploadSessionID = &file.Name + a.False(file.CanCopy()) +} + func TestFile_FileInfoInterface(t *testing.T) { asserts := assert.New(t) file := File{ diff --git a/models/folder_test.go b/models/folder_test.go index 7c74f489..8174708d 100644 --- a/models/folder_test.go +++ b/models/folder_test.go @@ -212,12 +212,14 @@ func TestFolder_MoveOrCopyFileTo(t *testing.T) { WithArgs( 1, 2, + 3, 1, 1, ).WillReturnRows( - sqlmock.NewRows([]string{"id", "size"}). - AddRow(1, 10). - AddRow(2, 20), + sqlmock.NewRows([]string{"id", "size", "upload_session_id"}). + AddRow(1, 10, nil). + AddRow(2, 20, nil). + AddRow(2, 20, &folder.Name), ) mock.ExpectBegin() mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) @@ -226,7 +228,7 @@ func TestFolder_MoveOrCopyFileTo(t *testing.T) { mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() storage, err := folder.MoveOrCopyFileTo( - []uint{1, 2}, + []uint{1, 2, 3}, &dstFolder, true, ) @@ -335,7 +337,7 @@ func TestFolder_CopyFolderTo(t *testing.T) { // 测试复制目录结构 // test(2)(5) // 1(3)(6) 2.txt - // 3(4)(7) 4.txt + // 3(4)(7) 4.txt 5.txt(上传中) // 正常情况 成功 { @@ -360,9 +362,10 @@ func TestFolder_CopyFolderTo(t *testing.T) { mock.ExpectQuery("SELECT(.+)"). WithArgs(1, 2, 3, 4). WillReturnRows( - sqlmock.NewRows([]string{"id", "name", "folder_id", "size"}). - AddRow(1, "2.txt", 2, 10). - AddRow(2, "3.txt", 3, 20), + sqlmock.NewRows([]string{"id", "name", "folder_id", "size", "upload_session_id"}). + AddRow(1, "2.txt", 2, 10, nil). + AddRow(2, "3.txt", 3, 20, nil). + AddRow(3, "5.txt", 3, 20, &dstFolder.Name), ) // 复制子文件 diff --git a/models/policy.go b/models/policy.go index 4a827fb2..a5f48268 100644 --- a/models/policy.go +++ b/models/policy.go @@ -179,27 +179,6 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string { return fileRule } -func (policy Policy) getOriginNameRule(origin string) string { - // 部分存储策略可以使用{origin}代表原始文件名 - if origin == "" { - // 如果上游未传回原始文件名，则使用占位符，让云存储端替换 - switch policy.Type { - case "qiniu": - // 七牛会将$(fname)自动替换为原始文件名 - return "$(fname)" - case "local", "remote": - return origin - case "oss", "cos": - // OSS会将${filename}自动替换为原始文件名 - return "${filename}" - case "upyun": - // Upyun会将{filename}{.suffix}自动替换为原始文件名 - return "{filename}{.suffix}" - } - } - return origin -} - // IsDirectlyPreview 返回此策略下文件是否可以直接预览（不需要重定向） func (policy *Policy) IsDirectlyPreview() bool { return policy.Type == "local" diff --git a/models/policy_test.go b/models/policy_test.go index 433d1ecc..c888ada9 100644 --- a/models/policy_test.go +++ b/models/policy_test.go @@ -104,7 +104,7 @@ func TestPolicy_GenerateFileName(t *testing.T) { asserts.Equal("123.txt", testPolicy.GenerateFileName(1, "123.txt")) testPolicy.Type = "oss" - asserts.Equal("${filename}", testPolicy.GenerateFileName(1, "")) + asserts.Equal("origin", testPolicy.GenerateFileName(1, "origin")) } // 重命名开启 @@ -145,19 +145,23 @@ func TestPolicy_GenerateFileName(t *testing.T) { testPolicy.Type = "oss" testPolicy.FileNameRule = "{uid}123{originname}" - asserts.Equal("1123${filename}", testPolicy.GenerateFileName(1, "")) + asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321")) testPolicy.Type = "upyun" testPolicy.FileNameRule = "{uid}123{originname}" - asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, "")) + asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321")) testPolicy.Type = "qiniu" testPolicy.FileNameRule = "{uid}123{originname}" - asserts.Equal("1123$(fname)", testPolicy.GenerateFileName(1, "")) + asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321")) testPolicy.Type = "local" testPolicy.FileNameRule = "{uid}123{originname}" asserts.Equal("1123", testPolicy.GenerateFileName(1, "")) + + testPolicy.Type = "local" + testPolicy.FileNameRule = "{ext}123{uuid}" + asserts.Contains(testPolicy.GenerateFileName(1, "123.txt"), ".txt123") } } @@ -170,78 +174,6 @@ func TestPolicy_IsDirectlyPreview(t *testing.T) { asserts.False(policy.IsDirectlyPreview()) } -func TestPolicy_GetUploadURL(t *testing.T) { - asserts := assert.New(t) - - // 本地 - { - cache.Set("setting_siteURL", "http://127.0.0.1", 0) - policy := Policy{Type: "local", Server: "http://127.0.0.1"} - asserts.Equal("/api/v3/file/upload", policy.GetUploadURL()) - } - - // 远程 - { - policy := Policy{Type: "remote", Server: "http://127.0.0.1"} - asserts.Equal("http://127.0.0.1/api/v3/slave/upload", policy.GetUploadURL()) - } - - // OSS - { - policy := Policy{Type: "oss", BucketName: "base", Server: "127.0.0.1"} - asserts.Equal("https://base.127.0.0.1", policy.GetUploadURL()) - } - - // cos - { - policy := Policy{Type: "cos", BaseURL: "base", Server: "http://127.0.0.1"} - asserts.Equal("http://127.0.0.1", policy.GetUploadURL()) - } - - // upyun - { - policy := Policy{Type: "upyun", BucketName: "base", Server: "http://127.0.0.1"} - asserts.Equal("https://v0.api.upyun.com/base", policy.GetUploadURL()) - } - - // 未知 - { - policy := Policy{Type: "unknown", Server: "http://127.0.0.1"} - asserts.Equal("http://127.0.0.1", policy.GetUploadURL()) - } - - // S3 未填写自动生成 - { - policy := Policy{ - Type: "s3", - Server: "", - BucketName: "bucket", - OptionsSerialized: PolicyOption{Region: "us-east"}, - } - asserts.Equal("https://bucket.s3.us-east.amazonaws.com/", policy.GetUploadURL()) - } - - // s3 自己指定 - { - policy := Policy{ - Type: "s3", - Server: "https://s3.us-east.amazonaws.com/", - BucketName: "bucket", - OptionsSerialized: PolicyOption{Region: "us-east"}, - } - asserts.Equal("https://s3.us-east.amazonaws.com/bucket", policy.GetUploadURL()) - } - -} - -func TestPolicy_IsPathGenerateNeeded(t *testing.T) { - asserts := assert.New(t) - policy := Policy{Type: "qiniu"} - asserts.True(policy.IsPathGenerateNeeded()) - policy.Type = "remote" - asserts.False(policy.IsPathGenerateNeeded()) -} - func TestPolicy_ClearCache(t *testing.T) { asserts := assert.New(t) cache.Set("policy_202", 1, 0) @@ -266,15 +198,18 @@ func TestPolicy_UpdateAccessKey(t *testing.T) { func TestPolicy_Props(t *testing.T) { asserts := assert.New(t) policy := Policy{Type: "onedrive"} + policy.OptionsSerialized.PlaceholderWithSize = true asserts.False(policy.IsThumbGenerateNeeded()) - asserts.True(policy.IsPathGenerateNeeded()) - asserts.True(policy.IsTransitUpload(4)) + asserts.False(policy.IsTransitUpload(4)) asserts.False(policy.IsTransitUpload(5 * 1024 * 1024)) asserts.True(policy.CanStructureBeListed()) + asserts.True(policy.IsUploadPlaceholderWithSize()) policy.Type = "local" asserts.True(policy.IsThumbGenerateNeeded()) - asserts.True(policy.IsPathGenerateNeeded()) asserts.False(policy.CanStructureBeListed()) + asserts.False(policy.IsUploadPlaceholderWithSize()) + policy.Type = "remote" + asserts.True(policy.IsUploadPlaceholderWithSize()) } func TestPolicy_IsThumbExist(t *testing.T) { diff --git a/models/setting_test.go b/models/setting_test.go index d9bf9626..96fc5e03 100644 --- a/models/setting_test.go +++ b/models/setting_test.go @@ -59,6 +59,15 @@ func TestGetSettingByType(t *testing.T) { asserts.Equal(map[string]string{}, settings) } +func TestGetSettingByNameWithDefault(t *testing.T) { + a := assert.New(t) + + rows := sqlmock.NewRows([]string{"name", "value", "type"}) + mock.ExpectQuery("^SELECT \\* FROM `(.+)` WHERE `(.+)`\\.`deleted_at` IS NULL AND(.+)$").WillReturnRows(rows) + settings := GetSettingByNameWithDefault("123", "123321") + a.Equal("123321", settings) +} + func TestGetSettingByNames(t *testing.T) { cache.Store = cache.NewMemoStore() asserts := assert.New(t) diff --git a/models/tag_test.go b/models/tag_test.go index 6063234e..be8d3fb5 100644 --- a/models/tag_test.go +++ b/models/tag_test.go @@ -56,8 +56,8 @@ func TestGetTagsByUID(t *testing.T) { func TestGetTagsByID(t *testing.T) { asserts := assert.New(t) mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"name"}).AddRow("tag")) - res, err := GetTagsByUID(1) + res, err := GetTagsByID(1, 1) asserts.NoError(mock.ExpectationsWereMet()) asserts.NoError(err) - asserts.EqualValues("tag", res[0].Name) + asserts.EqualValues("tag", res.Name) } From 636ac52a3f4f6b378caf257979f300ed275fc0ed Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Sat, 26 Mar 2022 15:32:57 +0800 Subject: [PATCH 16/21] Test: new changes in pkg: cache, cluster, conf --- bootstrap/init.go | 2 +- pkg/cache/driver.go | 4 +- pkg/cache/driver_test.go | 6 +- pkg/cluster/slave.go | 1 - pkg/cluster/slave_test.go | 236 +++++++++++++++---------------- pkg/conf/conf_test.go | 7 +- pkg/mocks/mocks.go | 10 -- pkg/mocks/requestmock/request.go | 15 ++ 8 files changed, 144 insertions(+), 137 deletions(-) create mode 100644 pkg/mocks/requestmock/request.go diff --git a/bootstrap/init.go b/bootstrap/init.go index 6463164c..11fa45eb 100644 --- a/bootstrap/init.go +++ b/bootstrap/init.go @@ -37,7 +37,7 @@ func Init(path string) { { "both", func() { - cache.Init() + cache.Init(conf.SystemConfig.Mode == "slave") }, }, { diff --git a/pkg/cache/driver.go b/pkg/cache/driver.go index 8df967dc..74c12192 100644 --- a/pkg/cache/driver.go +++ b/pkg/cache/driver.go @@ -10,7 +10,7 @@ import ( var Store Driver = NewMemoStore() // Init 初始化缓存 -func Init() { +func Init(isSlave bool) { if conf.RedisConfig.Server != "" && gin.Mode() != gin.TestMode { Store = NewRedisStore( 10, @@ -21,7 +21,7 @@ func Init() { ) } - if conf.SystemConfig.Mode == "slave" { + if isSlave { err := Store.Sets(conf.OptionOverwrite, "setting_") if err != nil { util.Log().Warning("无法覆盖数据库设置: %s", err) diff --git a/pkg/cache/driver_test.go b/pkg/cache/driver_test.go index d30a67f2..a0c5cfc0 100644 --- a/pkg/cache/driver_test.go +++ b/pkg/cache/driver_test.go @@ -56,6 +56,10 @@ func TestInit(t *testing.T) { asserts := assert.New(t) asserts.NotPanics(func() { - Init() + Init(false) + }) + + asserts.NotPanics(func() { + Init(true) }) } diff --git a/pkg/cluster/slave.go b/pkg/cluster/slave.go index 49e2a48d..79118b23 100644 --- a/pkg/cluster/slave.go +++ b/pkg/cluster/slave.go @@ -413,7 +413,6 @@ func getAria2RequestBody(body *serializer.SlaveAria2Call) (io.Reader, error) { return strings.NewReader(string(reqBodyEncoded)), nil } -// TODO: move to slave pkg // RemoteCallback 发送远程存储策略上传回调请求 func RemoteCallback(url string, body serializer.UploadCallback) error { callbackBody, err := json.Marshal(struct { diff --git a/pkg/cluster/slave_test.go b/pkg/cluster/slave_test.go index 25809361..47f4bf2b 100644 --- a/pkg/cluster/slave_test.go +++ b/pkg/cluster/slave_test.go @@ -1,8 +1,12 @@ package cluster import ( + "bytes" + "encoding/json" + "errors" model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/pkg/cache" + "github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock" "github.com/cloudreve/Cloudreve/v3/pkg/request" "github.com/cloudreve/Cloudreve/v3/pkg/serializer" "github.com/stretchr/testify/assert" @@ -442,124 +446,114 @@ func TestSlaveCaller_DeleteTempFile(t *testing.T) { } } -//func TestRemoteCallback(t *testing.T) { -// asserts := assert.New(t) -// -// // 回调成功 -// { -// clientMock := request.ClientMock{} -// mockResp, _ := json.Marshal(serializer.Response{Code: 0}) -// clientMock.On( -// "Request", -// "POST", -// "http://test/test/url", -// testMock.Anything, -// testMock.Anything, -// ).Return(&request.Response{ -// Err: nil, -// Response: &http.Response{ -// StatusCode: 200, -// Body: ioutil.NopCloser(bytes.NewReader(mockResp)), -// }, -// }) -// request.GeneralClient = clientMock -// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{ -// SourceName: "source", -// }) -// asserts.NoError(resp) -// clientMock.AssertExpectations(t) -// } -// -// // 服务端返回业务错误 -// { -// clientMock := request.ClientMock{} -// mockResp, _ := json.Marshal(serializer.Response{Code: 401}) -// clientMock.On( -// "Request", -// "POST", -// "http://test/test/url", -// testMock.Anything, -// testMock.Anything, -// ).Return(&request.Response{ -// Err: nil, -// Response: &http.Response{ -// StatusCode: 200, -// Body: ioutil.NopCloser(bytes.NewReader(mockResp)), -// }, -// }) -// request.GeneralClient = clientMock -// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{ -// SourceName: "source", -// }) -// asserts.EqualValues(401, resp.(serializer.AppError).Code) -// clientMock.AssertExpectations(t) -// } -// -// // 无法解析回调响应 -// { -// clientMock := request.ClientMock{} -// clientMock.On( -// "Request", -// "POST", -// "http://test/test/url", -// testMock.Anything, -// testMock.Anything, -// ).Return(&request.Response{ -// Err: nil, -// Response: &http.Response{ -// StatusCode: 200, -// Body: ioutil.NopCloser(strings.NewReader("mockResp")), -// }, -// }) -// request.GeneralClient = clientMock -// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{ -// SourceName: "source", -// }) -// asserts.Error(resp) -// clientMock.AssertExpectations(t) -// } -// -// // HTTP状态码非200 -// { -// clientMock := request.ClientMock{} -// clientMock.On( -// "Request", -// "POST", -// "http://test/test/url", -// testMock.Anything, -// testMock.Anything, -// ).Return(&request.Response{ -// Err: nil, -// Response: &http.Response{ -// StatusCode: 404, -// Body: ioutil.NopCloser(strings.NewReader("mockResp")), -// }, -// }) -// request.GeneralClient = clientMock -// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{ -// SourceName: "source", -// }) -// asserts.Error(resp) -// clientMock.AssertExpectations(t) -// } -// -// // 无法发起回调 -// { -// clientMock := request.ClientMock{} -// clientMock.On( -// "Request", -// "POST", -// "http://test/test/url", -// testMock.Anything, -// testMock.Anything, -// ).Return(&request.Response{ -// Err: errors.New("error"), -// }) -// request.GeneralClient = clientMock -// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{ -// SourceName: "source", -// }) -// asserts.Error(resp) -// clientMock.AssertExpectations(t) -// } -//} +func TestRemoteCallback(t *testing.T) { + asserts := assert.New(t) + + // 回调成功 + { + clientMock := controllermock.RequestMock{} + mockResp, _ := json.Marshal(serializer.Response{Code: 0}) + clientMock.On( + "Request", + "POST", + "http://test/test/url", + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: nil, + Response: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewReader(mockResp)), + }, + }) + request.GeneralClient = clientMock + resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{}) + asserts.NoError(resp) + clientMock.AssertExpectations(t) + } + + // 服务端返回业务错误 + { + clientMock := controllermock.RequestMock{} + mockResp, _ := json.Marshal(serializer.Response{Code: 401}) + clientMock.On( + "Request", + "POST", + "http://test/test/url", + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: nil, + Response: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewReader(mockResp)), + }, + }) + request.GeneralClient = clientMock + resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{}) + asserts.EqualValues(401, resp.(serializer.AppError).Code) + clientMock.AssertExpectations(t) + } + + // 无法解析回调响应 + { + clientMock := controllermock.RequestMock{} + clientMock.On( + "Request", + "POST", + "http://test/test/url", + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: nil, + Response: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(strings.NewReader("mockResp")), + }, + }) + request.GeneralClient = clientMock + resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{}) + asserts.Error(resp) + clientMock.AssertExpectations(t) + } + + // HTTP状态码非200 + { + clientMock := controllermock.RequestMock{} + clientMock.On( + "Request", + "POST", + "http://test/test/url", + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: nil, + Response: &http.Response{ + StatusCode: 404, + Body: ioutil.NopCloser(strings.NewReader("mockResp")), + }, + }) + request.GeneralClient = clientMock + resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{}) + asserts.Error(resp) + clientMock.AssertExpectations(t) + } + + // 无法发起回调 + { + clientMock := controllermock.RequestMock{} + clientMock.On( + "Request", + "POST", + "http://test/test/url", + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: errors.New("error"), + }) + request.GeneralClient = clientMock + resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{}) + asserts.Error(resp) + clientMock.AssertExpectations(t) + } +} diff --git a/pkg/conf/conf_test.go b/pkg/conf/conf_test.go index aa95a7ef..6d186ed4 100644 --- a/pkg/conf/conf_test.go +++ b/pkg/conf/conf_test.go @@ -56,7 +56,11 @@ User = root Password = root Host = 127.0.0.1:3306 Name = v3 -TablePrefix = v3_` +TablePrefix = v3_ + +[OptionOverwrite] +key=value +` err := ioutil.WriteFile("testConf.ini", []byte(testCase), 0644) defer func() { err = os.Remove("testConf.ini") }() if err != nil { @@ -65,6 +69,7 @@ TablePrefix = v3_` asserts.NotPanics(func() { Init("testConf.ini") }) + asserts.Equal(OptionOverwrite["key"], "value") } func TestMapSection(t *testing.T) { diff --git a/pkg/mocks/mocks.go b/pkg/mocks/mocks.go index 2b085f16..01c450b8 100644 --- a/pkg/mocks/mocks.go +++ b/pkg/mocks/mocks.go @@ -7,11 +7,9 @@ import ( "github.com/cloudreve/Cloudreve/v3/pkg/auth" "github.com/cloudreve/Cloudreve/v3/pkg/balancer" "github.com/cloudreve/Cloudreve/v3/pkg/cluster" - "github.com/cloudreve/Cloudreve/v3/pkg/request" "github.com/cloudreve/Cloudreve/v3/pkg/serializer" "github.com/cloudreve/Cloudreve/v3/pkg/task" testMock "github.com/stretchr/testify/mock" - "io" ) type NodePoolMock struct { @@ -151,11 +149,3 @@ func (t TaskPoolMock) Add(num int) { func (t TaskPoolMock) Submit(job task.Job) { t.Called(job) } - -type RequestMock struct { - testMock.Mock -} - -func (r RequestMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response { - return r.Called(method, target, body, opts).Get(0).(*request.Response) -} diff --git a/pkg/mocks/requestmock/request.go b/pkg/mocks/requestmock/request.go new file mode 100644 index 00000000..41581b2e --- /dev/null +++ b/pkg/mocks/requestmock/request.go @@ -0,0 +1,15 @@ +package controllermock + +import ( + "github.com/cloudreve/Cloudreve/v3/pkg/request" + "github.com/stretchr/testify/mock" + "io" +) + +type RequestMock struct { + mock.Mock +} + +func (r RequestMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response { + return r.Called(method, target, body, opts).Get(0).(*request.Response) +} From 31315c86ee34d1bdc7e1990c5ddebb37181988bc Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Sat, 26 Mar 2022 15:33:31 +0800 Subject: [PATCH 17/21] Feat: support option for cache streamed chunk data into temp file for potential retry. --- models/defaults.go | 1 + pkg/filesystem/chunk/chunk.go | 72 +++++++++++++++++++++----- pkg/filesystem/driver/onedrive/api.go | 10 +++- pkg/filesystem/driver/oss/handler.go | 4 +- pkg/filesystem/driver/remote/client.go | 2 +- pkg/filesystem/driver/s3/handler.go | 2 +- 6 files changed, 73 insertions(+), 18 deletions(-) diff --git a/models/defaults.go b/models/defaults.go index bd7d4a4c..ecb84280 100644 --- a/models/defaults.go +++ b/models/defaults.go @@ -45,6 +45,7 @@ var defaultSettings = []Setting{ {Name: "chunk_retries", Value: `5`, Type: "retry"}, {Name: "onedrive_source_timeout", Value: `1800`, Type: "timeout"}, {Name: "reset_after_upload_failed", Value: `0`, Type: "upload"}, + {Name: "use_temp_chunk_buffer", Value: `1`, Type: "upload"}, {Name: "login_captcha", Value: `0`, Type: "login"}, {Name: "reg_captcha", Value: `0`, Type: "login"}, {Name: "email_active", Value: `0`, Type: "register"}, diff --git a/pkg/filesystem/chunk/chunk.go b/pkg/filesystem/chunk/chunk.go index 82e6cac7..5313558c 100644 --- a/pkg/filesystem/chunk/chunk.go +++ b/pkg/filesystem/chunk/chunk.go @@ -7,29 +7,35 @@ import ( "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" "github.com/cloudreve/Cloudreve/v3/pkg/util" "io" + "os" ) +const bufferTempPattern = "cdChunk.*.tmp" + // ChunkProcessFunc callback function for processing a chunk type ChunkProcessFunc func(c *ChunkGroup, chunk io.Reader) error // ChunkGroup manage groups of chunks type ChunkGroup struct { - file fsctx.FileHeader - chunkSize uint64 - backoff backoff.Backoff + file fsctx.FileHeader + chunkSize uint64 + backoff backoff.Backoff + enableRetryBuffer bool fileInfo *fsctx.UploadTaskInfo currentIndex int chunkNum uint64 + bufferTemp *os.File } -func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Backoff) *ChunkGroup { +func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Backoff, useBuffer bool) *ChunkGroup { c := &ChunkGroup{ - file: file, - chunkSize: chunkSize, - backoff: backoff, - fileInfo: file.Info(), - currentIndex: -1, + file: file, + chunkSize: chunkSize, + backoff: backoff, + fileInfo: file.Info(), + currentIndex: -1, + enableRetryBuffer: useBuffer, } if c.chunkSize == 0 { @@ -44,13 +50,53 @@ func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Back return c } +// TempAvailable returns if current chunk temp file is available to be read +func (c *ChunkGroup) TempAvailable() bool { + if c.bufferTemp != nil { + state, _ := c.bufferTemp.Stat() + return state != nil && state.Size() == c.Length() + } + + return false +} + // Process a chunk with retry logic func (c *ChunkGroup) Process(processor ChunkProcessFunc) error { - err := processor(c, io.LimitReader(c.file, int64(c.chunkSize))) + reader := io.LimitReader(c.file, int64(c.chunkSize)) + + // If useBuffer is enabled, tee the reader to a temp file + if c.enableRetryBuffer && c.bufferTemp == nil && !c.file.Seekable() { + c.bufferTemp, _ = os.CreateTemp("", bufferTempPattern) + reader = io.TeeReader(reader, c.bufferTemp) + } + + if c.bufferTemp != nil { + defer func() { + if c.bufferTemp != nil { + c.bufferTemp.Close() + os.Remove(c.bufferTemp.Name()) + c.bufferTemp = nil + } + }() + + // if temp buffer file is available, use it + if c.TempAvailable() { + if _, err := c.bufferTemp.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("failed to seek temp file back to chunk start: %w", err) + } + + util.Log().Debug("Chunk %d will be read from temp file %q.", c.Index(), c.bufferTemp.Name()) + reader = c.bufferTemp + } + } + + err := processor(c, reader) if err != nil { - if err != context.Canceled && c.file.Seekable() && c.backoff.Next() { - if _, seekErr := c.file.Seek(c.Start(), io.SeekStart); seekErr != nil { - return fmt.Errorf("failed to seek back to chunk start: %w, last error: %w", seekErr, err) + if err != context.Canceled && (c.file.Seekable() || c.TempAvailable()) && c.backoff.Next() { + if c.file.Seekable() { + if _, seekErr := c.file.Seek(c.Start(), io.SeekStart); seekErr != nil { + return fmt.Errorf("failed to seek back to chunk start: %w, last error: %w", seekErr, err) + } } util.Log().Debug("Retrying chunk %d, last error: %s", c.currentIndex, err) diff --git a/pkg/filesystem/driver/onedrive/api.go b/pkg/filesystem/driver/onedrive/api.go index 14c12789..27236262 100644 --- a/pkg/filesystem/driver/onedrive/api.go +++ b/pkg/filesystem/driver/onedrive/api.go @@ -221,8 +221,16 @@ func (client *Client) GetUploadSessionStatus(ctx context.Context, uploadURL stri return &uploadSession, nil } +var index = 0 + // UploadChunk 上传分片 func (client *Client) UploadChunk(ctx context.Context, uploadURL string, content io.Reader, current *chunk.ChunkGroup) (*UploadSessionResponse, error) { + index++ + if index == 1 || index == 2 { + request.BlackHole(content) + return nil, errors.New("error") + } + res, err := client.request( ctx, "PUT", uploadURL, content, request.WithContentLength(current.Length()), @@ -281,7 +289,7 @@ func (client *Client) Upload(ctx context.Context, file fsctx.FileHeader) error { chunks := chunk.NewChunkGroup(file, client.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{ Max: model.GetIntSetting("chunk_retries", 5), Sleep: chunkRetrySleep, - }) + }, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer"))) uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error { _, err := client.UploadChunk(ctx, uploadURL, content, current) diff --git a/pkg/filesystem/driver/oss/handler.go b/pkg/filesystem/driver/oss/handler.go index 2e156741..c7eadc23 100644 --- a/pkg/filesystem/driver/oss/handler.go +++ b/pkg/filesystem/driver/oss/handler.go @@ -252,7 +252,7 @@ func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error { chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{ Max: model.GetIntSetting("chunk_retries", 5), Sleep: chunkRetrySleep, - }) + }, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer"))) uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error { _, err := handler.bucket.UploadPart(imur, content, current.Length(), current.Index()+1) @@ -435,7 +435,7 @@ func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *seri uploadSession.UploadID = imur.UploadID // 为每个分片签名上传 URL - chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}) + chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false) urls := make([]string, chunks.Num()) for chunks.Next() { err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error { diff --git a/pkg/filesystem/driver/remote/client.go b/pkg/filesystem/driver/remote/client.go index b6759f1f..2f267eb7 100644 --- a/pkg/filesystem/driver/remote/client.go +++ b/pkg/filesystem/driver/remote/client.go @@ -92,7 +92,7 @@ func (c *remoteClient) Upload(ctx context.Context, file fsctx.FileHeader) error chunks := chunk.NewChunkGroup(file, c.policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{ Max: model.GetIntSetting("chunk_retries", 5), Sleep: chunkRetrySleep, - }) + }, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer"))) uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error { return c.uploadChunk(ctx, session.Key, current.Index(), content, overwrite, current.Length()) diff --git a/pkg/filesystem/driver/s3/handler.go b/pkg/filesystem/driver/s3/handler.go index a84178f4..ba9ce60a 100644 --- a/pkg/filesystem/driver/s3/handler.go +++ b/pkg/filesystem/driver/s3/handler.go @@ -342,7 +342,7 @@ func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *seri uploadSession.UploadID = *res.UploadId // 为每个分片签名上传 URL - chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}) + chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false) urls := make([]string, chunks.Num()) for chunks.Next() { err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error { From c6130ab078fc7f5a669ac46b777a943e90a9aa42 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Sun, 27 Mar 2022 11:14:30 +0800 Subject: [PATCH 18/21] Feat: new changes in pkg: chunk, backoff, local, onedrive --- pkg/filesystem/chunk/backoff/backoff_test.go | 22 ++ pkg/filesystem/chunk/chunk.go | 14 +- pkg/filesystem/chunk/chunk_test.go | 250 ++++++++++++++++++ pkg/filesystem/driver/local/handler.go | 2 +- pkg/filesystem/driver/local/handler_test.go | 115 ++++++-- pkg/filesystem/driver/onedrive/api.go | 18 -- pkg/filesystem/driver/onedrive/api_test.go | 136 +++++----- .../driver/onedrive/handler_test.go | 121 +++++---- .../driver/onedrive/handller_test.go | 38 --- pkg/filesystem/driver/onedrive/types.go | 14 - 10 files changed, 515 insertions(+), 215 deletions(-) create mode 100644 pkg/filesystem/chunk/backoff/backoff_test.go create mode 100644 pkg/filesystem/chunk/chunk_test.go delete mode 100644 pkg/filesystem/driver/onedrive/handller_test.go diff --git a/pkg/filesystem/chunk/backoff/backoff_test.go b/pkg/filesystem/chunk/backoff/backoff_test.go new file mode 100644 index 00000000..6419c715 --- /dev/null +++ b/pkg/filesystem/chunk/backoff/backoff_test.go @@ -0,0 +1,22 @@ +package backoff + +import ( + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestConstantBackoff_Next(t *testing.T) { + a := assert.New(t) + + b := &ConstantBackoff{Sleep: time.Duration(0), Max: 3} + a.True(b.Next()) + a.True(b.Next()) + a.True(b.Next()) + a.False(b.Next()) + b.Reset() + a.True(b.Next()) + a.True(b.Next()) + a.True(b.Next()) + a.False(b.Next()) +} diff --git a/pkg/filesystem/chunk/chunk.go b/pkg/filesystem/chunk/chunk.go index 5313558c..24e50a1c 100644 --- a/pkg/filesystem/chunk/chunk.go +++ b/pkg/filesystem/chunk/chunk.go @@ -42,9 +42,13 @@ func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Back c.chunkSize = c.fileInfo.Size } - c.chunkNum = c.fileInfo.Size / c.chunkSize - if c.fileInfo.Size%c.chunkSize != 0 || c.fileInfo.Size == 0 { - c.chunkNum++ + if c.fileInfo.Size == 0 { + c.chunkNum = 1 + } else { + c.chunkNum = c.fileInfo.Size / c.chunkSize + if c.fileInfo.Size%c.chunkSize != 0 { + c.chunkNum++ + } } return c @@ -95,7 +99,7 @@ func (c *ChunkGroup) Process(processor ChunkProcessFunc) error { if err != context.Canceled && (c.file.Seekable() || c.TempAvailable()) && c.backoff.Next() { if c.file.Seekable() { if _, seekErr := c.file.Seek(c.Start(), io.SeekStart); seekErr != nil { - return fmt.Errorf("failed to seek back to chunk start: %w, last error: %w", seekErr, err) + return fmt.Errorf("failed to seek back to chunk start: %w, last error: %s", seekErr, err) } } @@ -115,7 +119,7 @@ func (c *ChunkGroup) Start() int64 { return int64(uint64(c.Index()) * c.chunkSize) } -// Total returns the total length current chunk +// Total returns the total length func (c *ChunkGroup) Total() int64 { return int64(c.fileInfo.Size) } diff --git a/pkg/filesystem/chunk/chunk_test.go b/pkg/filesystem/chunk/chunk_test.go new file mode 100644 index 00000000..c6af9d93 --- /dev/null +++ b/pkg/filesystem/chunk/chunk_test.go @@ -0,0 +1,250 @@ +package chunk + +import ( + "errors" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" + "github.com/stretchr/testify/assert" + "io" + "os" + "strings" + "testing" +) + +func TestNewChunkGroup(t *testing.T) { + a := assert.New(t) + + testCases := []struct { + fileSize uint64 + chunkSize uint64 + expectedInnerChunkSize uint64 + expectedChunkNum uint64 + expectedInfo [][2]int //Start, Index,Length + }{ + {10, 0, 10, 1, [][2]int{{0, 10}}}, + {0, 0, 0, 1, [][2]int{{0, 0}}}, + {0, 10, 10, 1, [][2]int{{0, 0}}}, + {50, 10, 10, 5, [][2]int{ + {0, 10}, + {10, 10}, + {20, 10}, + {30, 10}, + {40, 10}, + }}, + {50, 50, 50, 1, [][2]int{ + {0, 50}, + }}, + + {50, 15, 15, 4, [][2]int{ + {0, 15}, + {15, 15}, + {30, 15}, + {45, 5}, + }}, + } + + for index, testCase := range testCases { + file := &fsctx.FileStream{Size: testCase.fileSize} + chunkGroup := NewChunkGroup(file, testCase.chunkSize, &backoff.ConstantBackoff{}, true) + a.EqualValues(testCase.expectedChunkNum, chunkGroup.Num(), + "TestCase:%d,ChunkNum()", index) + a.EqualValues(testCase.expectedInnerChunkSize, chunkGroup.chunkSize, + "TestCase:%d,InnerChunkSize()", index) + a.EqualValues(testCase.expectedChunkNum, chunkGroup.Num(), + "TestCase:%d,len(Chunks)", index) + a.EqualValues(testCase.fileSize, chunkGroup.Total()) + + for cIndex, info := range testCase.expectedInfo { + a.True(chunkGroup.Next()) + a.EqualValues(info[1], chunkGroup.Length(), + "TestCase:%d,Chunks[%d].Length()", index, cIndex) + a.EqualValues(info[0], chunkGroup.Start(), + "TestCase:%d,Chunks[%d].Start()", index, cIndex) + + a.Equal(cIndex == len(testCase.expectedInfo)-1, chunkGroup.IsLast(), + "TestCase:%d,Chunks[%d].IsLast()", index, cIndex) + + a.NotEmpty(chunkGroup.RangeHeader()) + } + a.False(chunkGroup.Next()) + } +} + +func TestChunkGroup_TempAvailablet(t *testing.T) { + a := assert.New(t) + + file := &fsctx.FileStream{Size: 1} + c := NewChunkGroup(file, 0, &backoff.ConstantBackoff{}, true) + a.False(c.TempAvailable()) + + f, err := os.CreateTemp("", "TestChunkGroup_TempAvailablet.*") + defer func() { + f.Close() + os.Remove(f.Name()) + }() + a.NoError(err) + c.bufferTemp = f + + a.False(c.TempAvailable()) + f.Write([]byte("1")) + a.True(c.TempAvailable()) + +} + +func TestChunkGroup_Process(t *testing.T) { + a := assert.New(t) + file := &fsctx.FileStream{Size: 10} + + // success + { + file.File = io.NopCloser(strings.NewReader("1234567890")) + c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{}, true) + count := 0 + a.True(c.Next()) + a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error { + count++ + res, err := io.ReadAll(chunk) + a.NoError(err) + a.EqualValues("12345", string(res)) + return nil + })) + a.True(c.Next()) + a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error { + count++ + res, err := io.ReadAll(chunk) + a.NoError(err) + a.EqualValues("67890", string(res)) + return nil + })) + a.False(c.Next()) + a.Equal(2, count) + } + + // retry, read from buffer file + { + file.File = io.NopCloser(strings.NewReader("1234567890")) + c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, true) + count := 0 + a.True(c.Next()) + a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error { + count++ + res, err := io.ReadAll(chunk) + a.NoError(err) + a.EqualValues("12345", string(res)) + return nil + })) + a.True(c.Next()) + a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error { + count++ + res, err := io.ReadAll(chunk) + a.NoError(err) + a.EqualValues("67890", string(res)) + if count == 2 { + return errors.New("error") + } + return nil + })) + a.False(c.Next()) + a.Equal(3, count) + } + + // retry, read from seeker + { + f, _ := os.CreateTemp("", "TestChunkGroup_Process.*") + f.Write([]byte("1234567890")) + f.Seek(0, 0) + defer func() { + f.Close() + os.Remove(f.Name()) + }() + file.File = f + file.Seeker = f + c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false) + count := 0 + a.True(c.Next()) + a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error { + count++ + res, err := io.ReadAll(chunk) + a.NoError(err) + a.EqualValues("12345", string(res)) + return nil + })) + a.True(c.Next()) + a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error { + count++ + res, err := io.ReadAll(chunk) + a.NoError(err) + a.EqualValues("67890", string(res)) + if count == 2 { + return errors.New("error") + } + return nil + })) + a.False(c.Next()) + a.Equal(3, count) + } + + // retry, seek error + { + f, _ := os.CreateTemp("", "TestChunkGroup_Process.*") + f.Write([]byte("1234567890")) + f.Seek(0, 0) + defer func() { + f.Close() + os.Remove(f.Name()) + }() + file.File = f + file.Seeker = f + c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false) + count := 0 + a.True(c.Next()) + a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error { + count++ + res, err := io.ReadAll(chunk) + a.NoError(err) + a.EqualValues("12345", string(res)) + return nil + })) + a.True(c.Next()) + f.Close() + a.Error(c.Process(func(c *ChunkGroup, chunk io.Reader) error { + count++ + if count == 2 { + return errors.New("error") + } + return nil + })) + a.False(c.Next()) + a.Equal(2, count) + } + + // retry, finally error + { + f, _ := os.CreateTemp("", "TestChunkGroup_Process.*") + f.Write([]byte("1234567890")) + f.Seek(0, 0) + defer func() { + f.Close() + os.Remove(f.Name()) + }() + file.File = f + file.Seeker = f + c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false) + count := 0 + a.True(c.Next()) + a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error { + count++ + res, err := io.ReadAll(chunk) + a.NoError(err) + a.EqualValues("12345", string(res)) + return nil + })) + a.True(c.Next()) + a.Error(c.Process(func(c *ChunkGroup, chunk io.Reader) error { + count++ + return errors.New("error") + })) + a.False(c.Next()) + a.Equal(1, count) + } +} diff --git a/pkg/filesystem/driver/local/handler.go b/pkg/filesystem/driver/local/handler.go index 7e665c89..e5e8994a 100644 --- a/pkg/filesystem/driver/local/handler.go +++ b/pkg/filesystem/driver/local/handler.go @@ -161,7 +161,7 @@ func (handler Driver) Truncate(ctx context.Context, src string, size uint64) err util.Log().Warning("截断文件 [%s] 至 [%d]", src, size) out, err := os.OpenFile(src, os.O_WRONLY, Perm) if err != nil { - util.Log().Warning("无法打开或创建文件，%s", err) + util.Log().Warning("无法打开文件，%s", err) return err } diff --git a/pkg/filesystem/driver/local/handler_test.go b/pkg/filesystem/driver/local/handler_test.go index dac4d54d..9ce5fe74 100644 --- a/pkg/filesystem/driver/local/handler_test.go +++ b/pkg/filesystem/driver/local/handler_test.go @@ -4,13 +4,12 @@ import ( "context" model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/pkg/auth" - "github.com/cloudreve/Cloudreve/v3/pkg/conf" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" + "github.com/cloudreve/Cloudreve/v3/pkg/serializer" "github.com/cloudreve/Cloudreve/v3/pkg/util" "github.com/jinzhu/gorm" "github.com/stretchr/testify/assert" "io" - "io/ioutil" "net/url" "os" "strings" @@ -20,42 +19,64 @@ import ( func TestHandler_Put(t *testing.T) { asserts := assert.New(t) handler := Driver{} - ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true) - os.Remove(util.RelativePath("test/test/txt")) + + defer func() { + os.Remove(util.RelativePath("TestHandler_Put.txt")) + os.Remove(util.RelativePath("inner/TestHandler_Put.txt")) + }() testCases := []struct { - file io.ReadCloser - dst string - err bool + file fsctx.FileHeader + errContains string }{ - { - file: ioutil.NopCloser(strings.NewReader("test input file")), - dst: "test/test/txt", - err: false, - }, - { - file: ioutil.NopCloser(strings.NewReader("test input file")), - dst: "test/test/txt", - err: true, - }, - { - file: ioutil.NopCloser(strings.NewReader("test input file")), - dst: "/notexist:/S.TXT", - err: true, - }, + {&fsctx.FileStream{ + SavePath: "TestHandler_Put.txt", + File: io.NopCloser(strings.NewReader("")), + }, ""}, + {&fsctx.FileStream{ + SavePath: "TestHandler_Put.txt", + File: io.NopCloser(strings.NewReader("")), + }, "物理同名文件已存在或不可用"}, + {&fsctx.FileStream{ + SavePath: "inner/TestHandler_Put.txt", + File: io.NopCloser(strings.NewReader("")), + }, ""}, + {&fsctx.FileStream{ + Mode: fsctx.Append | fsctx.Overwrite, + SavePath: "inner/TestHandler_Put.txt", + File: io.NopCloser(strings.NewReader("123")), + }, ""}, + {&fsctx.FileStream{ + AppendStart: 10, + Mode: fsctx.Append | fsctx.Overwrite, + SavePath: "inner/TestHandler_Put.txt", + File: io.NopCloser(strings.NewReader("123")), + }, "未上传完成的文件分片与预期大小不一致"}, + {&fsctx.FileStream{ + Mode: fsctx.Append | fsctx.Overwrite, + SavePath: "inner/TestHandler_Put.txt", + File: io.NopCloser(strings.NewReader("123")), + }, ""}, } for _, testCase := range testCases { - err := handler.Put(ctx, testCase.file, testCase.dst, 15) - if testCase.err { + err := handler.Put(context.Background(), testCase.file) + if testCase.errContains != "" { asserts.Error(err) + asserts.Contains(err.Error(), testCase.errContains) } else { asserts.NoError(err) - asserts.True(util.Exists(util.RelativePath(testCase.dst))) + asserts.True(util.Exists(util.RelativePath(testCase.file.Info().SavePath))) } } } +func TestDriver_TruncateFailed(t *testing.T) { + a := assert.New(t) + h := Driver{} + a.Error(h.Truncate(context.Background(), "TestDriver_TruncateFailed", 0)) +} + func TestHandler_Delete(t *testing.T) { asserts := assert.New(t) handler := Driver{} @@ -116,7 +137,7 @@ func TestHandler_Thumb(t *testing.T) { asserts := assert.New(t) handler := Driver{} ctx := context.Background() - file, err := os.Create(util.RelativePath("TestHandler_Thumb" + conf.ThumbConfig.FileSuffix)) + file, err := os.Create(util.RelativePath("TestHandler_Thumb._thumb")) asserts.NoError(err) file.Close() @@ -160,6 +181,25 @@ func TestHandler_Source(t *testing.T) { asserts.Contains(sourceURL, "https://cloudreve.org") } + // 下载 + { + file := model.File{ + Model: gorm.Model{ + ID: 1, + }, + Name: "test.jpg", + } + ctx := context.WithValue(ctx, fsctx.FileModelCtx, file) + baseURL, err := url.Parse("https://cloudreve.org") + asserts.NoError(err) + sourceURL, err := handler.Source(ctx, "", *baseURL, 0, true, 0) + asserts.NoError(err) + asserts.NotEmpty(sourceURL) + asserts.Contains(sourceURL, "sign=") + asserts.Contains(sourceURL, "download") + asserts.Contains(sourceURL, "https://cloudreve.org") + } + // 无法获取上下文 { baseURL, err := url.Parse("https://cloudreve.org") @@ -241,10 +281,29 @@ func TestHandler_GetDownloadURL(t *testing.T) { func TestHandler_Token(t *testing.T) { asserts := assert.New(t) - handler := Driver{} + handler := Driver{ + Policy: &model.Policy{}, + } ctx := context.Background() - _, err := handler.Token(ctx, 10, "123") + upSession := &serializer.UploadSession{SavePath: "TestHandler_Token"} + _, err := handler.Token(ctx, 10, upSession, &fsctx.FileStream{}) asserts.NoError(err) + + file, _ := os.Create("TestHandler_Token") + defer func() { + file.Close() + os.Remove("TestHandler_Token") + }() + + _, err = handler.Token(ctx, 10, upSession, &fsctx.FileStream{}) + asserts.Error(err) + asserts.Contains(err.Error(), "already exist") +} + +func TestDriver_CancelToken(t *testing.T) { + a := assert.New(t) + handler := Driver{} + a.NoError(handler.CancelToken(context.Background(), &serializer.UploadSession{})) } func TestDriver_List(t *testing.T) { diff --git a/pkg/filesystem/driver/onedrive/api.go b/pkg/filesystem/driver/onedrive/api.go index 27236262..5dc2ec99 100644 --- a/pkg/filesystem/driver/onedrive/api.go +++ b/pkg/filesystem/driver/onedrive/api.go @@ -221,16 +221,8 @@ func (client *Client) GetUploadSessionStatus(ctx context.Context, uploadURL stri return &uploadSession, nil } -var index = 0 - // UploadChunk 上传分片 func (client *Client) UploadChunk(ctx context.Context, uploadURL string, content io.Reader, current *chunk.ChunkGroup) (*UploadSessionResponse, error) { - index++ - if index == 1 || index == 2 { - request.BlackHole(content) - return nil, errors.New("error") - } - res, err := client.request( ctx, "PUT", uploadURL, content, request.WithContentLength(current.Length()), @@ -331,16 +323,6 @@ func (client *Client) SimpleUpload(ctx context.Context, dst string, body io.Read request.WithTimeout(time.Duration(150)*time.Second), ) if err != nil { - retried := 0 - if v, ok := ctx.Value(fsctx.RetryCtx).(int); ok { - retried = v - } - if retried < model.GetIntSetting("chunk_retries", 5) { - retried++ - util.Log().Debug("文件[%s]上传失败[%s]，5秒钟后重试", dst, err) - time.Sleep(time.Duration(5) * time.Second) - return client.SimpleUpload(context.WithValue(ctx, fsctx.RetryCtx, retried), dst, body, size, opts...) - } return nil, err } diff --git a/pkg/filesystem/driver/onedrive/api_test.go b/pkg/filesystem/driver/onedrive/api_test.go index 8acc6dbe..fb6393db 100644 --- a/pkg/filesystem/driver/onedrive/api_test.go +++ b/pkg/filesystem/driver/onedrive/api_test.go @@ -4,6 +4,11 @@ import ( "context" "errors" "fmt" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" + "github.com/cloudreve/Cloudreve/v3/pkg/mq" + "io" "io/ioutil" "net/http" "strings" @@ -12,7 +17,6 @@ import ( model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/pkg/cache" - "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" "github.com/cloudreve/Cloudreve/v3/pkg/request" "github.com/stretchr/testify/assert" testMock "github.com/stretchr/testify/mock" @@ -307,6 +311,31 @@ func TestClient_Meta(t *testing.T) { asserts.NotNil(res) asserts.Equal("123321", res.Name) } + + // 返回正常, 使用资源id + { + client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix() + clientMock := ClientMock{} + clientMock.On( + "Request", + "GET", + testMock.Anything, + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: nil, + Response: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(strings.NewReader(`{"name":"123321"}`)), + }, + }) + client.Request = clientMock + res, err := client.Meta(context.Background(), "123321", "123") + clientMock.AssertExpectations(t) + asserts.NoError(err) + asserts.NotNil(res) + asserts.Equal("123321", res.Name) + } } func TestClient_CreateUploadSession(t *testing.T) { @@ -442,9 +471,11 @@ func TestClient_UploadChunk(t *testing.T) { client, _ := NewClient(&model.Policy{}) client.Credential.AccessToken = "AccessToken" client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix() + cg := chunk.NewChunkGroup(&fsctx.FileStream{Size: 15}, 10, &backoff.ConstantBackoff{}, false) // 非最后分片，正常 { + cg.Next() client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix() clientMock := ClientMock{} clientMock.On( @@ -453,6 +484,10 @@ func TestClient_UploadChunk(t *testing.T) { "http://dev.com", testMock.Anything, testMock.Anything, + testMock.Anything, + testMock.Anything, + testMock.Anything, + testMock.Anything, ).Return(&request.Response{ Err: nil, Response: &http.Response{ @@ -461,13 +496,7 @@ func TestClient_UploadChunk(t *testing.T) { }, }) client.Request = clientMock - res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{ - Offset: 0, - ChunkSize: 10, - Total: 100, - Retried: 0, - Data: []byte("12313121231312"), - }) + res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("1234567890"), cg) clientMock.AssertExpectations(t) asserts.NoError(err) asserts.Equal("http://dev.com/2", res.UploadURL) @@ -491,13 +520,7 @@ func TestClient_UploadChunk(t *testing.T) { }, }) client.Request = clientMock - res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{ - Offset: 0, - ChunkSize: 10, - Total: 100, - Retried: 0, - Data: []byte("12313112313122"), - }) + res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("1234567890"), cg) clientMock.AssertExpectations(t) asserts.Error(err) asserts.Nil(res) @@ -505,6 +528,7 @@ func TestClient_UploadChunk(t *testing.T) { // 最后分片，正常 { + cg.Next() client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix() clientMock := ClientMock{} clientMock.On( @@ -521,19 +545,13 @@ func TestClient_UploadChunk(t *testing.T) { }, }) client.Request = clientMock - res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{ - Offset: 95, - ChunkSize: 5, - Total: 100, - Retried: 0, - Data: []byte("1231312"), - }) + res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("12345"), cg) clientMock.AssertExpectations(t) asserts.NoError(err) asserts.Nil(res) } - // 最后分片，第一次失败，重试后成功 + // 最后分片，失败 { cache.Set("setting_chunk_retries", "1", 0) client.Credential.ExpiresIn = 0 @@ -542,32 +560,11 @@ func TestClient_UploadChunk(t *testing.T) { client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix() }() clientMock := ClientMock{} - clientMock.On( - "Request", - "PUT", - "http://dev.com", - testMock.Anything, - testMock.Anything, - ).Return(&request.Response{ - Err: nil, - Response: &http.Response{ - StatusCode: 200, - Body: ioutil.NopCloser(strings.NewReader(`???`)), - }, - }) client.Request = clientMock - chunk := &Chunk{ - Offset: 95, - ChunkSize: 5, - Total: 100, - Retried: 0, - Data: []byte("1231312"), - } - res, err := client.UploadChunk(context.Background(), "http://dev.com", chunk) + res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("12345"), cg) clientMock.AssertExpectations(t) - asserts.NoError(err) + asserts.Error(err) asserts.Nil(res) - asserts.EqualValues(1, chunk.Retried) } } @@ -576,16 +573,21 @@ func TestClient_Upload(t *testing.T) { client, _ := NewClient(&model.Policy{}) client.Credential.AccessToken = "AccessToken" client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix() - ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true) + ctx := context.Background() + cache.Set("setting_chunk_retries", "1", 0) + cache.Set("setting_use_temp_chunk_buffer", "false", 0) // 小文件，简单上传，失败 { client.Credential.ExpiresIn = 0 - err := client.Upload(ctx, "123.jpg", 3, strings.NewReader("123")) + err := client.Upload(ctx, &fsctx.FileStream{ + Size: 5, + File: io.NopCloser(strings.NewReader("12345")), + }) asserts.Error(err) } - // 上下文取消 + // 无法创建分片会话 { client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix() clientMock := ClientMock{} @@ -598,20 +600,20 @@ func TestClient_Upload(t *testing.T) { ).Return(&request.Response{ Err: nil, Response: &http.Response{ - StatusCode: 200, + StatusCode: 400, Body: ioutil.NopCloser(strings.NewReader(`{"uploadUrl":"123321"}`)), }, }) client.Request = clientMock - ctx, cancel := context.WithCancel(context.Background()) - cancel() - err := client.Upload(ctx, "123.jpg", 15*1024*1024, strings.NewReader("123")) + err := client.Upload(context.Background(), &fsctx.FileStream{ + Size: SmallFileSize + 1, + File: io.NopCloser(strings.NewReader("12345")), + }) clientMock.AssertExpectations(t) asserts.Error(err) - asserts.Equal(ErrClientCanceled, err) } - // 无法创建分片会话 + // 分片上传失败 { client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix() clientMock := ClientMock{} @@ -621,6 +623,19 @@ func TestClient_Upload(t *testing.T) { testMock.Anything, testMock.Anything, testMock.Anything, + ).Return(&request.Response{ + Err: nil, + Response: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(strings.NewReader(`{"uploadUrl":"123321"}`)), + }, + }) + clientMock.On( + "Request", + "PUT", + testMock.Anything, + testMock.Anything, + testMock.Anything, ).Return(&request.Response{ Err: nil, Response: &http.Response{ @@ -629,9 +644,13 @@ func TestClient_Upload(t *testing.T) { }, }) client.Request = clientMock - err := client.Upload(context.Background(), "123.jpg", 15*1024*1024, strings.NewReader("123")) + err := client.Upload(context.Background(), &fsctx.FileStream{ + Size: SmallFileSize + 1, + File: io.NopCloser(strings.NewReader("12345")), + }) clientMock.AssertExpectations(t) asserts.Error(err) + asserts.Contains(err.Error(), "failed to upload chunk") } } @@ -643,7 +662,7 @@ func TestClient_SimpleUpload(t *testing.T) { client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix() cache.Set("setting_chunk_retries", "1", 0) - // 请求失败，并重试 + // 请求失败 { client.Credential.ExpiresIn = 0 res, err := client.SimpleUpload(context.Background(), "123.jpg", strings.NewReader("123"), 3) @@ -651,7 +670,6 @@ func TestClient_SimpleUpload(t *testing.T) { asserts.Nil(res) } - cache.Set("setting_chunk_retries", "0", 0) // 返回未知响应 { client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix() @@ -988,7 +1006,7 @@ func TestClient_MonitorUpload(t *testing.T) { asserts.NotPanics(func() { go func() { time.Sleep(time.Duration(1) * time.Second) - FinishCallback("key") + mq.GlobalMQ.Publish("key", mq.Message{}) }() client.MonitorUpload("url", "key", "path", 10, 10) }) diff --git a/pkg/filesystem/driver/onedrive/handler_test.go b/pkg/filesystem/driver/onedrive/handler_test.go index c2b00ae1..7700e7af 100644 --- a/pkg/filesystem/driver/onedrive/handler_test.go +++ b/pkg/filesystem/driver/onedrive/handler_test.go @@ -4,6 +4,9 @@ import ( "context" "fmt" "github.com/cloudreve/Cloudreve/v3/pkg/auth" + "github.com/cloudreve/Cloudreve/v3/pkg/mq" + "github.com/cloudreve/Cloudreve/v3/pkg/serializer" + "github.com/jinzhu/gorm" "io" "io/ioutil" "net/http" @@ -12,51 +15,23 @@ import ( "testing" "time" - "github.com/DATA-DOG/go-sqlmock" model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/pkg/cache" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" "github.com/cloudreve/Cloudreve/v3/pkg/request" - "github.com/cloudreve/Cloudreve/v3/pkg/serializer" "github.com/stretchr/testify/assert" testMock "github.com/stretchr/testify/mock" ) func TestDriver_Token(t *testing.T) { asserts := assert.New(t) - handler := Driver{ - Policy: &model.Policy{ - AccessKey: "ak", - SecretKey: "sk", - BucketName: "test", - Server: "test.com", - }, - } - - // 无法获取文件路径 - { - ctx := context.WithValue(context.Background(), fsctx.FileSizeCtx, uint64(10)) - res, err := handler.Token(ctx, 10, "key", nil) - asserts.Error(err) - asserts.Equal(serializer.UploadCredential{}, res) - } - - // 无法获取文件大小 - { - ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123") - res, err := handler.Token(ctx, 10, "key", nil) - asserts.Error(err) - asserts.Equal(serializer.UploadCredential{}, res) - } - - // 小文件成功 - { - ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123") - ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(10)) - res, err := handler.Token(ctx, 10, "key", nil) - asserts.NoError(err) - asserts.Equal(serializer.UploadCredential{}, res) - } + h, _ := NewDriver(&model.Policy{ + AccessKey: "ak", + SecretKey: "sk", + BucketName: "test", + Server: "test.com", + }) + handler := h.(Driver) // 分片上传 失败 { @@ -78,11 +53,9 @@ func TestDriver_Token(t *testing.T) { }, }) handler.Client.Request = clientMock - ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123") - ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(20*1024*1024)) - res, err := handler.Token(ctx, 10, "key", nil) + res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{}) asserts.Error(err) - asserts.Equal(serializer.UploadCredential{}, res) + asserts.Nil(res) } // 分片上传 成功 @@ -108,15 +81,13 @@ func TestDriver_Token(t *testing.T) { }, }) handler.Client.Request = clientMock - ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123") - ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(20*1024*1024)) go func() { time.Sleep(time.Duration(1) * time.Second) - FinishCallback("key") + mq.GlobalMQ.Publish("TestDriver_Token", mq.Message{}) }() - res, err := handler.Token(ctx, 10, "key", nil) + res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{Key: "TestDriver_Token"}, &fsctx.FileStream{}) asserts.NoError(err) - asserts.Equal("123321", res.Policy) + asserts.Equal("123321", res.UploadURLs[0]) } } @@ -295,12 +266,8 @@ func TestDriver_Thumb(t *testing.T) { // 失败 { ctx := context.WithValue(context.Background(), fsctx.ThumbSizeCtx, [2]uint{10, 20}) - ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{}) - mock.ExpectBegin() - mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() + ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{PicInfo: "1,1", Model: gorm.Model{ID: 1}}) res, err := handler.Thumb(ctx, "123.jpg") - asserts.NoError(mock.ExpectationsWereMet()) asserts.Error(err) asserts.Empty(res.URL) } @@ -308,7 +275,6 @@ func TestDriver_Thumb(t *testing.T) { // 上下文错误 { _, err := handler.Thumb(context.Background(), "123.jpg") - asserts.NoError(mock.ExpectationsWereMet()) asserts.Error(err) } } @@ -329,7 +295,6 @@ func TestDriver_Delete(t *testing.T) { // 失败 { _, err := handler.Delete(context.Background(), []string{"1"}) - asserts.NoError(mock.ExpectationsWereMet()) asserts.Error(err) } @@ -350,7 +315,7 @@ func TestDriver_Put(t *testing.T) { // 失败 { - err := handler.Put(context.Background(), ioutil.NopCloser(strings.NewReader("")), "dst", 0) + err := handler.Put(context.Background(), &fsctx.FileStream{}) asserts.Error(err) } } @@ -418,3 +383,55 @@ func TestDriver_Get(t *testing.T) { asserts.NoError(err) asserts.Equal("123", string(content)) } + +func TestDriver_replaceSourceHost(t *testing.T) { + tests := []struct { + name string + origin string + cdn string + want string + wantErr bool + }{ + {"TestNoReplace", "http://1dr.ms/download.aspx?123456", "", "http://1dr.ms/download.aspx?123456", false}, + {"TestReplaceCorrect", "http://1dr.ms/download.aspx?123456", "https://test.com:8080", "https://test.com:8080/download.aspx?123456", false}, + {"TestCdnFormatError", "http://1dr.ms/download.aspx?123456", string([]byte{0x7f}), "", true}, + {"TestSrcFormatError", string([]byte{0x7f}), "https://test.com:8080", "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + policy := &model.Policy{} + policy.OptionsSerialized.OdProxy = tt.cdn + handler := Driver{ + Policy: policy, + } + got, err := handler.replaceSourceHost(tt.origin) + if (err != nil) != tt.wantErr { + t.Errorf("replaceSourceHost() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("replaceSourceHost() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDriver_CancelToken(t *testing.T) { + asserts := assert.New(t) + handler := Driver{ + Policy: &model.Policy{ + AccessKey: "ak", + SecretKey: "sk", + BucketName: "test", + Server: "test.com", + }, + } + handler.Client, _ = NewClient(&model.Policy{}) + handler.Client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix() + + // 失败 + { + err := handler.CancelToken(context.Background(), &serializer.UploadSession{}) + asserts.Error(err) + } +} diff --git a/pkg/filesystem/driver/onedrive/handller_test.go b/pkg/filesystem/driver/onedrive/handller_test.go deleted file mode 100644 index 147a5477..00000000 --- a/pkg/filesystem/driver/onedrive/handller_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package onedrive - -import ( - model "github.com/cloudreve/Cloudreve/v3/models" - "testing" -) - -func TestDriver_replaceSourceHost(t *testing.T) { - tests := []struct { - name string - origin string - cdn string - want string - wantErr bool - }{ - {"TestNoReplace", "http://1dr.ms/download.aspx?123456", "", "http://1dr.ms/download.aspx?123456", false}, - {"TestReplaceCorrect", "http://1dr.ms/download.aspx?123456", "https://test.com:8080", "https://test.com:8080/download.aspx?123456", false}, - {"TestCdnFormatError", "http://1dr.ms/download.aspx?123456", string([]byte{0x7f}), "", true}, - {"TestSrcFormatError", string([]byte{0x7f}), "https://test.com:8080", "", true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - policy := &model.Policy{} - policy.OptionsSerialized.OdProxy = tt.cdn - handler := Driver{ - Policy: policy, - } - got, err := handler.replaceSourceHost(tt.origin) - if (err != nil) != tt.wantErr { - t.Errorf("replaceSourceHost() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("replaceSourceHost() got = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/filesystem/driver/onedrive/types.go b/pkg/filesystem/driver/onedrive/types.go index aefa6385..2a4307f2 100644 --- a/pkg/filesystem/driver/onedrive/types.go +++ b/pkg/filesystem/driver/onedrive/types.go @@ -98,15 +98,6 @@ type ListResponse struct { Context string `json:"@odata.context"` } -// Chunk 文件分片 -type Chunk struct { - Offset int - ChunkSize int - Total int - Retried int - Data []byte -} - // oauthEndpoint OAuth接口地址 type oauthEndpoint struct { token url.URL @@ -142,8 +133,3 @@ type Site struct { func init() { gob.Register(Credential{}) } - -// IsLast 返回是否为最后一个分片 -func (chunk *Chunk) IsLast() bool { - return chunk.Total-chunk.Offset == chunk.ChunkSize -} From 1c0a735df8adeaf63b9b423a2e3a42cfbb5d4b4b Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Tue, 29 Mar 2022 20:13:05 +0800 Subject: [PATCH 19/21] Test: new changes pkg remote, fsctx, part of filesystem --- pkg/cluster/slave_test.go | 10 +- pkg/filesystem/archive_test.go | 6 +- pkg/filesystem/driver/oss/callback_test.go | 188 ------- pkg/filesystem/driver/oss/handler_test.go | 354 ------------ pkg/filesystem/driver/remote/client_test.go | 262 +++++++++ pkg/filesystem/driver/remote/handler_test.go | 179 +++---- pkg/filesystem/file.go | 19 +- pkg/filesystem/file_test.go | 61 ++- pkg/filesystem/filesystem.go | 9 +- pkg/filesystem/filesystem_test.go | 45 +- pkg/filesystem/fsctx/stream_test.go | 74 ++- pkg/filesystem/fsctx/taskinfo/taskinfo.go | 1 - pkg/filesystem/hooks.go | 1 - pkg/filesystem/hooks_test.go | 537 +++++++++---------- pkg/filesystem/image_test.go | 59 +- pkg/filesystem/manage_test.go | 92 ++-- pkg/filesystem/upload_test.go | 388 +++++++------- pkg/mocks/remoteclientmock/mock.go | 32 ++ pkg/mocks/requestmock/request.go | 2 +- 19 files changed, 1022 insertions(+), 1297 deletions(-) delete mode 100644 pkg/filesystem/driver/oss/callback_test.go delete mode 100644 pkg/filesystem/driver/oss/handler_test.go create mode 100644 pkg/filesystem/driver/remote/client_test.go delete mode 100644 pkg/filesystem/fsctx/taskinfo/taskinfo.go create mode 100644 pkg/mocks/remoteclientmock/mock.go diff --git a/pkg/cluster/slave_test.go b/pkg/cluster/slave_test.go index 47f4bf2b..1b1510f6 100644 --- a/pkg/cluster/slave_test.go +++ b/pkg/cluster/slave_test.go @@ -451,7 +451,7 @@ func TestRemoteCallback(t *testing.T) { // 回调成功 { - clientMock := controllermock.RequestMock{} + clientMock := requestmock.RequestMock{} mockResp, _ := json.Marshal(serializer.Response{Code: 0}) clientMock.On( "Request", @@ -474,7 +474,7 @@ func TestRemoteCallback(t *testing.T) { // 服务端返回业务错误 { - clientMock := controllermock.RequestMock{} + clientMock := requestmock.RequestMock{} mockResp, _ := json.Marshal(serializer.Response{Code: 401}) clientMock.On( "Request", @@ -497,7 +497,7 @@ func TestRemoteCallback(t *testing.T) { // 无法解析回调响应 { - clientMock := controllermock.RequestMock{} + clientMock := requestmock.RequestMock{} clientMock.On( "Request", "POST", @@ -519,7 +519,7 @@ func TestRemoteCallback(t *testing.T) { // HTTP状态码非200 { - clientMock := controllermock.RequestMock{} + clientMock := requestmock.RequestMock{} clientMock.On( "Request", "POST", @@ -541,7 +541,7 @@ func TestRemoteCallback(t *testing.T) { // 无法发起回调 { - clientMock := controllermock.RequestMock{} + clientMock := requestmock.RequestMock{} clientMock.On( "Request", "POST", diff --git a/pkg/filesystem/archive_test.go b/pkg/filesystem/archive_test.go index 7a697ce7..be65b29a 100644 --- a/pkg/filesystem/archive_test.go +++ b/pkg/filesystem/archive_test.go @@ -3,6 +3,9 @@ package filesystem import ( "context" "errors" + "github.com/cloudreve/Cloudreve/v3/pkg/request" + "github.com/cloudreve/Cloudreve/v3/pkg/util" + testMock "github.com/stretchr/testify/mock" "io" "os" "strings" @@ -12,11 +15,8 @@ import ( model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/pkg/cache" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" - "github.com/cloudreve/Cloudreve/v3/pkg/request" - "github.com/cloudreve/Cloudreve/v3/pkg/util" "github.com/jinzhu/gorm" "github.com/stretchr/testify/assert" - testMock "github.com/stretchr/testify/mock" ) func TestFileSystem_Compress(t *testing.T) { diff --git a/pkg/filesystem/driver/oss/callback_test.go b/pkg/filesystem/driver/oss/callback_test.go deleted file mode 100644 index 1d293414..00000000 --- a/pkg/filesystem/driver/oss/callback_test.go +++ /dev/null @@ -1,188 +0,0 @@ -package oss - -import ( - "io/ioutil" - "net/http" - "net/url" - "strings" - "testing" - - "github.com/cloudreve/Cloudreve/v3/pkg/cache" - "github.com/stretchr/testify/assert" -) - -func TestGetPublicKey(t *testing.T) { - asserts := assert.New(t) - testCases := []struct { - Request http.Request - ResNil bool - Error bool - }{ - // Header解码失败 - { - Request: http.Request{ - Header: http.Header{ - "X-Oss-Pub-Key-Url": {"中文"}, - }, - }, - ResNil: true, - Error: true, - }, - // 公钥URL无效 - { - Request: http.Request{ - Header: http.Header{ - "X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9wb3JuaHViLmNvbQ=="}, - }, - }, - ResNil: true, - Error: true, - }, - // 请求失败 - { - Request: http.Request{ - Header: http.Header{ - "X-Oss-Pub-Key-Url": {"aHR0cDovL2dvc3NwdWJsaWMuYWxpY2RuLmNvbS8yMzQyMzQ="}, - }, - }, - ResNil: true, - Error: true, - }, - // 成功 - { - Request: http.Request{ - Header: http.Header{ - "X-Oss-Pub-Key-Url": {"aHR0cDovL2dvc3NwdWJsaWMuYWxpY2RuLmNvbS9jYWxsYmFja19wdWJfa2V5X3YxLnBlbQ=="}, - }, - }, - ResNil: false, - Error: false, - }, - } - - for i, testCase := range testCases { - asserts.NoError(cache.Deletes([]string{"oss_public_key"}, "")) - res, err := GetPublicKey(&testCase.Request) - if testCase.Error { - asserts.Error(err, "Test Case #%d", i) - } else { - asserts.NoError(err, "Test Case #%d", i) - } - if testCase.ResNil { - asserts.Empty(res, "Test Case #%d", i) - } else { - asserts.NotEmpty(res, "Test Case #%d", i) - } - } - - // 测试缓存 - asserts.NoError(cache.Set("oss_public_key", []byte("123"), 0)) - res, err := GetPublicKey(nil) - asserts.NoError(err) - asserts.Equal([]byte("123"), res) -} - -func TestVerifyCallbackSignature(t *testing.T) { - asserts := assert.New(t) - testPubKey := `-----BEGIN PUBLIC KEY----- -MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKs/JBGzwUB2aVht4crBx3oIPBLNsjGs -C0fTXv+nvlmklvkcolvpvXLTjaxUHR3W9LXxQ2EHXAJfCB+6H2YF1k8CAwEAAQ== ------END PUBLIC KEY----- -` - - // 成功 - { - asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0)) - r := http.Request{ - URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"}, - Header: map[string][]string{ - "Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="}, - "X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="}, - }, - Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)), - } - asserts.NoError(VerifyCallbackSignature(&r)) - } - - // 签名错误 - { - asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0)) - r := http.Request{ - URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"}, - Header: map[string][]string{ - "Authorization": {"e3LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="}, - "X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="}, - }, - Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)), - } - asserts.Error(VerifyCallbackSignature(&r)) - } - - // GetPubKey 失败 - { - asserts.NoError(cache.Deletes([]string{"oss_public_key"}, "")) - r := http.Request{ - URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"}, - Header: map[string][]string{ - "Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="}, - }, - Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)), - } - asserts.Error(VerifyCallbackSignature(&r)) - } - - // getRequestMD5 失败 - { - asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0)) - r := http.Request{ - URL: &url.URL{Path: "%测试"}, - Header: map[string][]string{ - "Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="}, - "X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="}, - }, - Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)), - } - asserts.Error(VerifyCallbackSignature(&r)) - } - - // 无 Authorization 头 - { - asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0)) - r := http.Request{ - URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"}, - Header: map[string][]string{ - "X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="}, - }, - Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)), - } - asserts.Error(VerifyCallbackSignature(&r)) - } - - // pub block 不存在 - { - asserts.NoError(cache.Set("oss_public_key", []byte(""), 0)) - r := http.Request{ - URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"}, - Header: map[string][]string{ - "Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="}, - "X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="}, - }, - Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)), - } - asserts.Error(VerifyCallbackSignature(&r)) - } - - // ParsePKIXPublicKey出错 - { - asserts.NoError(cache.Set("oss_public_key", []byte("-----BEGIN PUBLIC KEY-----\n-----END PUBLIC KEY-----"), 0)) - r := http.Request{ - URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"}, - Header: map[string][]string{ - "Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="}, - "X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="}, - }, - Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)), - } - asserts.Error(VerifyCallbackSignature(&r)) - } -} diff --git a/pkg/filesystem/driver/oss/handler_test.go b/pkg/filesystem/driver/oss/handler_test.go deleted file mode 100644 index d69c9312..00000000 --- a/pkg/filesystem/driver/oss/handler_test.go +++ /dev/null @@ -1,354 +0,0 @@ -package oss - -import ( - "context" - "io" - "io/ioutil" - "net/http" - "net/url" - "strings" - "testing" - - model "github.com/cloudreve/Cloudreve/v3/models" - "github.com/cloudreve/Cloudreve/v3/pkg/cache" - "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" - "github.com/cloudreve/Cloudreve/v3/pkg/request" - "github.com/stretchr/testify/assert" - testMock "github.com/stretchr/testify/mock" -) - -func TestDriver_InitOSSClient(t *testing.T) { - asserts := assert.New(t) - handler := Driver{ - Policy: &model.Policy{ - AccessKey: "ak", - SecretKey: "sk", - BucketName: "test", - Server: "test.com", - }, - } - - // 成功 - { - asserts.NoError(handler.InitOSSClient(false)) - } - - // 使用内网Endpoint - { - handler.Policy.OptionsSerialized.ServerSideEndpoint = "endpoint2" - asserts.NoError(handler.InitOSSClient(false)) - } - - // 未指定存储策略 - { - handler := Driver{} - asserts.Error(handler.InitOSSClient(false)) - } -} - -func TestDriver_CORS(t *testing.T) { - asserts := assert.New(t) - handler := Driver{ - Policy: &model.Policy{ - AccessKey: "ak", - SecretKey: "sk", - BucketName: "test", - Server: "test.com", - }, - } - - // 失败 - { - asserts.NotPanics(func() { - handler.CORS() - }) - } -} - -func TestDriver_Token(t *testing.T) { - asserts := assert.New(t) - handler := Driver{ - Policy: &model.Policy{ - AccessKey: "ak", - SecretKey: "sk", - BucketName: "test", - Server: "test.com", - }, - } - - // 成功 - { - ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123") - cache.Set("setting_siteURL", "http://test.cloudreve.org", 0) - res, err := handler.Token(ctx, 10, "key", nil) - asserts.NoError(err) - asserts.NotEmpty(res.Policy) - asserts.NotEmpty(res.Token) - asserts.Equal(handler.Policy.AccessKey, res.AccessKey) - asserts.Equal("/123", res.Path) - } - - // 上下文错误 - { - ctx := context.Background() - _, err := handler.Token(ctx, 10, "key", nil) - asserts.Error(err) - } - -} - -func TestDriver_Source(t *testing.T) { - asserts := assert.New(t) - handler := Driver{ - Policy: &model.Policy{ - AccessKey: "ak", - SecretKey: "sk", - BucketName: "test", - Server: "test.com", - IsPrivate: true, - }, - } - - // 正常 非下载 无限速 - { - res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0) - asserts.NoError(err) - resURL, err := url.Parse(res) - asserts.NoError(err) - query := resURL.Query() - asserts.NotEmpty(query.Get("Signature")) - asserts.NotEmpty(query.Get("Expires")) - asserts.Equal("ak", query.Get("OSSAccessKeyId")) - } - - // 限速 + 下载 - { - ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"}) - res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 102401) - asserts.NoError(err) - resURL, err := url.Parse(res) - asserts.NoError(err) - query := resURL.Query() - asserts.NotEmpty(query.Get("Signature")) - asserts.NotEmpty(query.Get("Expires")) - asserts.Equal("ak", query.Get("OSSAccessKeyId")) - asserts.EqualValues("819208", query.Get("x-oss-traffic-limit")) - asserts.NotEmpty(query.Get("response-content-disposition")) - } - - // 限速超出范围 + 下载 - { - ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"}) - res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 10) - asserts.NoError(err) - resURL, err := url.Parse(res) - asserts.NoError(err) - query := resURL.Query() - asserts.NotEmpty(query.Get("Signature")) - asserts.NotEmpty(query.Get("Expires")) - asserts.Equal("ak", query.Get("OSSAccessKeyId")) - asserts.EqualValues("819200", query.Get("x-oss-traffic-limit")) - asserts.NotEmpty(query.Get("response-content-disposition")) - } - - // 限速超出范围 + 下载 - { - ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"}) - res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 838860801) - asserts.NoError(err) - resURL, err := url.Parse(res) - asserts.NoError(err) - query := resURL.Query() - asserts.NotEmpty(query.Get("Signature")) - asserts.NotEmpty(query.Get("Expires")) - asserts.Equal("ak", query.Get("OSSAccessKeyId")) - asserts.EqualValues("838860800", query.Get("x-oss-traffic-limit")) - asserts.NotEmpty(query.Get("response-content-disposition")) - } - - // 公共空间 - { - handler.Policy.IsPrivate = false - res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0) - asserts.NoError(err) - resURL, err := url.Parse(res) - asserts.NoError(err) - query := resURL.Query() - asserts.Empty(query.Get("Signature")) - } - - // 正常 指定了CDN域名 - { - handler.Policy.BaseURL = "https://cqu.edu.cn" - res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0) - asserts.NoError(err) - resURL, err := url.Parse(res) - asserts.NoError(err) - query := resURL.Query() - asserts.Empty(query.Get("Signature")) - asserts.Contains(resURL.String(), handler.Policy.BaseURL) - } - - // 强制使用公网 Endpoint - { - handler.Policy.BaseURL = "" - handler.Policy.OptionsSerialized.ServerSideEndpoint = "endpoint.com" - res, err := handler.Source(context.WithValue(context.Background(), fsctx.ForceUsePublicEndpointCtx, false), "/123", url.URL{}, 10, false, 0) - asserts.NoError(err) - resURL, err := url.Parse(res) - asserts.NoError(err) - query := resURL.Query() - asserts.Empty(query.Get("Signature")) - asserts.Contains(resURL.String(), "endpoint.com") - } -} - -func TestDriver_Thumb(t *testing.T) { - asserts := assert.New(t) - handler := Driver{ - Policy: &model.Policy{ - AccessKey: "ak", - SecretKey: "sk", - BucketName: "test", - Server: "test.com", - }, - } - - // 上下文不存在 - { - ctx := context.Background() - res, err := handler.Thumb(ctx, "/123.txt") - asserts.Error(err) - asserts.Nil(res) - } - - // 成功 - { - cache.Set("setting_preview_timeout", "60", 0) - ctx := context.WithValue(context.Background(), fsctx.ThumbSizeCtx, [2]uint{10, 20}) - res, err := handler.Thumb(ctx, "/123.jpg") - asserts.NoError(err) - resURL, err := url.Parse(res.URL) - asserts.NoError(err) - urlQuery := resURL.Query() - asserts.Equal("image/resize,m_lfit,h_20,w_10", urlQuery.Get("x-oss-process")) - } -} - -func TestDriver_Delete(t *testing.T) { - asserts := assert.New(t) - handler := Driver{ - Policy: &model.Policy{ - AccessKey: "ak", - SecretKey: "sk", - BucketName: "test", - Server: "oss-cn-shanghai.aliyuncs.com", - }, - } - - // 失败 - { - res, err := handler.Delete(context.Background(), []string{"1", "2", "3"}) - asserts.Error(err) - asserts.Equal([]string{"1", "2", "3"}, res) - } -} - -func TestDriver_Put(t *testing.T) { - asserts := assert.New(t) - handler := Driver{ - Policy: &model.Policy{ - AccessKey: "ak", - SecretKey: "sk", - BucketName: "test", - Server: "oss-cn-shanghai.aliyuncs.com", - }, - } - cache.Set("setting_upload_credential_timeout", "3600", 0) - ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true) - - // 失败 - { - err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("123")), "/123.txt", 3) - asserts.Error(err) - } -} - -type ClientMock struct { - testMock.Mock -} - -func (m ClientMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response { - args := m.Called(method, target, body, opts) - return args.Get(0).(*request.Response) -} - -func TestDriver_Get(t *testing.T) { - asserts := assert.New(t) - handler := Driver{ - Policy: &model.Policy{ - AccessKey: "ak", - SecretKey: "sk", - BucketName: "test", - Server: "oss-cn-shanghai.aliyuncs.com", - }, - HTTPClient: request.NewClient(), - } - cache.Set("setting_preview_timeout", "3600", 0) - - // 响应失败 - { - res, err := handler.Get(context.Background(), "123.txt") - asserts.Error(err) - asserts.Nil(res) - } - - // 响应成功 - { - ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Size: 3}) - clientMock := ClientMock{} - clientMock.On( - "Request", - "GET", - testMock.Anything, - testMock.Anything, - testMock.Anything, - ).Return(&request.Response{ - Err: nil, - Response: &http.Response{ - StatusCode: 200, - Body: ioutil.NopCloser(strings.NewReader(`123`)), - }, - }) - handler.HTTPClient = clientMock - res, err := handler.Get(ctx, "123.txt") - clientMock.AssertExpectations(t) - asserts.NoError(err) - n, err := res.Seek(0, io.SeekEnd) - asserts.NoError(err) - asserts.EqualValues(3, n) - content, err := ioutil.ReadAll(res) - asserts.NoError(err) - asserts.Equal("123", string(content)) - } -} - -func TestDriver_List(t *testing.T) { - asserts := assert.New(t) - handler := Driver{ - Policy: &model.Policy{ - AccessKey: "ak", - SecretKey: "sk", - BucketName: "test", - Server: "test.com", - IsPrivate: true, - }, - } - - // 连接失败 - { - res, err := handler.List(context.Background(), "/", true) - asserts.Error(err) - asserts.Empty(res) - } -} diff --git a/pkg/filesystem/driver/remote/client_test.go b/pkg/filesystem/driver/remote/client_test.go new file mode 100644 index 00000000..c195521a --- /dev/null +++ b/pkg/filesystem/driver/remote/client_test.go @@ -0,0 +1,262 @@ +package remote + +import ( + "context" + "errors" + model "github.com/cloudreve/Cloudreve/v3/models" + "github.com/cloudreve/Cloudreve/v3/pkg/cache" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" + "github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock" + "github.com/cloudreve/Cloudreve/v3/pkg/request" + "github.com/stretchr/testify/assert" + testMock "github.com/stretchr/testify/mock" + "io/ioutil" + "net/http" + "strings" + "testing" +) + +func TestNewClient(t *testing.T) { + a := assert.New(t) + policy := &model.Policy{} + + // 无法解析服务端url + { + policy.Server = string([]byte{0x7f}) + c, err := NewClient(policy) + a.Error(err) + a.Nil(c) + } + + // 成功 + { + policy.Server = "" + c, err := NewClient(policy) + a.NoError(err) + a.NotNil(c) + } +} + +func TestRemoteClient_Upload(t *testing.T) { + a := assert.New(t) + c, _ := NewClient(&model.Policy{}) + + // 无法创建上传会话 + { + clientMock := requestmock.RequestMock{} + c.(*remoteClient).httpClient = &clientMock + clientMock.On( + "Request", + "PUT", + "upload", + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: errors.New("error"), + }) + err := c.Upload(context.Background(), &fsctx.FileStream{}) + a.Error(err) + a.Contains(err.Error(), "error") + clientMock.AssertExpectations(t) + } + + // 分片上传失败，成功删除上传会话 + { + cache.Set("setting_chunk_retries", "1", 0) + clientMock := requestmock.RequestMock{} + c.(*remoteClient).httpClient = &clientMock + clientMock.On( + "Request", + "PUT", + "upload", + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: nil, + Response: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)), + }, + }) + clientMock.On( + "Request", + "POST", + testMock.Anything, + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: errors.New("error"), + }) + clientMock.On( + "Request", + "DELETE", + testMock.Anything, + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: nil, + Response: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)), + }, + }) + err := c.Upload(context.Background(), &fsctx.FileStream{}) + a.Error(err) + a.Contains(err.Error(), "error") + clientMock.AssertExpectations(t) + } + + // 分片上传失败，无法删除上传会话 + { + cache.Set("setting_chunk_retries", "1", 0) + clientMock := requestmock.RequestMock{} + c.(*remoteClient).httpClient = &clientMock + clientMock.On( + "Request", + "PUT", + "upload", + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: nil, + Response: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)), + }, + }) + clientMock.On( + "Request", + "POST", + testMock.Anything, + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: errors.New("error"), + }) + clientMock.On( + "Request", + "DELETE", + testMock.Anything, + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: errors.New("error2"), + Response: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)), + }, + }) + err := c.Upload(context.Background(), &fsctx.FileStream{}) + a.Error(err) + a.Contains(err.Error(), "error") + clientMock.AssertExpectations(t) + } + + // 成功 + { + cache.Set("setting_chunk_retries", "1", 0) + clientMock := requestmock.RequestMock{} + c.(*remoteClient).httpClient = &clientMock + clientMock.On( + "Request", + "PUT", + "upload", + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: nil, + Response: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)), + }, + }) + clientMock.On( + "Request", + "POST", + testMock.Anything, + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Response: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)), + }, + }) + err := c.Upload(context.Background(), &fsctx.FileStream{}) + a.NoError(err) + clientMock.AssertExpectations(t) + } +} + +func TestRemoteClient_CreateUploadSessionFailed(t *testing.T) { + a := assert.New(t) + c, _ := NewClient(&model.Policy{}) + + clientMock := requestmock.RequestMock{} + c.(*remoteClient).httpClient = &clientMock + clientMock.On( + "Request", + "PUT", + "upload", + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: nil, + Response: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(strings.NewReader(`{"code":500,"msg":"error"}`)), + }, + }) + err := c.Upload(context.Background(), &fsctx.FileStream{}) + a.Error(err) + a.Contains(err.Error(), "error") + clientMock.AssertExpectations(t) +} + +func TestRemoteClient_UploadChunkFailed(t *testing.T) { + a := assert.New(t) + c, _ := NewClient(&model.Policy{}) + + clientMock := requestmock.RequestMock{} + c.(*remoteClient).httpClient = &clientMock + clientMock.On( + "Request", + "POST", + testMock.Anything, + testMock.Anything, + testMock.Anything, + ).Return(&request.Response{ + Err: nil, + Response: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(strings.NewReader(`{"code":500,"msg":"error"}`)), + }, + }) + err := c.(*remoteClient).uploadChunk(context.Background(), "", 0, strings.NewReader(""), false, 0) + a.Error(err) + a.Contains(err.Error(), "error") + clientMock.AssertExpectations(t) +} + +func TestRemoteClient_GetUploadURL(t *testing.T) { + a := assert.New(t) + c, _ := NewClient(&model.Policy{}) + + // url 解析失败 + { + c.(*remoteClient).policy.Server = string([]byte{0x7f}) + res, sign, err := c.GetUploadURL(0, "") + a.Error(err) + a.Empty(res) + a.Empty(sign) + } + + // 成功 + { + c.(*remoteClient).policy.Server = "" + res, sign, err := c.GetUploadURL(0, "") + a.NoError(err) + a.NotEmpty(res) + a.NotEmpty(sign) + } +} diff --git a/pkg/filesystem/driver/remote/handler_test.go b/pkg/filesystem/driver/remote/handler_test.go index 478b2905..9320bf2d 100644 --- a/pkg/filesystem/driver/remote/handler_test.go +++ b/pkg/filesystem/driver/remote/handler_test.go @@ -2,6 +2,9 @@ package remote import ( "context" + "errors" + "github.com/cloudreve/Cloudreve/v3/pkg/mocks/remoteclientmock" + "github.com/cloudreve/Cloudreve/v3/pkg/serializer" "io" "io/ioutil" "net/http" @@ -14,45 +17,26 @@ import ( "github.com/cloudreve/Cloudreve/v3/pkg/cache" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" "github.com/cloudreve/Cloudreve/v3/pkg/request" - "github.com/cloudreve/Cloudreve/v3/pkg/serializer" "github.com/stretchr/testify/assert" testMock "github.com/stretchr/testify/mock" ) -func TestHandler_Token(t *testing.T) { - asserts := assert.New(t) - handler := Driver{ - Policy: &model.Policy{ - MaxSize: 10, - AutoRename: true, - DirNameRule: "dir", - FileNameRule: "file", - OptionsSerialized: model.PolicyOption{ - FileType: []string{"txt"}, - }, - Server: "http://test.com", - }, - AuthInstance: auth.HMACAuth{}, +func TestNewDriver(t *testing.T) { + a := assert.New(t) + + // remoteClient 初始化失败 + { + d, err := NewDriver(&model.Policy{Server: string([]byte{0x7f})}) + a.Error(err) + a.Nil(d) } - ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true) - auth.General = auth.HMACAuth{SecretKey: []byte("test")} // 成功 { - cache.Set("setting_siteURL", "http://test.cloudreve.org", 0) - credential, err := handler.Token(ctx, 10, "123", nil) - asserts.NoError(err) - policy, err := serializer.DecodeUploadPolicy(credential.Policy) - asserts.NoError(err) - asserts.Equal("http://test.cloudreve.org/api/v3/callback/remote/123", policy.CallbackURL) - asserts.Equal(uint64(10), policy.MaxSize) - asserts.Equal(true, policy.AutoRename) - asserts.Equal("dir", policy.SavePath) - asserts.Equal("file", policy.FileName) - asserts.Equal("file", policy.FileName) - asserts.Equal([]string{"txt"}, policy.AllowedExtension) + d, err := NewDriver(&model.Policy{}) + a.NoError(err) + a.NotNil(d) } - } func TestHandler_Source(t *testing.T) { @@ -369,6 +353,20 @@ func TestHandler_Get(t *testing.T) { } func TestHandler_Put(t *testing.T) { + a := assert.New(t) + handler, _ := NewDriver(&model.Policy{ + Type: "remote", + SecretKey: "test", + Server: "http://test.com", + }) + clientMock := &remoteclientmock.RemoteClientMock{} + handler.uploadClient = clientMock + clientMock.On("Upload", testMock.Anything, testMock.Anything).Return(errors.New("error")) + a.Error(handler.Put(context.Background(), &fsctx.FileStream{})) + clientMock.AssertExpectations(t) +} + +func TestHandler_Thumb(t *testing.T) { asserts := assert.New(t) handler := Driver{ Policy: &model.Policy{ @@ -379,92 +377,65 @@ func TestHandler_Put(t *testing.T) { AuthInstance: auth.HMACAuth{}, } ctx := context.Background() - asserts.NoError(cache.Set("setting_upload_credential_timeout", "3600", 0)) + asserts.NoError(cache.Set("setting_preview_timeout", "60", 0)) + resp, err := handler.Thumb(ctx, "/1.txt") + asserts.NoError(err) + asserts.True(resp.Redirect) +} - // 成功 +func TestHandler_Token(t *testing.T) { + a := assert.New(t) + handler, _ := NewDriver(&model.Policy{}) + + // 无法创建上传会话 { - ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{}) - clientMock := ClientMock{} - clientMock.On( - "Request", - "POST", - "http://test.com/api/v3/slave/upload", - testMock.Anything, - testMock.Anything, - ).Return(&request.Response{ - Err: nil, - Response: &http.Response{ - StatusCode: 200, - Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)), - }, - }) - handler.Client = clientMock - err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15) + clientMock := &remoteclientmock.RemoteClientMock{} + handler.uploadClient = clientMock + clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10)).Return(errors.New("error")) + res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{}) + a.Error(err) + a.Contains(err.Error(), "error") + a.Nil(res) clientMock.AssertExpectations(t) - asserts.NoError(err) } - // 请求失败 + // 无法创建上传地址 { - ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{}) - clientMock := ClientMock{} - clientMock.On( - "Request", - "POST", - "http://test.com/api/v3/slave/upload", - testMock.Anything, - testMock.Anything, - ).Return(&request.Response{ - Err: nil, - Response: &http.Response{ - StatusCode: 404, - Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)), - }, - }) - handler.Client = clientMock - err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15) + clientMock := &remoteclientmock.RemoteClientMock{} + handler.uploadClient = clientMock + clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10)).Return(nil) + clientMock.On("GetUploadURL", int64(10), "").Return("", "", errors.New("error")) + res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{}) + a.Error(err) + a.Contains(err.Error(), "error") + a.Nil(res) clientMock.AssertExpectations(t) - asserts.Error(err) } - // 返回错误 + // 成功 { - ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{}) - clientMock := ClientMock{} - clientMock.On( - "Request", - "POST", - "http://test.com/api/v3/slave/upload", - testMock.Anything, - testMock.Anything, - ).Return(&request.Response{ - Err: nil, - Response: &http.Response{ - StatusCode: 200, - Body: ioutil.NopCloser(strings.NewReader(`{"code":1}`)), - }, - }) - handler.Client = clientMock - err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15) + clientMock := &remoteclientmock.RemoteClientMock{} + handler.uploadClient = clientMock + clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10)).Return(nil) + clientMock.On("GetUploadURL", int64(10), "").Return("1", "2", nil) + res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{}) + a.NoError(err) + a.NotNil(res) + a.Equal("1", res.UploadURLs[0]) + a.Equal("2", res.Credential) clientMock.AssertExpectations(t) - asserts.Error(err) } - } -func TestHandler_Thumb(t *testing.T) { - asserts := assert.New(t) - handler := Driver{ - Policy: &model.Policy{ - Type: "remote", - SecretKey: "test", - Server: "http://test.com", - }, - AuthInstance: auth.HMACAuth{}, - } - ctx := context.Background() - asserts.NoError(cache.Set("setting_preview_timeout", "60", 0)) - resp, err := handler.Thumb(ctx, "/1.txt") - asserts.NoError(err) - asserts.True(resp.Redirect) +func TestDriver_CancelToken(t *testing.T) { + a := assert.New(t) + handler, _ := NewDriver(&model.Policy{}) + + clientMock := &remoteclientmock.RemoteClientMock{} + handler.uploadClient = clientMock + clientMock.On("DeleteUploadSession", testMock.Anything, "key").Return(errors.New("error")) + err := handler.CancelToken(context.Background(), &serializer.UploadSession{Key: "key"}) + a.Error(err) + a.Contains(err.Error(), "error") + clientMock.AssertExpectations(t) } diff --git a/pkg/filesystem/file.go b/pkg/filesystem/file.go index 3d244e44..0aa67800 100644 --- a/pkg/filesystem/file.go +++ b/pkg/filesystem/file.go @@ -48,9 +48,6 @@ func (fs *FileSystem) AddFile(ctx context.Context, parent *model.Folder, file fs // 添加文件记录前的钩子 err := fs.Trigger(ctx, "BeforeAddFile", file) if err != nil { - if err := fs.Trigger(ctx, "BeforeAddFileFailed", file); err != nil { - util.Log().Debug("BeforeAddFileFailed 钩子执行失败，%s", err) - } return nil, err } @@ -180,6 +177,7 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m for policyID, toBeDeletedFiles := range files { // 列举出需要物理删除的文件的物理路径 sourceNamesAll := make([]string, 0, len(toBeDeletedFiles)) + uploadSessions := make([]*serializer.UploadSession, 0, len(toBeDeletedFiles)) for i := 0; i < len(toBeDeletedFiles); i++ { sourceNamesAll = append(sourceNamesAll, toBeDeletedFiles[i].SourceName) @@ -187,11 +185,7 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m if toBeDeletedFiles[i].UploadSessionID != nil { if session, ok := cache.Get(UploadSessionCachePrefix + *toBeDeletedFiles[i].UploadSessionID); ok { uploadSession := session.(serializer.UploadSession) - if err := fs.Handler.CancelToken(ctx, &uploadSession); err != nil { - util.Log().Warning("无法取消 [%s] 的上传会话: %s", err) - } - - cache.Deletes([]string{*toBeDeletedFiles[i].UploadSessionID}, UploadSessionCachePrefix) + uploadSessions = append(uploadSessions, &uploadSession) } } @@ -205,6 +199,15 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m continue } + // 取消上传会话 + for _, upSession := range uploadSessions { + if err := fs.Handler.CancelToken(ctx, upSession); err != nil { + util.Log().Warning("无法取消 [%s] 的上传会话: %s", upSession.Name, err) + } + + cache.Deletes([]string{upSession.Key}, UploadSessionCachePrefix) + } + // 执行删除 failedFile, _ := fs.Handler.Delete(ctx, sourceNamesAll) failed[policyID] = failedFile diff --git a/pkg/filesystem/file_test.go b/pkg/filesystem/file_test.go index cb3d24f6..5e495c75 100644 --- a/pkg/filesystem/file_test.go +++ b/pkg/filesystem/file_test.go @@ -2,6 +2,7 @@ package filesystem import ( "context" + "errors" "os" "testing" @@ -19,8 +20,9 @@ import ( func TestFileSystem_AddFile(t *testing.T) { asserts := assert.New(t) file := fsctx.FileStream{ - Size: 5, - Name: "1.png", + Size: 5, + Name: "1.png", + SavePath: "/Uploads/1_sad.png", } folder := model.Folder{ Model: gorm.Model{ @@ -39,24 +41,55 @@ func TestFileSystem_AddFile(t *testing.T) { }, }, }, + Policy: &model.Policy{Type: "cos"}, } - ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, file) - ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/Uploads/1_sad.png") - _, err := fs.AddFile(ctx, &folder) + _, err := fs.AddFile(context.Background(), &folder, &file) asserts.Error(err) mock.ExpectBegin() mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() - f, err := fs.AddFile(ctx, &folder) + f, err := fs.AddFile(context.Background(), &folder, &file) asserts.NoError(err) asserts.NoError(mock.ExpectationsWereMet()) asserts.Equal("/Uploads/1_sad.png", f.SourceName) asserts.NotEmpty(f.PicInfo) + + // 前置钩子执行失败 + { + hookExecuted := false + fs.Use("BeforeAddFile", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error { + hookExecuted = true + return errors.New("error") + }) + f, err := fs.AddFile(context.Background(), &folder, &file) + asserts.Error(err) + asserts.Nil(f) + asserts.True(hookExecuted) + } + + // 后置钩子执行失败 + { + hookExecuted := false + mock.ExpectBegin() + mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error")) + mock.ExpectRollback() + fs.Hooks = map[string][]Hook{} + fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error { + hookExecuted = true + return errors.New("error") + }) + f, err := fs.AddFile(context.Background(), &folder, &file) + asserts.Error(err) + asserts.Nil(f) + asserts.True(hookExecuted) + asserts.NoError(mock.ExpectationsWereMet()) + } } func TestFileSystem_GetContent(t *testing.T) { @@ -263,6 +296,22 @@ func TestFileSystem_deleteGroupedFile(t *testing.T) { 3: {}, }, failed) } + // 包含上传会话文件 + { + sessionID := "session" + cache.Set(UploadSessionCachePrefix+sessionID, serializer.UploadSession{Key: sessionID}, 0) + files[1].Policy.Type = "local" + files[3].Policy.Type = "local" + files[0].UploadSessionID = &sessionID + failed := fs.deleteGroupedFile(ctx, fs.GroupFileByPolicy(ctx, files)) + asserts.Equal(map[uint][]string{ + 1: {}, + 2: {}, + 3: {}, + }, failed) + _, ok := cache.Get(UploadSessionCachePrefix + sessionID) + asserts.False(ok) + } } func TestFileSystem_GetSource(t *testing.T) { diff --git a/pkg/filesystem/filesystem.go b/pkg/filesystem/filesystem.go index 51a13a85..b0da90e7 100644 --- a/pkg/filesystem/filesystem.go +++ b/pkg/filesystem/filesystem.go @@ -121,12 +121,11 @@ func NewAnonymousFileSystem() (*FileSystem, error) { // DispatchHandler 根据存储策略分配文件适配器 func (fs *FileSystem) DispatchHandler() error { - currentPolicy := fs.Policy - policyType := currentPolicy.Type - - if currentPolicy == nil { - return ErrUnknownPolicyType + if fs.Policy == nil { + return errors.New("未设置存储策略") } + policyType := fs.Policy.Type + currentPolicy := fs.Policy switch policyType { case "mock", "anonymous": diff --git a/pkg/filesystem/filesystem_test.go b/pkg/filesystem/filesystem_test.go index 0c558d79..8b7aae37 100644 --- a/pkg/filesystem/filesystem_test.go +++ b/pkg/filesystem/filesystem_test.go @@ -2,16 +2,16 @@ package filesystem import ( "github.com/cloudreve/Cloudreve/v3/pkg/cluster" + "github.com/cloudreve/Cloudreve/v3/pkg/conf" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/masterinslave" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/slaveinmaster" + "github.com/cloudreve/Cloudreve/v3/pkg/serializer" "net/http/httptest" "github.com/DATA-DOG/go-sqlmock" model "github.com/cloudreve/Cloudreve/v3/models" - "github.com/cloudreve/Cloudreve/v3/pkg/cache" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/remote" - "github.com/cloudreve/Cloudreve/v3/pkg/serializer" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" @@ -36,7 +36,7 @@ func TestNewFileSystem(t *testing.T) { fs, err = NewFileSystem(&user) asserts.NoError(err) asserts.NotNil(fs.Handler) - asserts.IsType(remote.Driver{}, fs.Handler) + asserts.IsType(&remote.Driver{}, fs.Handler) user.Policy.Type = "unknown" fs, err = NewFileSystem(&user) @@ -64,9 +64,10 @@ func TestNewFileSystemFromContext(t *testing.T) { func TestDispatchHandler(t *testing.T) { asserts := assert.New(t) fs := &FileSystem{ - User: &model.User{Policy: model.Policy{ + User: &model.User{}, + Policy: &model.Policy{ Type: "local", - }}, + }, } // 未指定，使用用户默认 @@ -95,7 +96,7 @@ func TestDispatchHandler(t *testing.T) { err = fs.DispatchHandler() asserts.NoError(err) - fs.Policy = &model.Policy{Type: "oss"} + fs.Policy = &model.Policy{Type: "oss", Server: "https://s.com", BucketName: "1234"} err = fs.DispatchHandler() asserts.NoError(err) @@ -140,23 +141,6 @@ func TestNewFileSystemFromCallback(t *testing.T) { asserts.Error(err) } - // 找不到上传策略 - { - c, _ := gin.CreateTestContext(httptest.NewRecorder()) - c.Set("user", &model.User{ - Policy: model.Policy{ - Type: "local", - }, - }) - c.Set("callbackSession", &serializer.UploadSession{PolicyID: 138}) - cache.Deletes([]string{"138"}, "policy_") - mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"})) - fs, err := NewFileSystemFromCallback(c) - asserts.NoError(mock.ExpectationsWereMet()) - asserts.Nil(fs) - asserts.Error(err) - } - // 成功 { c, _ := gin.CreateTestContext(httptest.NewRecorder()) @@ -165,11 +149,8 @@ func TestNewFileSystemFromCallback(t *testing.T) { Type: "local", }, }) - c.Set("callbackSession", &serializer.UploadSession{PolicyID: 138}) - cache.Deletes([]string{"138"}, "policy_") - mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id", "type", "options"}).AddRow(138, "local", "{}")) + c.Set(UploadSessionCtx, &serializer.UploadSession{Policy: model.Policy{Type: "local"}}) fs, err := NewFileSystemFromCallback(c) - asserts.NoError(mock.ExpectationsWereMet()) asserts.NotNil(fs) asserts.NoError(err) } @@ -234,6 +215,16 @@ func TestNewAnonymousFileSystem(t *testing.T) { asserts.Error(err) asserts.Nil(fs) } + + // 从机 + { + conf.SystemConfig.Mode = "slave" + fs, err := NewAnonymousFileSystem() + asserts.NoError(mock.ExpectationsWereMet()) + asserts.NoError(err) + asserts.NotNil(fs) + asserts.NotNil(fs.Handler) + } } func TestFileSystem_Recycle(t *testing.T) { diff --git a/pkg/filesystem/fsctx/stream_test.go b/pkg/filesystem/fsctx/stream_test.go index 8cc0c859..1ef6e1fa 100644 --- a/pkg/filesystem/fsctx/stream_test.go +++ b/pkg/filesystem/fsctx/stream_test.go @@ -1,30 +1,15 @@ package fsctx import ( + model "github.com/cloudreve/Cloudreve/v3/models" "github.com/stretchr/testify/assert" + "io" "io/ioutil" + "os" "strings" "testing" ) -func TestFileStream_GetFileName(t *testing.T) { - asserts := assert.New(t) - file := FileStream{Name: "123"} - asserts.Equal("123", file.GetFileName()) -} - -func TestFileStream_GetMIMEType(t *testing.T) { - asserts := assert.New(t) - file := FileStream{MIMEType: "123"} - asserts.Equal("123", file.GetMIMEType()) -} - -func TestFileStream_GetSize(t *testing.T) { - asserts := assert.New(t) - file := FileStream{Size: 123} - asserts.Equal(uint64(123), file.GetSize()) -} - func TestFileStream_Read(t *testing.T) { asserts := assert.New(t) file := FileStream{ @@ -40,9 +25,54 @@ func TestFileStream_Read(t *testing.T) { func TestFileStream_Close(t *testing.T) { asserts := assert.New(t) - file := FileStream{ - File: ioutil.NopCloser(strings.NewReader("123")), + { + file := FileStream{ + File: ioutil.NopCloser(strings.NewReader("123")), + } + err := file.Close() + asserts.NoError(err) + } + + { + file := FileStream{} + err := file.Close() + asserts.NoError(err) + } +} + +func TestFileStream_Seek(t *testing.T) { + asserts := assert.New(t) + f, _ := os.CreateTemp("", "*") + defer func() { + f.Close() + os.Remove(f.Name()) + }() + { + file := FileStream{ + File: f, + Seeker: f, + } + res, err := file.Seek(0, io.SeekStart) + asserts.NoError(err) + asserts.EqualValues(0, res) } - err := file.Close() - asserts.NoError(err) + + { + file := FileStream{} + res, err := file.Seek(0, io.SeekStart) + asserts.Error(err) + asserts.EqualValues(0, res) + } +} + +func TestFileStream_Info(t *testing.T) { + a := assert.New(t) + file := FileStream{} + a.NotNil(file.Info()) + + file.SetSize(10) + a.EqualValues(10, file.Info().Size) + + file.SetModel(&model.File{}) + a.NotNil(file.Info().Model) } diff --git a/pkg/filesystem/fsctx/taskinfo/taskinfo.go b/pkg/filesystem/fsctx/taskinfo/taskinfo.go deleted file mode 100644 index 1899c1bf..00000000 --- a/pkg/filesystem/fsctx/taskinfo/taskinfo.go +++ /dev/null @@ -1 +0,0 @@ -package taskinfo diff --git a/pkg/filesystem/hooks.go b/pkg/filesystem/hooks.go index 0cd1cb99..fc4a68fa 100644 --- a/pkg/filesystem/hooks.go +++ b/pkg/filesystem/hooks.go @@ -148,7 +148,6 @@ func HookCancelContext(ctx context.Context, fs *FileSystem, file fsctx.FileHeade } // HookUpdateSourceName 更新文件SourceName -// TODO：测试 func HookUpdateSourceName(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error { originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File) if !ok { diff --git a/pkg/filesystem/hooks_test.go b/pkg/filesystem/hooks_test.go index a247490e..0daa8ec6 100644 --- a/pkg/filesystem/hooks_test.go +++ b/pkg/filesystem/hooks_test.go @@ -3,22 +3,20 @@ package filesystem import ( "context" "errors" - "io" - "io/ioutil" - "net/http" - "os" - "strings" - "sync" - "testing" - "github.com/DATA-DOG/go-sqlmock" - model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/pkg/cache" "github.com/cloudreve/Cloudreve/v3/pkg/conf" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" + "github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock" "github.com/cloudreve/Cloudreve/v3/pkg/request" "github.com/cloudreve/Cloudreve/v3/pkg/serializer" + "io/ioutil" + "net/http" + "strings" + "testing" + + model "github.com/cloudreve/Cloudreve/v3/models" "github.com/jinzhu/gorm" "github.com/stretchr/testify/assert" testMock "github.com/stretchr/testify/mock" @@ -26,78 +24,72 @@ import ( func TestGenericBeforeUpload(t *testing.T) { asserts := assert.New(t) - file := fsctx.FileStream{ + file := &fsctx.FileStream{ Size: 5, Name: "1.txt", } + ctx := context.Background() cache.Set("pack_size_0", uint64(0), 0) - ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, file) fs := FileSystem{ User: &model.User{ Storage: 0, Group: model.Group{ MaxStorage: 11, }, - Policy: model.Policy{ - MaxSize: 4, - OptionsSerialized: model.PolicyOption{ - FileType: []string{"txt"}, - }, + }, + Policy: &model.Policy{ + MaxSize: 4, + OptionsSerialized: model.PolicyOption{ + FileType: []string{"txt"}, }, }, } - asserts.Error(HookValidateFile(ctx, &fs)) + asserts.Error(HookValidateFile(ctx, &fs, file)) file.Size = 1 file.Name = "1" - ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file) - asserts.Error(HookValidateFile(ctx, &fs)) + asserts.Error(HookValidateFile(ctx, &fs, file)) file.Name = "1.txt" - ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file) - asserts.NoError(HookValidateFile(ctx, &fs)) + asserts.NoError(HookValidateFile(ctx, &fs, file)) file.Name = "1.t/xt" - ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file) - asserts.Error(HookValidateFile(ctx, &fs)) + asserts.Error(HookValidateFile(ctx, &fs, file)) } func TestGenericAfterUploadCanceled(t *testing.T) { asserts := assert.New(t) - f, err := os.Create("TestGenericAfterUploadCanceled") - asserts.NoError(err) - f.Close() - file := fsctx.FileStream{ - Size: 5, - Name: "TestGenericAfterUploadCanceled", + file := &fsctx.FileStream{ + Size: 5, + Name: "TestGenericAfterUploadCanceled", + SavePath: "TestGenericAfterUploadCanceled", } - ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "TestGenericAfterUploadCanceled") - ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file) + ctx := context.Background() fs := FileSystem{ - User: &model.User{Storage: 5}, - Handler: local.Driver{}, + User: &model.User{}, } // 成功 - err = HookDeleteTempFile(ctx, &fs) - asserts.NoError(err) - err = HookGiveBackCapacity(ctx, &fs) - asserts.NoError(err) - asserts.Equal(uint64(0), fs.User.Storage) - - f, err = os.Create("TestGenericAfterUploadCanceled") - asserts.NoError(err) - f.Close() + { + mockHandler := &FileHeaderMock{} + fs.Handler = mockHandler + mockHandler.On("Delete", testMock.Anything, testMock.Anything).Return([]string{}, nil) + err := HookDeleteTempFile(ctx, &fs, file) + asserts.NoError(err) + mockHandler.AssertExpectations(t) + } - // 容量不能再降低 - err = HookGiveBackCapacity(ctx, &fs) - asserts.Error(err) + // 失败 + { + mockHandler := &FileHeaderMock{} + fs.Handler = mockHandler + mockHandler.On("Delete", testMock.Anything, testMock.Anything).Return([]string{}, errors.New("")) + err := HookDeleteTempFile(ctx, &fs, file) + asserts.NoError(err) + mockHandler.AssertExpectations(t) + } - //文件不存在 - fs.User.Storage = 5 - err = HookDeleteTempFile(ctx, &fs) - asserts.NoError(err) } func TestGenericAfterUpload(t *testing.T) { @@ -108,13 +100,14 @@ func TestGenericAfterUpload(t *testing.T) { ID: 1, }, }, + Policy: &model.Policy{}, } - ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{ + ctx := context.Background() + file := &fsctx.FileStream{ VirtualPath: "/我的文件", Name: "test.txt", - }) - ctx = context.WithValue(ctx, fsctx.SavePathCtx, "") + } // 正常 mock.ExpectQuery("SELECT(.+)"). @@ -127,9 +120,10 @@ func TestGenericAfterUpload(t *testing.T) { mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found")) mock.ExpectBegin() mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() - err := GenericAfterUpload(ctx, &fs) + err := GenericAfterUpload(ctx, &fs, file) asserts.NoError(err) asserts.NoError(mock.ExpectationsWereMet()) @@ -137,7 +131,7 @@ func TestGenericAfterUpload(t *testing.T) { mock.ExpectQuery("SELECT(.+)folders(.+)").WillReturnRows( mock.NewRows([]string{"name"}), ) - err = GenericAfterUpload(ctx, &fs) + err = GenericAfterUpload(ctx, &fs, file) asserts.Equal(ErrRootProtected, err) asserts.NoError(mock.ExpectationsWereMet()) @@ -152,10 +146,25 @@ func TestGenericAfterUpload(t *testing.T) { mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnRows( mock.NewRows([]string{"name"}).AddRow("test.txt"), ) - err = GenericAfterUpload(ctx, &fs) + err = GenericAfterUpload(ctx, &fs, file) asserts.Equal(ErrFileExisted, err) asserts.NoError(mock.ExpectationsWereMet()) + // 文件已存在, 且为上传占位符 + mock.ExpectQuery("SELECT(.+)"). + WithArgs(1). + WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1)) + // 1 + mock.ExpectQuery("SELECT(.+)"). + WithArgs(1, 1, "我的文件"). + WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1)) + mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnRows( + mock.NewRows([]string{"name", "upload_session_id"}).AddRow("test.txt", "1"), + ) + err = GenericAfterUpload(ctx, &fs, file) + asserts.Equal(ErrFileUploadSessionExisted, err) + asserts.NoError(mock.ExpectationsWereMet()) + // 插入失败 mock.ExpectQuery("SELECT(.+)"). WithArgs(1). @@ -170,7 +179,7 @@ func TestGenericAfterUpload(t *testing.T) { mock.ExpectExec("INSERT(.+)files(.+)").WillReturnError(errors.New("error")) mock.ExpectRollback() - err = GenericAfterUpload(ctx, &fs) + err = GenericAfterUpload(ctx, &fs, file) asserts.Equal(ErrInsertFileRecord, err) asserts.NoError(mock.ExpectationsWereMet()) @@ -180,7 +189,7 @@ func TestFileSystem_Use(t *testing.T) { asserts := assert.New(t) fs := FileSystem{} - hook := func(ctx context.Context, fs *FileSystem) error { + hook := func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error { return nil } @@ -215,77 +224,79 @@ func TestFileSystem_Trigger(t *testing.T) { } ctx := context.Background() - hook := func(ctx context.Context, fs *FileSystem) error { + hook := func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error { fs.User.Storage++ return nil } // 一个 fs.Use("BeforeUpload", hook) - err := fs.Trigger(ctx, "BeforeUpload") + err := fs.Trigger(ctx, "BeforeUpload", nil) asserts.NoError(err) asserts.Equal(uint64(1), fs.User.Storage) // 多个 fs.Use("BeforeUpload", hook) fs.Use("BeforeUpload", hook) - err = fs.Trigger(ctx, "BeforeUpload") + err = fs.Trigger(ctx, "BeforeUpload", nil) asserts.NoError(err) asserts.Equal(uint64(4), fs.User.Storage) + + // 多个,有失败 + fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error { + return errors.New("error") + }) + fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error { + asserts.Fail("following hooks executed") + return nil + }) + err = fs.Trigger(ctx, "BeforeUpload", nil) + asserts.Error(err) } -func TestHookIsFileExist(t *testing.T) { +func TestHookValidateCapacity(t *testing.T) { asserts := assert.New(t) + cache.Set("pack_size_1", uint64(0), 0) fs := &FileSystem{User: &model.User{ - Model: gorm.Model{ - ID: 1, + Model: gorm.Model{ID: 1}, + Storage: 0, + Group: model.Group{ + MaxStorage: 11, }, }} - ctx := context.WithValue(context.Background(), fsctx.PathCtx, "/test.txt") + ctx := context.Background() + file := &fsctx.FileStream{Size: 11} { - mock.ExpectQuery("SELECT(.+)"). - WithArgs(1). - WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1)) - mock.ExpectQuery("SELECT(.+)").WithArgs(uint(1), "test.txt").WillReturnRows( - sqlmock.NewRows([]string{"Name"}).AddRow("s"), - ) - err := HookIsFileExist(ctx, fs) - asserts.NoError(mock.ExpectationsWereMet()) + err := HookValidateCapacity(ctx, fs, file) asserts.NoError(err) } { - mock.ExpectQuery("SELECT(.+)"). - WithArgs(1). - WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1)) - mock.ExpectQuery("SELECT(.+)").WithArgs(uint(1), "test.txt").WillReturnRows( - sqlmock.NewRows([]string{"Name"}), - ) - err := HookIsFileExist(ctx, fs) - asserts.NoError(mock.ExpectationsWereMet()) + file.Size = 12 + err := HookValidateCapacity(ctx, fs, file) asserts.Error(err) } - } -func TestHookValidateCapacity(t *testing.T) { - asserts := assert.New(t) - cache.Set("pack_size_1", uint64(0), 0) +func TestHookValidateCapacityDiff(t *testing.T) { + a := assert.New(t) fs := &FileSystem{User: &model.User{ - Model: gorm.Model{ID: 1}, - Storage: 0, Group: model.Group{ MaxStorage: 11, }, }} - ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{Size: 10}) + file := model.File{Size: 10} + ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file) + + // 无需操作 { - err := HookValidateCapacity(ctx, fs) - asserts.NoError(err) + a.NoError(HookValidateCapacityDiff(ctx, fs, &fsctx.FileStream{Size: 10})) } + + // 需要验证 { - err := HookValidateCapacity(ctx, fs) - asserts.Error(err) + a.Error(HookValidateCapacityDiff(ctx, fs, &fsctx.FileStream{Size: 12})) } + } func TestHookResetPolicy(t *testing.T) { @@ -301,7 +312,7 @@ func TestHookResetPolicy(t *testing.T) { mock.ExpectQuery("SELECT(.+)policies(.+)"). WillReturnRows(sqlmock.NewRows([]string{"id", "type"}).AddRow(2, "local")) ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file) - err := HookResetPolicy(ctx, fs) + err := HookResetPolicy(ctx, fs, nil) asserts.NoError(mock.ExpectationsWereMet()) asserts.NoError(err) } @@ -310,76 +321,22 @@ func TestHookResetPolicy(t *testing.T) { { cache.Deletes([]string{"2"}, "policy_") ctx := context.Background() - err := HookResetPolicy(ctx, fs) + err := HookResetPolicy(ctx, fs, nil) asserts.Error(err) } } -func TestHookChangeCapacity(t *testing.T) { - asserts := assert.New(t) - cache.Set("pack_size_1", uint64(0), 0) - - // 容量增加 失败 - { - fs := &FileSystem{User: &model.User{ - Model: gorm.Model{ID: 1}, - }} - - newFile := fsctx.FileStream{Size: 10} - oldFile := model.File{Size: 9} - ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile) - ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile) - err := HookChangeCapacity(ctx, fs) - asserts.Equal(ErrInsufficientCapacity, err) - } - - // 容量增加 成功 - { - fs := &FileSystem{User: &model.User{ - Model: gorm.Model{ID: 1}, - Group: model.Group{MaxStorage: 1}, - }} - - newFile := fsctx.FileStream{Size: 10} - oldFile := model.File{Size: 9} - ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile) - ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile) - mock.ExpectBegin() - mock.ExpectExec("UPDATE(.+)").WithArgs(1, sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1)) - err := HookChangeCapacity(ctx, fs) - asserts.NoError(mock.ExpectationsWereMet()) - asserts.NoError(err) - asserts.Equal(uint64(1), fs.User.Storage) - } - - // 容量减少 - { - fs := &FileSystem{User: &model.User{ - Model: gorm.Model{ID: 1}, - Storage: 1, - }} - - newFile := fsctx.FileStream{Size: 9} - oldFile := model.File{Size: 10} - ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile) - ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile) - err := HookChangeCapacity(ctx, fs) - asserts.NoError(err) - asserts.Equal(uint64(0), fs.User.Storage) - } -} - func TestHookCleanFileContent(t *testing.T) { asserts := assert.New(t) fs := &FileSystem{User: &model.User{ Model: gorm.Model{ID: 1}, }} - ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "123/123") + file := &fsctx.FileStream{SavePath: "123/123"} handlerMock := FileHeaderMock{} - handlerMock.On("Put", testMock.Anything, testMock.Anything, "123/123").Return(errors.New("error")) + handlerMock.On("Put", testMock.Anything, testMock.Anything).Return(errors.New("error")) fs.Handler = handlerMock - err := HookCleanFileContent(ctx, fs) + err := HookCleanFileContent(context.Background(), fs, file) asserts.Error(err) handlerMock.AssertExpectations(t) } @@ -395,14 +352,17 @@ func TestHookClearFileSize(t *testing.T) { ctx := context.WithValue( context.Background(), fsctx.FileModelCtx, - model.File{Model: gorm.Model{ID: 1}}, + model.File{Model: gorm.Model{ID: 1}, Size: 10}, ) mock.ExpectBegin() - mock.ExpectExec("UPDATE(.+)"). - WithArgs(0, sqlmock.AnyArg(), 1). + mock.ExpectExec("UPDATE(.+)files(.+)"). + WithArgs(0, sqlmock.AnyArg(), 1, 10). + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE(.+)users(.+)"). + WithArgs(10, sqlmock.AnyArg()). WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() - err := HookClearFileSize(ctx, fs) + err := HookClearFileSize(ctx, fs, nil) asserts.NoError(mock.ExpectationsWereMet()) asserts.NoError(err) } @@ -410,7 +370,7 @@ func TestHookClearFileSize(t *testing.T) { // 上下文对象不存在 { ctx := context.Background() - err := HookClearFileSize(ctx, fs) + err := HookClearFileSize(ctx, fs, nil) asserts.Error(err) } @@ -432,7 +392,7 @@ func TestHookUpdateSourceName(t *testing.T) { mock.ExpectBegin() mock.ExpectExec("UPDATE(.+)").WithArgs("new.txt", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() - err := HookUpdateSourceName(ctx, fs) + err := HookUpdateSourceName(ctx, fs, nil) asserts.NoError(mock.ExpectationsWereMet()) asserts.NoError(err) } @@ -440,7 +400,7 @@ func TestHookUpdateSourceName(t *testing.T) { // 上下文错误 { ctx := context.Background() - err := HookUpdateSourceName(ctx, fs) + err := HookUpdateSourceName(ctx, fs, nil) asserts.Error(err) } } @@ -457,41 +417,32 @@ func TestGenericAfterUpdate(t *testing.T) { Model: gorm.Model{ID: 1}, PicInfo: "1,1", } - newFile := fsctx.FileStream{Size: 10} + newFile := &fsctx.FileStream{Size: 10} ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile) - ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile) handlerMock := FileHeaderMock{} handlerMock.On("Delete", testMock.Anything, []string{"._thumb"}).Return([]string{}, nil) fs.Handler = handlerMock mock.ExpectBegin() - mock.ExpectExec("UPDATE(.+)"). - WithArgs(10, sqlmock.AnyArg(), 1). + mock.ExpectExec("UPDATE(.+)files(.+)"). + WithArgs(10, sqlmock.AnyArg(), 1, 0). + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE(.+)users(.+)"). + WithArgs(10, sqlmock.AnyArg()). WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() - err := GenericAfterUpdate(ctx, fs) + err := GenericAfterUpdate(ctx, fs, newFile) asserts.NoError(mock.ExpectationsWereMet()) asserts.NoError(err) } - // 新文件上下文不存在 - { - originFile := model.File{ - Model: gorm.Model{ID: 1}, - PicInfo: "1,1", - } - ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile) - err := GenericAfterUpdate(ctx, fs) - asserts.Error(err) - } - // 原始文件上下文不存在 { - newFile := fsctx.FileStream{Size: 10} - ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, newFile) - err := GenericAfterUpdate(ctx, fs) + newFile := &fsctx.FileStream{Size: 10} + ctx := context.Background() + err := GenericAfterUpdate(ctx, fs, newFile) asserts.Error(err) } @@ -502,91 +453,41 @@ func TestGenericAfterUpdate(t *testing.T) { Model: gorm.Model{ID: 1}, PicInfo: "1,1", } - newFile := fsctx.FileStream{Size: 10} + newFile := &fsctx.FileStream{Size: 10} ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile) - ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile) mock.ExpectBegin() mock.ExpectExec("UPDATE(.+)"). - WithArgs(10, sqlmock.AnyArg(), 1). + WithArgs(10, sqlmock.AnyArg(), 1, 0). WillReturnError(errors.New("error")) mock.ExpectRollback() - err := GenericAfterUpdate(ctx, fs) + err := GenericAfterUpdate(ctx, fs, newFile) asserts.NoError(mock.ExpectationsWereMet()) asserts.Error(err) } } -func TestHookSlaveUploadValidate(t *testing.T) { - asserts := assert.New(t) - conf.SystemConfig.Mode = "slave" - fs, err := NewAnonymousFileSystem() - conf.SystemConfig.Mode = "master" - asserts.NoError(err) - - // 正常 - { - policy := serializer.UploadPolicy{ - SavePath: "", - MaxSize: 10, - AllowedExtension: nil, - } - file := fsctx.FileStream{Name: "1.txt", Size: 10} - ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy) - ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file) - asserts.NoError(HookSlaveUploadValidate(ctx, fs)) - } - - // 尺寸太大 - { - policy := serializer.UploadPolicy{ - SavePath: "", - MaxSize: 10, - AllowedExtension: nil, - } - file := fsctx.FileStream{Name: "1.txt", Size: 11} - ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy) - ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file) - asserts.Equal(ErrFileSizeTooBig, HookSlaveUploadValidate(ctx, fs)) - } - - // 文件名非法 - { - policy := serializer.UploadPolicy{ - SavePath: "", - MaxSize: 10, - AllowedExtension: nil, - } - file := fsctx.FileStream{Name: "/1.txt", Size: 10} - ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy) - ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file) - asserts.Equal(ErrIllegalObjectName, HookSlaveUploadValidate(ctx, fs)) - } - - // 扩展名非法 - { - policy := serializer.UploadPolicy{ - SavePath: "", - MaxSize: 10, - AllowedExtension: []string{"jpg"}, - } - file := fsctx.FileStream{Name: "1.txt", Size: 10} - ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy) - ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file) - asserts.Equal(ErrFileExtensionNotAllowed, HookSlaveUploadValidate(ctx, fs)) +func TestHookGenerateThumb(t *testing.T) { + a := assert.New(t) + mockHandler := &FileHeaderMock{} + fs := &FileSystem{ + User: &model.User{ + Model: gorm.Model{ID: 1}, + }, + Handler: mockHandler, + Policy: &model.Policy{Type: "local"}, } -} - -type ClientMock struct { - testMock.Mock -} - -func (m ClientMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response { - args := m.Called(method, target, body, opts) - return args.Get(0).(*request.Response) + mockHandler.On("Delete", testMock.Anything, []string{"1.txt._thumb"}).Return([]string{}, nil) + a.NoError(HookGenerateThumb(context.Background(), fs, &fsctx.FileStream{ + Model: &model.File{ + SourceName: "1.txt", + }, + })) + fs.Recycle() + mockHandler.AssertExpectations(t) } func TestSlaveAfterUpload(t *testing.T) { @@ -598,7 +499,7 @@ func TestSlaveAfterUpload(t *testing.T) { // 成功 { - clientMock := ClientMock{} + clientMock := requestmock.RequestMock{} clientMock.On( "Request", "POST", @@ -613,19 +514,28 @@ func TestSlaveAfterUpload(t *testing.T) { }, }) request.GeneralClient = clientMock - ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{ + file := &fsctx.FileStream{ Size: 10, VirtualPath: "/my", Name: "test.txt", - }) - ctx = context.WithValue(ctx, fsctx.UploadPolicyCtx, serializer.UploadPolicy{ - CallbackURL: "http://test/callbakc", - }) - ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/not_exist") - err := SlaveAfterUpload(ctx, fs) + SavePath: "/not_exist", + } + err := SlaveAfterUpload(&serializer.UploadSession{Callback: "http://test/callbakc"})(context.Background(), fs, file) clientMock.AssertExpectations(t) asserts.NoError(err) } + + // 跳过回调 + { + file := &fsctx.FileStream{ + Size: 10, + VirtualPath: "/my", + Name: "test.txt", + SavePath: "/not_exist", + } + err := SlaveAfterUpload(&serializer.UploadSession{})(context.Background(), fs, file) + asserts.NoError(err) + } } func TestFileSystem_CleanHooks(t *testing.T) { @@ -663,7 +573,7 @@ func TestHookCancelContext(t *testing.T) { // empty ctx { - asserts.NoError(HookCancelContext(ctx, fs)) + asserts.NoError(HookCancelContext(ctx, fs, nil)) select { case <-ctx.Done(): t.Errorf("Channel should not be closed") @@ -675,62 +585,99 @@ func TestHookCancelContext(t *testing.T) { // with cancel ctx { ctx = context.WithValue(ctx, fsctx.CancelFuncCtx, cancel) - asserts.NoError(HookCancelContext(ctx, fs)) + asserts.NoError(HookCancelContext(ctx, fs, nil)) _, ok := <-ctx.Done() asserts.False(ok) } } -func TestHookGiveBackCapacity(t *testing.T) { - asserts := assert.New(t) - fs := &FileSystem{ - User: &model.User{ - Model: gorm.Model{ID: 1}, - Storage: 10, +func TestHookClearFileHeaderSize(t *testing.T) { + a := assert.New(t) + fs := &FileSystem{} + file := &fsctx.FileStream{Size: 10} + a.NoError(HookClearFileHeaderSize(context.Background(), fs, file)) + a.EqualValues(0, file.Size) +} + +func TestHookTruncateFileTo(t *testing.T) { + a := assert.New(t) + fs := &FileSystem{} + file := &fsctx.FileStream{} + a.NoError(HookTruncateFileTo(0)(context.Background(), fs, file)) + + fs.Handler = local.Driver{} + a.Error(HookTruncateFileTo(0)(context.Background(), fs, file)) +} + +func TestHookChunkUploaded(t *testing.T) { + a := assert.New(t) + fs := &FileSystem{} + file := &fsctx.FileStream{ + AppendStart: 10, + Size: 10, + Model: &model.File{ + Model: gorm.Model{ID: 1}, }, } - ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{Size: 1}) - // without once limit - { - asserts.NoError(HookGiveBackCapacity(ctx, fs)) - asserts.EqualValues(9, fs.User.Storage) - asserts.NoError(HookGiveBackCapacity(ctx, fs)) - asserts.EqualValues(8, fs.User.Storage) - } + mock.ExpectBegin() + mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(20, sqlmock.AnyArg(), 1, 0).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE(.+)users(.+)"). + WithArgs(20, sqlmock.AnyArg()). + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + a.NoError(HookChunkUploaded(context.Background(), fs, file)) + a.NoError(mock.ExpectationsWereMet()) +} - // with once limit - { - ctx = context.WithValue(ctx, fsctx.ValidateCapacityOnceCtx, &sync.Once{}) - asserts.NoError(HookGiveBackCapacity(ctx, fs)) - asserts.EqualValues(7, fs.User.Storage) - asserts.NoError(HookGiveBackCapacity(ctx, fs)) - asserts.EqualValues(7, fs.User.Storage) +func TestHookChunkUploadFailed(t *testing.T) { + a := assert.New(t) + fs := &FileSystem{} + file := &fsctx.FileStream{ + AppendStart: 10, + Size: 10, + Model: &model.File{ + Model: gorm.Model{ID: 1}, + }, } + + mock.ExpectBegin() + mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(10, sqlmock.AnyArg(), 1, 0).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE(.+)users(.+)"). + WithArgs(10, sqlmock.AnyArg()). + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + a.NoError(HookChunkUploadFailed(context.Background(), fs, file)) + a.NoError(mock.ExpectationsWereMet()) } -func TestHookValidateCapacityWithoutIncrease(t *testing.T) { +func TestHookPopPlaceholderToFile(t *testing.T) { a := assert.New(t) - fs := &FileSystem{ - User: &model.User{ - Model: gorm.Model{ID: 1}, - Storage: 10, - Group: model.Group{}, + fs := &FileSystem{} + file := &fsctx.FileStream{ + Model: &model.File{ + Model: gorm.Model{ID: 1}, }, } - ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{Size: 1}) - // not enough - { - fs.User.Group.MaxStorage = 10 - a.Error(HookValidateCapacity(ctx, fs)) - a.EqualValues(10, fs.User.Storage) - } + mock.ExpectBegin() + mock.ExpectExec("UPDATE(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + a.NoError(HookPopPlaceholderToFile("1,1")(context.Background(), fs, file)) + a.NoError(mock.ExpectationsWereMet()) +} - // enough - { - fs.User.Group.MaxStorage = 11 - a.NoError(HookValidateCapacity(ctx, fs)) - a.EqualValues(10, fs.User.Storage) +func TestHookDeleteUploadSession(t *testing.T) { + a := assert.New(t) + fs := &FileSystem{} + file := &fsctx.FileStream{ + Model: &model.File{ + Model: gorm.Model{ID: 1}, + }, } + + cache.Set(UploadSessionCachePrefix+"TestHookDeleteUploadSession", "", 0) + a.NoError(HookDeleteUploadSession("TestHookDeleteUploadSession")(context.Background(), fs, file)) + _, ok := cache.Get(UploadSessionCachePrefix + "TestHookDeleteUploadSession") + a.False(ok) } diff --git a/pkg/filesystem/image_test.go b/pkg/filesystem/image_test.go index 9678df92..42c7f456 100644 --- a/pkg/filesystem/image_test.go +++ b/pkg/filesystem/image_test.go @@ -1,43 +1,38 @@ package filesystem import ( - "context" "testing" - model "github.com/cloudreve/Cloudreve/v3/models" - "github.com/cloudreve/Cloudreve/v3/pkg/cache" - "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response" "github.com/stretchr/testify/assert" - testMock "github.com/stretchr/testify/mock" ) -func TestFileSystem_GetThumb(t *testing.T) { - asserts := assert.New(t) - fs := &FileSystem{User: &model.User{}} - - // 非图像文件 - { - fs.SetTargetFile(&[]model.File{{}}) - _, err := fs.GetThumb(context.Background(), 1) - asserts.Equal(err, ErrObjectNotExist) - } - - // 成功 - { - cache.Set("setting_thumb_width", "10", 0) - cache.Set("setting_thumb_height", "10", 0) - cache.Set("setting_preview_timeout", "50", 0) - testHandller2 := new(FileHeaderMock) - testHandller2.On("Thumb", testMock.Anything, "").Return(&response.ContentResponse{}, nil) - fs.CleanTargets() - fs.SetTargetFile(&[]model.File{{PicInfo: "1,1", Policy: model.Policy{Type: "mock"}}}) - fs.FileTarget[0].Policy.ID = 1 - fs.Handler = testHandller2 - res, err := fs.GetThumb(context.Background(), 1) - asserts.NoError(err) - asserts.EqualValues(50, res.MaxAge) - } -} +//func TestFileSystem_GetThumb(t *testing.T) { +// asserts := assert.New(t) +// fs := &FileSystem{User: &model.User{}} +// +// // 非图像文件 +// { +// fs.SetTargetFile(&[]model.File{{}}) +// _, err := fs.GetThumb(context.Background(), 1) +// asserts.Equal(err, ErrObjectNotExist) +// } +// +// // 成功 +// { +// cache.Set("setting_thumb_width", "10", 0) +// cache.Set("setting_thumb_height", "10", 0) +// cache.Set("setting_preview_timeout", "50", 0) +// testHandller2 := new(FileHeaderMock) +// testHandller2.On("Thumb", testMock.Anything, "").Return(&response.ContentResponse{}, nil) +// fs.CleanTargets() +// fs.SetTargetFile(&[]model.File{{PicInfo: "1,1", Policy: model.Policy{Type: "mock"}}}) +// fs.FileTarget[0].Policy.ID = 1 +// fs.Handler = testHandller2 +// res, err := fs.GetThumb(context.Background(), 1) +// asserts.NoError(err) +// asserts.EqualValues(50, res.MaxAge) +// } +//} func TestFileSystem_ThumbWorker(t *testing.T) { asserts := assert.New(t) diff --git a/pkg/filesystem/manage_test.go b/pkg/filesystem/manage_test.go index f3186ef7..73573714 100644 --- a/pkg/filesystem/manage_test.go +++ b/pkg/filesystem/manage_test.go @@ -11,59 +11,57 @@ import ( "github.com/cloudreve/Cloudreve/v3/pkg/cache" "github.com/cloudreve/Cloudreve/v3/pkg/conf" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" - "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response" "github.com/cloudreve/Cloudreve/v3/pkg/serializer" "github.com/cloudreve/Cloudreve/v3/pkg/util" "github.com/jinzhu/gorm" "github.com/stretchr/testify/assert" - testMock "github.com/stretchr/testify/mock" ) -func TestFileSystem_ListPhysical(t *testing.T) { - asserts := assert.New(t) - fs := &FileSystem{ - User: &model.User{ - Model: gorm.Model{ - ID: 1, - }, - }, - Policy: &model.Policy{Type: "mock"}, - } - ctx := context.Background() - - // 未知存储策略 - { - fs.Policy.Type = "unknown" - res, err := fs.ListPhysical(ctx, "/") - asserts.Equal(ErrUnknownPolicyType, err) - asserts.Empty(res) - fs.Policy.Type = "mock" - } - - // 无法列取目录 - { - testHandler := new(FileHeaderMock) - testHandler.On("List", testMock.Anything, "/", testMock.Anything).Return([]response.Object{}, errors.New("error")) - fs.Handler = testHandler - res, err := fs.ListPhysical(ctx, "/") - asserts.EqualError(err, "error") - asserts.Empty(res) - } - - // 成功 - { - testHandler := new(FileHeaderMock) - testHandler.On("List", testMock.Anything, "/", testMock.Anything).Return( - []response.Object{{IsDir: true, Name: "1"}, {IsDir: false, Name: "2"}}, - nil, - ) - fs.Handler = testHandler - res, err := fs.ListPhysical(ctx, "/") - asserts.NoError(err) - asserts.Len(res, 1) - asserts.Equal("1", res[0].Name) - } -} +//func TestFileSystem_ListPhysical(t *testing.T) { +// asserts := assert.New(t) +// fs := &FileSystem{ +// User: &model.User{ +// Model: gorm.Model{ +// ID: 1, +// }, +// }, +// Policy: &model.Policy{Type: "mock"}, +// } +// ctx := context.Background() +// +// // 未知存储策略 +// { +// fs.Policy.Type = "unknown" +// res, err := fs.ListPhysical(ctx, "/") +// asserts.Equal(ErrUnknownPolicyType, err) +// asserts.Empty(res) +// fs.Policy.Type = "mock" +// } +// +// // 无法列取目录 +// { +// testHandler := new(FileHeaderMock) +// testHandler.On("List", testMock.Anything, "/", testMock.Anything).Return([]response.Object{}, errors.New("error")) +// fs.Handler = testHandler +// res, err := fs.ListPhysical(ctx, "/") +// asserts.EqualError(err, "error") +// asserts.Empty(res) +// } +// +// // 成功 +// { +// testHandler := new(FileHeaderMock) +// testHandler.On("List", testMock.Anything, "/", testMock.Anything).Return( +// []response.Object{{IsDir: true, Name: "1"}, {IsDir: false, Name: "2"}}, +// nil, +// ) +// fs.Handler = testHandler +// res, err := fs.ListPhysical(ctx, "/") +// asserts.NoError(err) +// asserts.Len(res, 1) +// asserts.Equal("1", res[0].Name) +// } +//} func TestFileSystem_List(t *testing.T) { asserts := assert.New(t) diff --git a/pkg/filesystem/upload_test.go b/pkg/filesystem/upload_test.go index 74f7abe3..1f1b9637 100644 --- a/pkg/filesystem/upload_test.go +++ b/pkg/filesystem/upload_test.go @@ -2,30 +2,32 @@ package filesystem import ( "context" - "errors" - "io" - "io/ioutil" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "testing" - - model "github.com/cloudreve/Cloudreve/v3/models" - "github.com/cloudreve/Cloudreve/v3/pkg/cache" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response" "github.com/cloudreve/Cloudreve/v3/pkg/serializer" - "github.com/gin-gonic/gin" - "github.com/jinzhu/gorm" - "github.com/stretchr/testify/assert" testMock "github.com/stretchr/testify/mock" + "net/url" ) type FileHeaderMock struct { testMock.Mock } +func (m FileHeaderMock) Put(ctx context.Context, file fsctx.FileHeader) error { + args := m.Called(ctx, file) + return args.Error(0) +} + +func (m FileHeaderMock) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) { + args := m.Called(ctx, ttl, uploadSession, file) + return args.Get(0).(*serializer.UploadCredential), args.Error(1) +} + +func (m FileHeaderMock) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error { + args := m.Called(ctx, uploadSession) + return args.Error(0) +} + func (m FileHeaderMock) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) { args := m.Called(ctx, path, recursive) return args.Get(0).([]response.Object), args.Error(1) @@ -36,11 +38,6 @@ func (m FileHeaderMock) Get(ctx context.Context, path string) (response.RSCloser return args.Get(0).(response.RSCloser), args.Error(1) } -func (m FileHeaderMock) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error { - args := m.Called(ctx, file, dst) - return args.Error(0) -} - func (m FileHeaderMock) Delete(ctx context.Context, files []string) ([]string, error) { args := m.Called(ctx, files) return args.Get(0).([]string), args.Error(1) @@ -56,182 +53,177 @@ func (m FileHeaderMock) Source(ctx context.Context, path string, url url.URL, ex return args.Get(0).(string), args.Error(1) } -func (m FileHeaderMock) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) { - args := m.Called(ctx, ttl, uploadSession) - return args.Get(0).(serializer.UploadCredential), args.Error(1) -} - -func TestFileSystem_Upload(t *testing.T) { - asserts := assert.New(t) - - // 正常 - testHandler := new(FileHeaderMock) - testHandler.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(nil) - fs := &FileSystem{ - Handler: testHandler, - User: &model.User{ - Model: gorm.Model{ - ID: 1, - }, - Policy: model.Policy{ - AutoRename: false, - DirNameRule: "{path}", - }, - }, - } - ctx, cancel := context.WithCancel(context.Background()) - c, _ := gin.CreateTestContext(httptest.NewRecorder()) - c.Request, _ = http.NewRequest("POST", "/", nil) - ctx = context.WithValue(ctx, fsctx.GinCtx, c) - cancel() - file := fsctx.FileStream{ - Size: 5, - VirtualPath: "/", - Name: "1.txt", - } - err := fs.Upload(ctx, file) - asserts.NoError(err) - - // 正常，上下文已指定源文件 - testHandler = new(FileHeaderMock) - testHandler.On("Put", testMock.Anything, testMock.Anything, "123/123.txt").Return(nil) - fs = &FileSystem{ - Handler: testHandler, - User: &model.User{ - Model: gorm.Model{ - ID: 1, - }, - Policy: model.Policy{ - AutoRename: false, - DirNameRule: "{path}", - }, - }, - } - ctx, cancel = context.WithCancel(context.Background()) - c, _ = gin.CreateTestContext(httptest.NewRecorder()) - c.Request, _ = http.NewRequest("POST", "/", nil) - ctx = context.WithValue(ctx, fsctx.GinCtx, c) - ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{SourceName: "123/123.txt"}) - cancel() - file = fsctx.FileStream{ - Size: 5, - VirtualPath: "/", - Name: "1.txt", - File: ioutil.NopCloser(strings.NewReader("")), - } - err = fs.Upload(ctx, file) - asserts.NoError(err) - - // BeforeUpload 返回错误 - fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem) error { - return errors.New("error") - }) - err = fs.Upload(ctx, file) - asserts.Error(err) - fs.Hooks["BeforeUpload"] = nil - testHandler.AssertExpectations(t) - - // 上传文件失败 - testHandler2 := new(FileHeaderMock) - testHandler2.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(errors.New("error")) - fs.Handler = testHandler2 - err = fs.Upload(ctx, file) - asserts.Error(err) - testHandler2.AssertExpectations(t) - - // AfterUpload失败 - testHandler3 := new(FileHeaderMock) - testHandler3.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(nil) - fs.Handler = testHandler3 - fs.Use("AfterUpload", func(ctx context.Context, fs *FileSystem) error { - return errors.New("error") - }) - fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem) error { - return errors.New("error") - }) - err = fs.Upload(ctx, file) - asserts.Error(err) - testHandler2.AssertExpectations(t) - -} - -func TestFileSystem_GenerateSavePath_Anonymous(t *testing.T) { - asserts := assert.New(t) - fs := FileSystem{User: &model.User{}} - ctx := context.WithValue( - context.Background(), - fsctx.UploadPolicyCtx, - serializer.UploadPolicy{ - SavePath: "{randomkey16}", - AutoRename: false, - }, - ) - - savePath := fs.GenerateSavePath(ctx, fsctx.FileStream{ - Name: "test.test", - }) - asserts.Len(savePath, 26) - asserts.Contains(savePath, "test.test") -} - -func TestFileSystem_GetUploadToken(t *testing.T) { - asserts := assert.New(t) - fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}} - ctx := context.Background() - - // 成功 - { - cache.SetSettings(map[string]string{ - "upload_credential_timeout": "10", - "upload_session_timeout": "10", - }, "setting_") - testHandler := new(FileHeaderMock) - testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{Token: "test"}, nil) - fs.Handler = testHandler - res, err := fs.CreateUploadSession(ctx, "/", 10, "123") - testHandler.AssertExpectations(t) - asserts.NoError(err) - asserts.Equal("test", res.Token) - } - - // 无法获取上传凭证 - { - cache.SetSettings(map[string]string{ - "upload_credential_timeout": "10", - "upload_session_timeout": "10", - }, "setting_") - testHandler := new(FileHeaderMock) - testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{}, errors.New("error")) - fs.Handler = testHandler - _, err := fs.CreateUploadSession(ctx, "/", 10, "123") - testHandler.AssertExpectations(t) - asserts.Error(err) - } -} - -func TestFileSystem_UploadFromStream(t *testing.T) { - asserts := assert.New(t) - fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}} - ctx := context.Background() - - err := fs.UploadFromStream(ctx, ioutil.NopCloser(strings.NewReader("123")), "/1.txt", 1) - asserts.Error(err) -} - -func TestFileSystem_UploadFromPath(t *testing.T) { - asserts := assert.New(t) - fs := FileSystem{User: &model.User{Policy: model.Policy{Type: "mock"}, Model: gorm.Model{ID: 1}}} - ctx := context.Background() - - // 文件不存在 - { - err := fs.UploadFromPath(ctx, "test/not_exist", "/", true) - asserts.Error(err) - } - - // 文存在,上传失败 - { - err := fs.UploadFromPath(ctx, "tests/test.zip", "/", true) - asserts.Error(err) - } -} +//func TestFileSystem_Upload(t *testing.T) { +// asserts := assert.New(t) +// +// // 正常 +// testHandler := new(FileHeaderMock) +// testHandler.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(nil) +// fs := &FileSystem{ +// Handler: testHandler, +// User: &model.User{ +// Model: gorm.Model{ +// ID: 1, +// }, +// Policy: model.Policy{ +// AutoRename: false, +// DirNameRule: "{path}", +// }, +// }, +// } +// ctx, cancel := context.WithCancel(context.Background()) +// c, _ := gin.CreateTestContext(httptest.NewRecorder()) +// c.Request, _ = http.NewRequest("POST", "/", nil) +// ctx = context.WithValue(ctx, fsctx.GinCtx, c) +// cancel() +// file := fsctx.FileStream{ +// Size: 5, +// VirtualPath: "/", +// Name: "1.txt", +// } +// err := fs.Upload(ctx, file) +// asserts.NoError(err) +// +// // 正常，上下文已指定源文件 +// testHandler = new(FileHeaderMock) +// testHandler.On("Put", testMock.Anything, testMock.Anything, "123/123.txt").Return(nil) +// fs = &FileSystem{ +// Handler: testHandler, +// User: &model.User{ +// Model: gorm.Model{ +// ID: 1, +// }, +// Policy: model.Policy{ +// AutoRename: false, +// DirNameRule: "{path}", +// }, +// }, +// } +// ctx, cancel = context.WithCancel(context.Background()) +// c, _ = gin.CreateTestContext(httptest.NewRecorder()) +// c.Request, _ = http.NewRequest("POST", "/", nil) +// ctx = context.WithValue(ctx, fsctx.GinCtx, c) +// ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{SourceName: "123/123.txt"}) +// cancel() +// file = fsctx.FileStream{ +// Size: 5, +// VirtualPath: "/", +// Name: "1.txt", +// File: ioutil.NopCloser(strings.NewReader("")), +// } +// err = fs.Upload(ctx, file) +// asserts.NoError(err) +// +// // BeforeUpload 返回错误 +// fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem) error { +// return errors.New("error") +// }) +// err = fs.Upload(ctx, file) +// asserts.Error(err) +// fs.Hooks["BeforeUpload"] = nil +// testHandler.AssertExpectations(t) +// +// // 上传文件失败 +// testHandler2 := new(FileHeaderMock) +// testHandler2.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(errors.New("error")) +// fs.Handler = testHandler2 +// err = fs.Upload(ctx, file) +// asserts.Error(err) +// testHandler2.AssertExpectations(t) +// +// // AfterUpload失败 +// testHandler3 := new(FileHeaderMock) +// testHandler3.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(nil) +// fs.Handler = testHandler3 +// fs.Use("AfterUpload", func(ctx context.Context, fs *FileSystem) error { +// return errors.New("error") +// }) +// fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem) error { +// return errors.New("error") +// }) +// err = fs.Upload(ctx, file) +// asserts.Error(err) +// testHandler2.AssertExpectations(t) +// +//} +// +//func TestFileSystem_GenerateSavePath_Anonymous(t *testing.T) { +// asserts := assert.New(t) +// fs := FileSystem{User: &model.User{}} +// ctx := context.WithValue( +// context.Background(), +// fsctx.UploadPolicyCtx, +// serializer.UploadPolicy{ +// SavePath: "{randomkey16}", +// AutoRename: false, +// }, +// ) +// +// savePath := fs.GenerateSavePath(ctx, fsctx.FileStream{ +// Name: "test.test", +// }) +// asserts.Len(savePath, 26) +// asserts.Contains(savePath, "test.test") +//} +// +//func TestFileSystem_GetUploadToken(t *testing.T) { +// asserts := assert.New(t) +// fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}} +// ctx := context.Background() +// +// // 成功 +// { +// cache.SetSettings(map[string]string{ +// "upload_credential_timeout": "10", +// "upload_session_timeout": "10", +// }, "setting_") +// testHandler := new(FileHeaderMock) +// testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{Token: "test"}, nil) +// fs.Handler = testHandler +// res, err := fs.CreateUploadSession(ctx, "/", 10, "123") +// testHandler.AssertExpectations(t) +// asserts.NoError(err) +// asserts.Equal("test", res.Token) +// } +// +// // 无法获取上传凭证 +// { +// cache.SetSettings(map[string]string{ +// "upload_credential_timeout": "10", +// "upload_session_timeout": "10", +// }, "setting_") +// testHandler := new(FileHeaderMock) +// testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{}, errors.New("error")) +// fs.Handler = testHandler +// _, err := fs.CreateUploadSession(ctx, "/", 10, "123") +// testHandler.AssertExpectations(t) +// asserts.Error(err) +// } +//} +// +//func TestFileSystem_UploadFromStream(t *testing.T) { +// asserts := assert.New(t) +// fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}} +// ctx := context.Background() +// +// err := fs.UploadFromStream(ctx, ioutil.NopCloser(strings.NewReader("123")), "/1.txt", 1) +// asserts.Error(err) +//} +// +//func TestFileSystem_UploadFromPath(t *testing.T) { +// asserts := assert.New(t) +// fs := FileSystem{User: &model.User{Policy: model.Policy{Type: "mock"}, Model: gorm.Model{ID: 1}}} +// ctx := context.Background() +// +// // 文件不存在 +// { +// err := fs.UploadFromPath(ctx, "test/not_exist", "/", true) +// asserts.Error(err) +// } +// +// // 文存在,上传失败 +// { +// err := fs.UploadFromPath(ctx, "tests/test.zip", "/", true) +// asserts.Error(err) +// } +//} diff --git a/pkg/mocks/remoteclientmock/mock.go b/pkg/mocks/remoteclientmock/mock.go new file mode 100644 index 00000000..dee47352 --- /dev/null +++ b/pkg/mocks/remoteclientmock/mock.go @@ -0,0 +1,32 @@ +package remoteclientmock + +import ( + "context" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" + "github.com/cloudreve/Cloudreve/v3/pkg/serializer" + "github.com/stretchr/testify/mock" +) + +type RemoteClientMock struct { + mock.Mock +} + +func (r *RemoteClientMock) CreateUploadSession(ctx context.Context, session *serializer.UploadSession, ttl int64) error { + return r.Called(ctx, session, ttl).Error(0) +} + +func (r *RemoteClientMock) GetUploadURL(ttl int64, sessionID string) (string, string, error) { + args := r.Called(ttl, sessionID) + + return args.String(0), args.String(1), args.Error(2) +} + +func (r *RemoteClientMock) Upload(ctx context.Context, file fsctx.FileHeader) error { + args := r.Called(ctx, file) + return args.Error(0) +} + +func (r *RemoteClientMock) DeleteUploadSession(ctx context.Context, sessionID string) error { + args := r.Called(ctx, sessionID) + return args.Error(0) +} diff --git a/pkg/mocks/requestmock/request.go b/pkg/mocks/requestmock/request.go index 41581b2e..7e6ca1b1 100644 --- a/pkg/mocks/requestmock/request.go +++ b/pkg/mocks/requestmock/request.go @@ -1,4 +1,4 @@ -package controllermock +package requestmock import ( "github.com/cloudreve/Cloudreve/v3/pkg/request" From d117080991831628257a12c48f9460d4071d8a20 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Wed, 30 Mar 2022 20:38:02 +0800 Subject: [PATCH 20/21] Test: new changes pkg filesystem --- models/defaults.go | 1 - pkg/conf/conf.go | 1 + pkg/filesystem/chunk/chunk_test.go | 2 +- pkg/filesystem/driver/local/handler_test.go | 8 +- pkg/filesystem/driver/oss/handler.go | 2 +- pkg/filesystem/driver/qiniu/handler.go | 2 +- pkg/filesystem/image_test.go | 80 ++-- pkg/filesystem/manage_test.go | 186 +++++----- pkg/filesystem/upload_test.go | 383 +++++++++++--------- pkg/filesystem/validator_test.go | 22 +- 10 files changed, 385 insertions(+), 302 deletions(-) diff --git a/models/defaults.go b/models/defaults.go index ecb84280..16a798b4 100644 --- a/models/defaults.go +++ b/models/defaults.go @@ -31,7 +31,6 @@ var defaultSettings = []Setting{ {Name: "download_timeout", Value: `60`, Type: "timeout"}, {Name: "preview_timeout", Value: `60`, Type: "timeout"}, {Name: "doc_preview_timeout", Value: `60`, Type: "timeout"}, - {Name: "upload_credential_timeout", Value: `1800`, Type: "timeout"}, {Name: "upload_session_timeout", Value: `86400`, Type: "timeout"}, {Name: "slave_api_timeout", Value: `60`, Type: "timeout"}, {Name: "slave_node_retry", Value: `3`, Type: "slave"}, diff --git a/pkg/conf/conf.go b/pkg/conf/conf.go index 4988ef34..35ed8b6a 100644 --- a/pkg/conf/conf.go +++ b/pkg/conf/conf.go @@ -65,6 +65,7 @@ type cors struct { var cfg *ini.File const defaultConf = `[System] +Debug = false Mode = master Listen = :5212 SessionSecret = {SessionSecret} diff --git a/pkg/filesystem/chunk/chunk_test.go b/pkg/filesystem/chunk/chunk_test.go index c6af9d93..4bdcd06d 100644 --- a/pkg/filesystem/chunk/chunk_test.go +++ b/pkg/filesystem/chunk/chunk_test.go @@ -245,6 +245,6 @@ func TestChunkGroup_Process(t *testing.T) { return errors.New("error") })) a.False(c.Next()) - a.Equal(1, count) + a.Equal(4, count) } } diff --git a/pkg/filesystem/driver/local/handler_test.go b/pkg/filesystem/driver/local/handler_test.go index 9ce5fe74..9167e82f 100644 --- a/pkg/filesystem/driver/local/handler_test.go +++ b/pkg/filesystem/driver/local/handler_test.go @@ -81,12 +81,12 @@ func TestHandler_Delete(t *testing.T) { asserts := assert.New(t) handler := Driver{} ctx := context.Background() - filePath := util.RelativePath("test.file") + filePath := util.RelativePath("TestHandler_Delete.file") file, err := os.Create(filePath) asserts.NoError(err) _ = file.Close() - list, err := handler.Delete(ctx, []string{"test.file"}) + list, err := handler.Delete(ctx, []string{"TestHandler_Delete.file"}) asserts.Equal([]string{}, list) asserts.NoError(err) @@ -94,7 +94,7 @@ func TestHandler_Delete(t *testing.T) { _ = file.Close() file, _ = os.OpenFile(filePath, os.O_RDWR, os.FileMode(0)) asserts.NoError(err) - list, err = handler.Delete(ctx, []string{"test.file", "test.notexist"}) + list, err = handler.Delete(ctx, []string{"TestHandler_Delete.file", "test.notexist"}) file.Close() asserts.Equal([]string{}, list) asserts.NoError(err) @@ -105,7 +105,7 @@ func TestHandler_Delete(t *testing.T) { file, err = os.Create(filePath) asserts.NoError(err) - list, err = handler.Delete(ctx, []string{"test.file"}) + list, err = handler.Delete(ctx, []string{"TestHandler_Delete.file"}) _ = file.Close() asserts.Equal([]string{}, list) asserts.NoError(err) diff --git a/pkg/filesystem/driver/oss/handler.go b/pkg/filesystem/driver/oss/handler.go index c7eadc23..b0b55d29 100644 --- a/pkg/filesystem/driver/oss/handler.go +++ b/pkg/filesystem/driver/oss/handler.go @@ -229,7 +229,7 @@ func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error { fileInfo := file.Info() // 凭证有效期 - credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600) + credentialTTL := model.GetIntSetting("upload_session_timeout", 3600) // 是否允许覆盖 overwrite := fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite diff --git a/pkg/filesystem/driver/qiniu/handler.go b/pkg/filesystem/driver/qiniu/handler.go index a47b8da4..22a97d06 100644 --- a/pkg/filesystem/driver/qiniu/handler.go +++ b/pkg/filesystem/driver/qiniu/handler.go @@ -156,7 +156,7 @@ func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error { defer file.Close() // 凭证有效期 - credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600) + credentialTTL := model.GetIntSetting("upload_session_timeout", 3600) // 生成上传策略 fileInfo := file.Info() diff --git a/pkg/filesystem/image_test.go b/pkg/filesystem/image_test.go index 42c7f456..cc33b190 100644 --- a/pkg/filesystem/image_test.go +++ b/pkg/filesystem/image_test.go @@ -1,38 +1,45 @@ package filesystem import ( + "context" + "errors" + model "github.com/cloudreve/Cloudreve/v3/models" + "github.com/cloudreve/Cloudreve/v3/pkg/cache" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response" + "github.com/cloudreve/Cloudreve/v3/pkg/request" + testMock "github.com/stretchr/testify/mock" "testing" "github.com/stretchr/testify/assert" ) -//func TestFileSystem_GetThumb(t *testing.T) { -// asserts := assert.New(t) -// fs := &FileSystem{User: &model.User{}} -// -// // 非图像文件 -// { -// fs.SetTargetFile(&[]model.File{{}}) -// _, err := fs.GetThumb(context.Background(), 1) -// asserts.Equal(err, ErrObjectNotExist) -// } -// -// // 成功 -// { -// cache.Set("setting_thumb_width", "10", 0) -// cache.Set("setting_thumb_height", "10", 0) -// cache.Set("setting_preview_timeout", "50", 0) -// testHandller2 := new(FileHeaderMock) -// testHandller2.On("Thumb", testMock.Anything, "").Return(&response.ContentResponse{}, nil) -// fs.CleanTargets() -// fs.SetTargetFile(&[]model.File{{PicInfo: "1,1", Policy: model.Policy{Type: "mock"}}}) -// fs.FileTarget[0].Policy.ID = 1 -// fs.Handler = testHandller2 -// res, err := fs.GetThumb(context.Background(), 1) -// asserts.NoError(err) -// asserts.EqualValues(50, res.MaxAge) -// } -//} +func TestFileSystem_GetThumb(t *testing.T) { + asserts := assert.New(t) + fs := &FileSystem{User: &model.User{}} + + // 非图像文件 + { + fs.SetTargetFile(&[]model.File{{}}) + _, err := fs.GetThumb(context.Background(), 1) + asserts.Equal(err, ErrObjectNotExist) + } + + // 成功 + { + cache.Set("setting_thumb_width", "10", 0) + cache.Set("setting_thumb_height", "10", 0) + cache.Set("setting_preview_timeout", "50", 0) + testHandller2 := new(FileHeaderMock) + testHandller2.On("Thumb", testMock.Anything, "").Return(&response.ContentResponse{}, nil) + fs.CleanTargets() + fs.SetTargetFile(&[]model.File{{PicInfo: "1,1", Policy: model.Policy{Type: "mock"}}}) + fs.FileTarget[0].Policy.ID = 1 + fs.Handler = testHandller2 + res, err := fs.GetThumb(context.Background(), 1) + asserts.NoError(err) + asserts.EqualValues(50, res.MaxAge) + } +} func TestFileSystem_ThumbWorker(t *testing.T) { asserts := assert.New(t) @@ -42,3 +49,22 @@ func TestFileSystem_ThumbWorker(t *testing.T) { getThumbWorker().releaseWorker() }) } + +func TestFileSystem_GenerateThumbnail(t *testing.T) { + fs := &FileSystem{User: &model.User{}} + + // 无法生成缩略图 + { + fs.SetTargetFile(&[]model.File{{}}) + fs.GenerateThumbnail(context.Background(), &model.File{}) + } + + // 无法获取文件数据 + { + testHandller := new(FileHeaderMock) + testHandller.On("Get", testMock.Anything, "").Return(request.NopRSCloser{}, errors.New("error")) + fs.Handler = testHandller + fs.GenerateThumbnail(context.Background(), &model.File{Name: "test.png"}) + testHandller.AssertExpectations(t) + } +} diff --git a/pkg/filesystem/manage_test.go b/pkg/filesystem/manage_test.go index 73573714..da91691c 100644 --- a/pkg/filesystem/manage_test.go +++ b/pkg/filesystem/manage_test.go @@ -3,6 +3,8 @@ package filesystem import ( "context" "errors" + "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response" + testMock "github.com/stretchr/testify/mock" "os" "testing" @@ -17,51 +19,51 @@ import ( "github.com/stretchr/testify/assert" ) -//func TestFileSystem_ListPhysical(t *testing.T) { -// asserts := assert.New(t) -// fs := &FileSystem{ -// User: &model.User{ -// Model: gorm.Model{ -// ID: 1, -// }, -// }, -// Policy: &model.Policy{Type: "mock"}, -// } -// ctx := context.Background() -// -// // 未知存储策略 -// { -// fs.Policy.Type = "unknown" -// res, err := fs.ListPhysical(ctx, "/") -// asserts.Equal(ErrUnknownPolicyType, err) -// asserts.Empty(res) -// fs.Policy.Type = "mock" -// } -// -// // 无法列取目录 -// { -// testHandler := new(FileHeaderMock) -// testHandler.On("List", testMock.Anything, "/", testMock.Anything).Return([]response.Object{}, errors.New("error")) -// fs.Handler = testHandler -// res, err := fs.ListPhysical(ctx, "/") -// asserts.EqualError(err, "error") -// asserts.Empty(res) -// } -// -// // 成功 -// { -// testHandler := new(FileHeaderMock) -// testHandler.On("List", testMock.Anything, "/", testMock.Anything).Return( -// []response.Object{{IsDir: true, Name: "1"}, {IsDir: false, Name: "2"}}, -// nil, -// ) -// fs.Handler = testHandler -// res, err := fs.ListPhysical(ctx, "/") -// asserts.NoError(err) -// asserts.Len(res, 1) -// asserts.Equal("1", res[0].Name) -// } -//} +func TestFileSystem_ListPhysical(t *testing.T) { + asserts := assert.New(t) + fs := &FileSystem{ + User: &model.User{ + Model: gorm.Model{ + ID: 1, + }, + }, + Policy: &model.Policy{Type: "mock"}, + } + ctx := context.Background() + + // 未知存储策略 + { + fs.Policy.Type = "unknown" + res, err := fs.ListPhysical(ctx, "/") + asserts.Equal(ErrUnknownPolicyType, err) + asserts.Empty(res) + fs.Policy.Type = "mock" + } + + // 无法列取目录 + { + testHandler := new(FileHeaderMock) + testHandler.On("List", testMock.Anything, "/", testMock.Anything).Return([]response.Object{}, errors.New("error")) + fs.Handler = testHandler + res, err := fs.ListPhysical(ctx, "/") + asserts.EqualError(err, "error") + asserts.Empty(res) + } + + // 成功 + { + testHandler := new(FileHeaderMock) + testHandler.On("List", testMock.Anything, "/", testMock.Anything).Return( + []response.Object{{IsDir: true, Name: "1"}, {IsDir: false, Name: "2"}}, + nil, + ) + fs.Handler = testHandler + res, err := fs.ListPhysical(ctx, "/") + asserts.NoError(err) + asserts.Len(res, 1) + asserts.Equal("1", res[0].Name) + } +} func TestFileSystem_List(t *testing.T) { asserts := assert.New(t) @@ -294,12 +296,12 @@ func TestFileSystem_ListDeleteDirs(t *testing.T) { { mock.ExpectQuery("SELECT(.+)"). WillReturnRows( - sqlmock.NewRows([]string{"id"}). - AddRow(1). - AddRow(2). - AddRow(3), + sqlmock.NewRows([]string{"id", "parent_id"}). + AddRow(1, 0). + AddRow(2, 0). + AddRow(3, 0), ) - mock.ExpectQuery("SELECT(.+)"). + mock.ExpectQuery("SELECT(.+)files(.+)"). WithArgs(1, 2, 3). WillReturnRows( sqlmock.NewRows([]string{"id", "name"}). @@ -314,21 +316,47 @@ func TestFileSystem_ListDeleteDirs(t *testing.T) { asserts.NoError(mock.ExpectationsWereMet()) } + // 成功,忽略根目录 + { + mock.ExpectQuery("SELECT(.+)"). + WillReturnRows( + sqlmock.NewRows([]string{"id", "parent_id"}). + AddRow(1, 0). + AddRow(2, nil). + AddRow(3, 0), + ) + mock.ExpectQuery("SELECT(.+)files(.+)"). + WithArgs(1, 3). + WillReturnRows( + sqlmock.NewRows([]string{"id", "name"}). + AddRow(4, "1.txt"). + AddRow(5, "2.txt"). + AddRow(6, "3.txt"), + ) + fs.CleanTargets() + err := fs.ListDeleteDirs(context.Background(), []uint{1}) + asserts.NoError(err) + asserts.Len(fs.FileTarget, 3) + asserts.Len(fs.DirTarget, 2) + asserts.NoError(mock.ExpectationsWereMet()) + } + // 检索文件发生错误 { mock.ExpectQuery("SELECT(.+)"). WillReturnRows( - sqlmock.NewRows([]string{"id"}). - AddRow(1). - AddRow(2). - AddRow(3), + sqlmock.NewRows([]string{"id", "parent_id"}). + AddRow(1, 0). + AddRow(2, 0). + AddRow(3, 0), ) mock.ExpectQuery("SELECT(.+)"). WithArgs(1, 2, 3). WillReturnError(errors.New("error")) + fs.CleanTargets() err := fs.ListDeleteDirs(context.Background(), []uint{1}) asserts.Error(err) - asserts.Len(fs.DirTarget, 6) + asserts.Len(fs.DirTarget, 3) asserts.NoError(mock.ExpectationsWereMet()) } // 检索目录发生错误 @@ -347,7 +375,7 @@ func TestFileSystem_Delete(t *testing.T) { cache.Set("pack_size_1", uint64(0), 0) fs := &FileSystem{User: &model.User{ Model: gorm.Model{ - ID: 1, + ID: 0, }, Storage: 3, Group: model.Group{MaxStorage: 3}, @@ -359,10 +387,10 @@ func TestFileSystem_Delete(t *testing.T) { fs.CleanTargets() mock.ExpectQuery("SELECT(.+)"). WillReturnRows( - sqlmock.NewRows([]string{"id"}). - AddRow(1). - AddRow(2). - AddRow(3), + sqlmock.NewRows([]string{"id", "parent_id"}). + AddRow(1, 0). + AddRow(2, 0). + AddRow(3, 0), ) mock.ExpectQuery("SELECT(.+)"). WithArgs(1, 2, 3). @@ -378,18 +406,16 @@ func TestFileSystem_Delete(t *testing.T) { // 删除文件记录 mock.ExpectBegin() mock.ExpectExec("DELETE(.+)"). - WillReturnResult(sqlmock.NewResult(0, 3)) + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("DELETE(.+)"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("UPDATE(.+)users(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() // 删除对应分享 mock.ExpectBegin() mock.ExpectExec("UPDATE(.+)shares"). WillReturnResult(sqlmock.NewResult(0, 3)) mock.ExpectCommit() - // 归还容量 - mock.ExpectBegin() - mock.ExpectExec("UPDATE(.+)"). - WillReturnResult(sqlmock.NewResult(0, 3)) - mock.ExpectCommit() // 删除目录 mock.ExpectBegin() mock.ExpectExec("DELETE(.+)"). @@ -405,7 +431,6 @@ func TestFileSystem_Delete(t *testing.T) { fs.DirTarget = []model.Folder{} err := fs.Delete(ctx, []uint{1}, []uint{1}, true) asserts.NoError(err) - asserts.Equal(uint64(0), fs.User.Storage) } //全部成功 { @@ -417,10 +442,10 @@ func TestFileSystem_Delete(t *testing.T) { asserts.NoError(err) mock.ExpectQuery("SELECT(.+)"). WillReturnRows( - sqlmock.NewRows([]string{"id"}). - AddRow(1). - AddRow(2). - AddRow(3), + sqlmock.NewRows([]string{"id", "parent_id"}). + AddRow(1, 0). + AddRow(2, 0). + AddRow(3, 0), ) mock.ExpectQuery("SELECT(.+)"). WithArgs(1, 2, 3). @@ -436,18 +461,16 @@ func TestFileSystem_Delete(t *testing.T) { // 删除文件记录 mock.ExpectBegin() mock.ExpectExec("DELETE(.+)"). - WillReturnResult(sqlmock.NewResult(0, 3)) + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("DELETE(.+)"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("UPDATE(.+)users(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() // 删除对应分享 mock.ExpectBegin() mock.ExpectExec("UPDATE(.+)shares"). WillReturnResult(sqlmock.NewResult(0, 3)) mock.ExpectCommit() - // 归还容量 - mock.ExpectBegin() - mock.ExpectExec("UPDATE(.+)"). - WillReturnResult(sqlmock.NewResult(0, 3)) - mock.ExpectCommit() // 删除目录 mock.ExpectBegin() mock.ExpectExec("DELETE(.+)"). @@ -463,7 +486,6 @@ func TestFileSystem_Delete(t *testing.T) { fs.DirTarget = []model.Folder{} err = fs.Delete(ctx, []uint{1}, []uint{1}, false) asserts.NoError(err) - asserts.Equal(uint64(0), fs.User.Storage) } } @@ -571,7 +593,9 @@ func TestFileSystem_Rename(t *testing.T) { Model: gorm.Model{ ID: 1, }, - }} + }, + Policy: &model.Policy{}, + } ctx := context.Background() // 重命名文件 成功 @@ -681,7 +705,7 @@ func TestFileSystem_Rename(t *testing.T) { // 新名字是文件，扩展名不合法 { - fs.User.Policy.OptionsSerialized.FileType = []string{"txt"} + fs.Policy.OptionsSerialized.FileType = []string{"txt"} err := fs.Rename(ctx, []uint{}, []uint{10}, "1.jpg") asserts.Error(err) asserts.Equal(ErrIllegalObjectName, err) @@ -689,7 +713,7 @@ func TestFileSystem_Rename(t *testing.T) { // 新名字是目录，不应该检测扩展名 { - fs.User.Policy.OptionsSerialized.FileType = []string{"txt"} + fs.Policy.OptionsSerialized.FileType = []string{"txt"} mock.ExpectQuery("SELECT(.+)folders(.+)"). WithArgs(10, 1). WillReturnRows(sqlmock.NewRows([]string{"id", "name"})) diff --git a/pkg/filesystem/upload_test.go b/pkg/filesystem/upload_test.go index 1f1b9637..30f24ccd 100644 --- a/pkg/filesystem/upload_test.go +++ b/pkg/filesystem/upload_test.go @@ -2,11 +2,23 @@ package filesystem import ( "context" + "errors" + "github.com/DATA-DOG/go-sqlmock" + model "github.com/cloudreve/Cloudreve/v3/models" + "github.com/cloudreve/Cloudreve/v3/pkg/cache" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response" "github.com/cloudreve/Cloudreve/v3/pkg/serializer" + "github.com/gin-gonic/gin" + "github.com/jinzhu/gorm" + "github.com/stretchr/testify/assert" testMock "github.com/stretchr/testify/mock" + "io/ioutil" + "net/http" + "net/http/httptest" "net/url" + "strings" + "testing" ) type FileHeaderMock struct { @@ -53,177 +65,200 @@ func (m FileHeaderMock) Source(ctx context.Context, path string, url url.URL, ex return args.Get(0).(string), args.Error(1) } -//func TestFileSystem_Upload(t *testing.T) { -// asserts := assert.New(t) -// -// // 正常 -// testHandler := new(FileHeaderMock) -// testHandler.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(nil) -// fs := &FileSystem{ -// Handler: testHandler, -// User: &model.User{ -// Model: gorm.Model{ -// ID: 1, -// }, -// Policy: model.Policy{ -// AutoRename: false, -// DirNameRule: "{path}", -// }, -// }, -// } -// ctx, cancel := context.WithCancel(context.Background()) -// c, _ := gin.CreateTestContext(httptest.NewRecorder()) -// c.Request, _ = http.NewRequest("POST", "/", nil) -// ctx = context.WithValue(ctx, fsctx.GinCtx, c) -// cancel() -// file := fsctx.FileStream{ -// Size: 5, -// VirtualPath: "/", -// Name: "1.txt", -// } -// err := fs.Upload(ctx, file) -// asserts.NoError(err) -// -// // 正常，上下文已指定源文件 -// testHandler = new(FileHeaderMock) -// testHandler.On("Put", testMock.Anything, testMock.Anything, "123/123.txt").Return(nil) -// fs = &FileSystem{ -// Handler: testHandler, -// User: &model.User{ -// Model: gorm.Model{ -// ID: 1, -// }, -// Policy: model.Policy{ -// AutoRename: false, -// DirNameRule: "{path}", -// }, -// }, -// } -// ctx, cancel = context.WithCancel(context.Background()) -// c, _ = gin.CreateTestContext(httptest.NewRecorder()) -// c.Request, _ = http.NewRequest("POST", "/", nil) -// ctx = context.WithValue(ctx, fsctx.GinCtx, c) -// ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{SourceName: "123/123.txt"}) -// cancel() -// file = fsctx.FileStream{ -// Size: 5, -// VirtualPath: "/", -// Name: "1.txt", -// File: ioutil.NopCloser(strings.NewReader("")), -// } -// err = fs.Upload(ctx, file) -// asserts.NoError(err) -// -// // BeforeUpload 返回错误 -// fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem) error { -// return errors.New("error") -// }) -// err = fs.Upload(ctx, file) -// asserts.Error(err) -// fs.Hooks["BeforeUpload"] = nil -// testHandler.AssertExpectations(t) -// -// // 上传文件失败 -// testHandler2 := new(FileHeaderMock) -// testHandler2.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(errors.New("error")) -// fs.Handler = testHandler2 -// err = fs.Upload(ctx, file) -// asserts.Error(err) -// testHandler2.AssertExpectations(t) -// -// // AfterUpload失败 -// testHandler3 := new(FileHeaderMock) -// testHandler3.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(nil) -// fs.Handler = testHandler3 -// fs.Use("AfterUpload", func(ctx context.Context, fs *FileSystem) error { -// return errors.New("error") -// }) -// fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem) error { -// return errors.New("error") -// }) -// err = fs.Upload(ctx, file) -// asserts.Error(err) -// testHandler2.AssertExpectations(t) -// -//} -// -//func TestFileSystem_GenerateSavePath_Anonymous(t *testing.T) { -// asserts := assert.New(t) -// fs := FileSystem{User: &model.User{}} -// ctx := context.WithValue( -// context.Background(), -// fsctx.UploadPolicyCtx, -// serializer.UploadPolicy{ -// SavePath: "{randomkey16}", -// AutoRename: false, -// }, -// ) -// -// savePath := fs.GenerateSavePath(ctx, fsctx.FileStream{ -// Name: "test.test", -// }) -// asserts.Len(savePath, 26) -// asserts.Contains(savePath, "test.test") -//} -// -//func TestFileSystem_GetUploadToken(t *testing.T) { -// asserts := assert.New(t) -// fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}} -// ctx := context.Background() -// -// // 成功 -// { -// cache.SetSettings(map[string]string{ -// "upload_credential_timeout": "10", -// "upload_session_timeout": "10", -// }, "setting_") -// testHandler := new(FileHeaderMock) -// testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{Token: "test"}, nil) -// fs.Handler = testHandler -// res, err := fs.CreateUploadSession(ctx, "/", 10, "123") -// testHandler.AssertExpectations(t) -// asserts.NoError(err) -// asserts.Equal("test", res.Token) -// } -// -// // 无法获取上传凭证 -// { -// cache.SetSettings(map[string]string{ -// "upload_credential_timeout": "10", -// "upload_session_timeout": "10", -// }, "setting_") -// testHandler := new(FileHeaderMock) -// testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{}, errors.New("error")) -// fs.Handler = testHandler -// _, err := fs.CreateUploadSession(ctx, "/", 10, "123") -// testHandler.AssertExpectations(t) -// asserts.Error(err) -// } -//} -// -//func TestFileSystem_UploadFromStream(t *testing.T) { -// asserts := assert.New(t) -// fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}} -// ctx := context.Background() -// -// err := fs.UploadFromStream(ctx, ioutil.NopCloser(strings.NewReader("123")), "/1.txt", 1) -// asserts.Error(err) -//} -// -//func TestFileSystem_UploadFromPath(t *testing.T) { -// asserts := assert.New(t) -// fs := FileSystem{User: &model.User{Policy: model.Policy{Type: "mock"}, Model: gorm.Model{ID: 1}}} -// ctx := context.Background() -// -// // 文件不存在 -// { -// err := fs.UploadFromPath(ctx, "test/not_exist", "/", true) -// asserts.Error(err) -// } -// -// // 文存在,上传失败 -// { -// err := fs.UploadFromPath(ctx, "tests/test.zip", "/", true) -// asserts.Error(err) -// } -//} +func TestFileSystem_Upload(t *testing.T) { + asserts := assert.New(t) + + // 正常 + testHandler := new(FileHeaderMock) + testHandler.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(nil) + fs := &FileSystem{ + Handler: testHandler, + User: &model.User{ + Model: gorm.Model{ + ID: 1, + }, + }, + Policy: &model.Policy{ + AutoRename: false, + DirNameRule: "{path}", + }, + } + ctx, cancel := context.WithCancel(context.Background()) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request, _ = http.NewRequest("POST", "/", nil) + ctx = context.WithValue(ctx, fsctx.GinCtx, c) + cancel() + file := &fsctx.FileStream{ + Size: 5, + VirtualPath: "/", + Name: "1.txt", + } + err := fs.Upload(ctx, file) + asserts.NoError(err) + + // 正常，上下文已指定源文件 + testHandler = new(FileHeaderMock) + testHandler.On("Put", testMock.Anything, testMock.Anything).Return(nil) + fs = &FileSystem{ + Handler: testHandler, + User: &model.User{ + Model: gorm.Model{ + ID: 1, + }, + }, + Policy: &model.Policy{ + AutoRename: false, + DirNameRule: "{path}", + }, + } + ctx, cancel = context.WithCancel(context.Background()) + c, _ = gin.CreateTestContext(httptest.NewRecorder()) + c.Request, _ = http.NewRequest("POST", "/", nil) + ctx = context.WithValue(ctx, fsctx.GinCtx, c) + ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{SourceName: "123/123.txt"}) + cancel() + file = &fsctx.FileStream{ + Size: 5, + VirtualPath: "/", + Name: "1.txt", + File: ioutil.NopCloser(strings.NewReader("")), + } + err = fs.Upload(ctx, file) + asserts.NoError(err) + + // BeforeUpload 返回错误 + fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error { + return errors.New("error") + }) + err = fs.Upload(ctx, file) + asserts.Error(err) + fs.Hooks["BeforeUpload"] = nil + testHandler.AssertExpectations(t) + + // 上传文件失败 + testHandler2 := new(FileHeaderMock) + testHandler2.On("Put", testMock.Anything, testMock.Anything).Return(errors.New("error")) + fs.Handler = testHandler2 + err = fs.Upload(ctx, file) + asserts.Error(err) + testHandler2.AssertExpectations(t) + + // AfterUpload失败 + testHandler3 := new(FileHeaderMock) + testHandler3.On("Put", testMock.Anything, testMock.Anything).Return(nil) + fs.Handler = testHandler3 + fs.Use("AfterUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error { + return errors.New("error") + }) + fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error { + return errors.New("error") + }) + err = fs.Upload(ctx, file) + asserts.Error(err) + testHandler2.AssertExpectations(t) + +} + +func TestFileSystem_GetUploadToken(t *testing.T) { + asserts := assert.New(t) + fs := FileSystem{ + User: &model.User{Model: gorm.Model{ID: 1}}, + Policy: &model.Policy{}, + } + ctx := context.Background() + + // 成功 + { + cache.SetSettings(map[string]string{ + "upload_session_timeout": "10", + }, "setting_") + testHandler := new(FileHeaderMock) + testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything, testMock.Anything).Return(&serializer.UploadCredential{Credential: "test"}, nil) + fs.Handler = testHandler + mock.ExpectQuery("SELECT(.+)"). + WithArgs(1). + WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1)) + mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found")) + mock.ExpectBegin() + mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + res, err := fs.CreateUploadSession(ctx, &fsctx.FileStream{ + Size: 0, + Name: "file", + VirtualPath: "/", + }) + asserts.NoError(mock.ExpectationsWereMet()) + testHandler.AssertExpectations(t) + asserts.NoError(err) + asserts.Equal("test", res.Credential) + } + + // 无法获取上传凭证 + { + cache.SetSettings(map[string]string{ + "upload_credential_timeout": "10", + "upload_session_timeout": "10", + }, "setting_") + testHandler := new(FileHeaderMock) + testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything, testMock.Anything).Return(&serializer.UploadCredential{}, errors.New("error")) + fs.Handler = testHandler + mock.ExpectQuery("SELECT(.+)"). + WithArgs(1). + WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1)) + mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found")) + mock.ExpectBegin() + mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + _, err := fs.CreateUploadSession(ctx, &fsctx.FileStream{ + Size: 0, + Name: "file", + VirtualPath: "/", + }) + asserts.NoError(mock.ExpectationsWereMet()) + testHandler.AssertExpectations(t) + asserts.Error(err) + } +} + +func TestFileSystem_UploadFromStream(t *testing.T) { + asserts := assert.New(t) + fs := FileSystem{ + User: &model.User{ + Model: gorm.Model{ID: 1}, + Policy: model.Policy{Type: "mock"}, + }, + Policy: &model.Policy{Type: "mock"}, + } + ctx := context.Background() + + err := fs.UploadFromStream(ctx, &fsctx.FileStream{ + File: ioutil.NopCloser(strings.NewReader("123")), + }, true) + asserts.Error(err) +} + +func TestFileSystem_UploadFromPath(t *testing.T) { + asserts := assert.New(t) + fs := FileSystem{ + User: &model.User{ + Model: gorm.Model{ID: 1}, + Policy: model.Policy{Type: "mock"}, + }, + Policy: &model.Policy{Type: "mock"}, + } + ctx := context.Background() + + // 文件不存在 + { + err := fs.UploadFromPath(ctx, "test/not_exist", "/", fsctx.Overwrite) + asserts.Error(err) + } + + // 文存在,上传失败 + { + err := fs.UploadFromPath(ctx, "tests/test.zip", "/", fsctx.Overwrite) + asserts.Error(err) + } +} diff --git a/pkg/filesystem/validator_test.go b/pkg/filesystem/validator_test.go index 8a39ae28..8f685f27 100644 --- a/pkg/filesystem/validator_test.go +++ b/pkg/filesystem/validator_test.go @@ -68,10 +68,9 @@ func TestFileSystem_ValidateFileSize(t *testing.T) { asserts := assert.New(t) ctx := context.Background() fs := FileSystem{ - User: &model.User{ - Policy: model.Policy{ - MaxSize: 10, - }, + User: &model.User{}, + Policy: &model.Policy{ + MaxSize: 10, }, } @@ -80,7 +79,7 @@ func TestFileSystem_ValidateFileSize(t *testing.T) { asserts.False(fs.ValidateFileSize(ctx, 11)) // 无限制 - fs.User.Policy.MaxSize = 0 + fs.Policy.MaxSize = 0 asserts.True(fs.ValidateFileSize(ctx, 11)) } @@ -88,11 +87,10 @@ func TestFileSystem_ValidateExtension(t *testing.T) { asserts := assert.New(t) ctx := context.Background() fs := FileSystem{ - User: &model.User{ - Policy: model.Policy{ - OptionsSerialized: model.PolicyOption{ - FileType: nil, - }, + User: &model.User{}, + Policy: &model.Policy{ + OptionsSerialized: model.PolicyOption{ + FileType: nil, }, }, } @@ -100,11 +98,11 @@ func TestFileSystem_ValidateExtension(t *testing.T) { asserts.True(fs.ValidateExtension(ctx, "1")) asserts.True(fs.ValidateExtension(ctx, "1.txt")) - fs.User.Policy.OptionsSerialized.FileType = []string{} + fs.Policy.OptionsSerialized.FileType = []string{} asserts.True(fs.ValidateExtension(ctx, "1")) asserts.True(fs.ValidateExtension(ctx, "1.txt")) - fs.User.Policy.OptionsSerialized.FileType = []string{"txt", "jpg"} + fs.Policy.OptionsSerialized.FileType = []string{"txt", "jpg"} asserts.False(fs.ValidateExtension(ctx, "1")) asserts.False(fs.ValidateExtension(ctx, "1.jpg.png")) asserts.True(fs.ValidateExtension(ctx, "1.txt")) From ec776ac837bce68729d79f4f82e0f4c2ef81b33c Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Thu, 31 Mar 2022 20:17:01 +0800 Subject: [PATCH 21/21] Test: new changes in pkg request, serializer, task, xml, router --- pkg/request/request_test.go | 2 + pkg/serializer/explorer_test.go | 15 +++ pkg/serializer/upload.go | 31 ------ pkg/serializer/upload_test.go | 63 ----------- pkg/serializer/user_test.go | 12 +-- pkg/task/import_test.go | 1 + pkg/webdav/internal/xml/xml.go | 4 +- routers/file_router_test.go | 180 -------------------------------- routers/router_test.go | 3 +- 9 files changed, 26 insertions(+), 285 deletions(-) create mode 100644 pkg/serializer/explorer_test.go delete mode 100644 pkg/serializer/upload_test.go delete mode 100644 routers/file_router_test.go diff --git a/pkg/request/request_test.go b/pkg/request/request_test.go index 7b1a48d8..1f00f7e8 100644 --- a/pkg/request/request_test.go +++ b/pkg/request/request_test.go @@ -82,6 +82,8 @@ func TestHTTPClient_Request(t *testing.T) { WithTimeout(time.Duration(1)*time.Microsecond), WithCredential(auth.HMACAuth{SecretKey: []byte("123")}, 10), WithContext(context.Background()), + WithoutHeader([]string{"s s", "s s"}), + WithMasterMeta(), ) asserts.Error(resp.Err) asserts.Nil(resp.Response) diff --git a/pkg/serializer/explorer_test.go b/pkg/serializer/explorer_test.go new file mode 100644 index 00000000..00c9efc9 --- /dev/null +++ b/pkg/serializer/explorer_test.go @@ -0,0 +1,15 @@ +package serializer + +import ( + model "github.com/cloudreve/Cloudreve/v3/models" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestBuildObjectList(t *testing.T) { + a := assert.New(t) + res := BuildObjectList(1, []Object{{}, {}}, &model.Policy{}) + a.NotEmpty(res.Parent) + a.NotNil(res.Policy) + a.Len(res.Objects, 2) +} diff --git a/pkg/serializer/upload.go b/pkg/serializer/upload.go index b9e90299..4e7150b1 100644 --- a/pkg/serializer/upload.go +++ b/pkg/serializer/upload.go @@ -1,9 +1,7 @@ package serializer import ( - "encoding/base64" "encoding/gob" - "encoding/json" model "github.com/cloudreve/Cloudreve/v3/models" "time" ) @@ -64,32 +62,3 @@ type GeneralUploadCallbackFailed struct { func init() { gob.Register(UploadSession{}) } - -// DecodeUploadPolicy 反序列化Header中携带的上传策略 -func DecodeUploadPolicy(raw string) (*UploadPolicy, error) { - var res UploadPolicy - - rawJSON, err := base64.StdEncoding.DecodeString(raw) - if err != nil { - return nil, err - } - - err = json.Unmarshal(rawJSON, &res) - if err != nil { - return nil, err - } - - return &res, err -} - -// EncodeUploadPolicy 序列化Header中携带的上传策略 -func (policy *UploadPolicy) EncodeUploadPolicy() (string, error) { - jsonRes, err := json.Marshal(policy) - if err != nil { - return "", err - } - - res := base64.StdEncoding.EncodeToString(jsonRes) - return res, nil - -} diff --git a/pkg/serializer/upload_test.go b/pkg/serializer/upload_test.go deleted file mode 100644 index 2ed89f1d..00000000 --- a/pkg/serializer/upload_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package serializer - -import ( - "github.com/stretchr/testify/assert" - "testing" -) - -func TestDecodeUploadPolicy(t *testing.T) { - asserts := assert.New(t) - - testCases := []struct { - input string - expectError bool - expectNil bool - expectRes *UploadPolicy - }{ - { - "错误的base64字符", - true, - true, - &UploadPolicy{}, - }, - { - "6ZSZ6K+v55qESlNPTuWtl+espg==", - true, - true, - &UploadPolicy{}, - }, - { - "e30=", - false, - false, - &UploadPolicy{}, - }, - { - "eyJjYWxsYmFja191cmwiOiJ0ZXN0In0=", - false, - false, - &UploadPolicy{CallbackURL: "test"}, - }, - } - - for _, testCase := range testCases { - res, err := DecodeUploadPolicy(testCase.input) - if testCase.expectError { - asserts.Error(err) - } - if testCase.expectNil { - asserts.Nil(res) - } - if !testCase.expectNil { - asserts.Equal(testCase.expectRes, res) - } - } -} - -func TestUploadPolicy_EncodeUploadPolicy(t *testing.T) { - asserts := assert.New(t) - testPolicy := UploadPolicy{} - res, err := testPolicy.EncodeUploadPolicy() - asserts.NoError(err) - asserts.NotEmpty(res) -} diff --git a/pkg/serializer/user_test.go b/pkg/serializer/user_test.go index cb70fcc8..29421861 100644 --- a/pkg/serializer/user_test.go +++ b/pkg/serializer/user_test.go @@ -29,23 +29,19 @@ func TestMain(m *testing.M) { func TestBuildUser(t *testing.T) { asserts := assert.New(t) - user := model.User{ - Policy: model.Policy{MaxSize: 1024 * 1024}, - } + user := model.User{} mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"})) res := BuildUser(user) asserts.NoError(mock.ExpectationsWereMet()) - asserts.Equal("1.00mb", res.Policy.MaxSize) + asserts.NotNil(res) } func TestBuildUserResponse(t *testing.T) { asserts := assert.New(t) - user := model.User{ - Policy: model.Policy{MaxSize: 1024 * 1024}, - } + user := model.User{} res := BuildUserResponse(user) - asserts.Equal("1.00mb", res.Data.(User).Policy.MaxSize) + asserts.NotNil(res) } func TestBuildUserStorageResponse(t *testing.T) { diff --git a/pkg/task/import_test.go b/pkg/task/import_test.go index ea22cb83..c75e17cd 100644 --- a/pkg/task/import_test.go +++ b/pkg/task/import_test.go @@ -182,6 +182,7 @@ func TestImportTask_Do(t *testing.T) { // 插入文件记录 mock.ExpectBegin() mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(2, 1)) + mock.ExpectExec("UPDATE(.+)users(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(2, 1)) mock.ExpectCommit() task.Do() diff --git a/pkg/webdav/internal/xml/xml.go b/pkg/webdav/internal/xml/xml.go index 5b79cbec..be282bbd 100644 --- a/pkg/webdav/internal/xml/xml.go +++ b/pkg/webdav/internal/xml/xml.go @@ -1040,7 +1040,7 @@ Input: d.buf.WriteByte(';') n, err := strconv.ParseUint(s, base, 64) if err == nil && n <= unicode.MaxRune { - text = string(n) + text = string(rune(n)) haveText = true } } @@ -1063,7 +1063,7 @@ Input: if isName(name) { s := string(name) if r, ok := entity[s]; ok { - text = string(r) + text = string(rune(r)) haveText = true } else if d.Entity != nil { text, haveText = d.Entity[s] diff --git a/routers/file_router_test.go b/routers/file_router_test.go deleted file mode 100644 index 41ff0721..00000000 --- a/routers/file_router_test.go +++ /dev/null @@ -1,180 +0,0 @@ -package routers - -import ( - "bytes" - "encoding/json" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/cloudreve/Cloudreve/v3/middleware" - model "github.com/cloudreve/Cloudreve/v3/models" - "github.com/cloudreve/Cloudreve/v3/pkg/serializer" - "github.com/cloudreve/Cloudreve/v3/service/explorer" - "github.com/stretchr/testify/assert" -) - -func TestListDirectoryRoute(t *testing.T) { - switchToMemDB() - asserts := assert.New(t) - router := InitMasterRouter() - w := httptest.NewRecorder() - - // 成功 - req, _ := http.NewRequest( - "GET", - "/api/v3/directory/", - nil, - ) - middleware.SessionMock = map[string]interface{}{"user_id": 1} - router.ServeHTTP(w, req) - asserts.Equal(200, w.Code) - resJSON := &serializer.Response{} - err := json.Unmarshal(w.Body.Bytes(), resJSON) - asserts.NoError(err) - asserts.Equal(0, resJSON.Code) - - w.Body.Reset() - -} - -func TestLocalFileUpload(t *testing.T) { - switchToMemDB() - asserts := assert.New(t) - router := InitMasterRouter() - w := httptest.NewRecorder() - middleware.SessionMock = map[string]interface{}{"user_id": 1} - - testCases := []struct { - GetRequest func() *http.Request - ExpectCode int - RollBack func() - }{ - // 文件大小指定错误 - { - GetRequest: func() *http.Request { - req, _ := http.NewRequest( - "POST", - "/api/v3/file/upload", - nil, - ) - req.Header.Add("Content-Length", "ddf") - return req - }, - ExpectCode: 40001, - }, - // 返回错误 - { - GetRequest: func() *http.Request { - req, _ := http.NewRequest( - "POST", - "/api/v3/file/upload", - strings.NewReader("2333"), - ) - req.Header.Add("Content-Length", "4") - req.Header.Add("X-Cr-FileName", "大地的%sfsf") - return req - }, - ExpectCode: 40002, - }, - // 成功 - { - GetRequest: func() *http.Request { - req, _ := http.NewRequest( - "POST", - "/api/v3/file/upload", - strings.NewReader("2333"), - ) - req.Header.Add("Content-Length", "4") - req.Header.Add("X-Cr-FileName", "TestFileUploadRoute.txt") - req.Header.Add("X-Cr-Path", "/") - return req - }, - ExpectCode: 0, - }, - } - - for key, testCase := range testCases { - req := testCase.GetRequest() - router.ServeHTTP(w, req) - asserts.Equal(200, w.Code) - resJSON := &serializer.Response{} - err := json.Unmarshal(w.Body.Bytes(), resJSON) - asserts.NoError(err, "测试用例%d", key) - asserts.Equal(testCase.ExpectCode, resJSON.Code, "测试用例%d", key) - if testCase.RollBack != nil { - testCase.RollBack() - } - w.Body.Reset() - } - -} - -func TestObjectDelete(t *testing.T) { - asserts := assert.New(t) - router := InitMasterRouter() - w := httptest.NewRecorder() - middleware.SessionMock = map[string]interface{}{"user_id": 1} - - testCases := []struct { - Mock []string - GetRequest func() *http.Request - ExpectCode int - RollBack []string - }{ - // 路径不存在，返回无错误 - { - GetRequest: func() *http.Request { - body := explorer.ItemService{ - Items: []uint{1}, - } - bodyStr, _ := json.Marshal(body) - req, _ := http.NewRequest( - "DELETE", - "/api/v3/object", - bytes.NewReader(bodyStr), - ) - return req - }, - ExpectCode: 0, - }, - // 文件删除失败，返回203 - { - Mock: []string{"INSERT INTO `files` (`id`, `created_at`, `updated_at`, `deleted_at`, `name`, `source_name`, `user_id`, `size`, `pic_info`, `folder_id`, `policy_id`) VALUES(5, '2019-11-30 07:08:33', '2019-11-30 07:08:33', NULL, 'pigeon.zip', '65azil3B_pigeon.zip', 1, 1667217, '', 1, 1);"}, - GetRequest: func() *http.Request { - body := explorer.ItemService{ - Items: []uint{5}, - } - bodyStr, _ := json.Marshal(body) - req, _ := http.NewRequest( - "DELETE", - "/api/v3/object", - bytes.NewReader(bodyStr), - ) - return req - }, - RollBack: []string{"DELETE FROM `v3_files` WHERE `id`=5"}, - ExpectCode: 203, - }, - } - - for key, testCase := range testCases { - for _, value := range testCase.Mock { - model.DB.Exec(value) - } - req := testCase.GetRequest() - router.ServeHTTP(w, req) - asserts.Equal(200, w.Code) - resJSON := &serializer.Response{} - err := json.Unmarshal(w.Body.Bytes(), resJSON) - asserts.NoError(err, "测试用例%d", key) - asserts.Equal(testCase.ExpectCode, resJSON.Code, "测试用例%d", key) - - for _, value := range testCase.RollBack { - model.DB.Exec(value) - } - - w.Body.Reset() - } -} diff --git a/routers/router_test.go b/routers/router_test.go index 4c65d4e4..2476de6a 100644 --- a/routers/router_test.go +++ b/routers/router_test.go @@ -1,6 +1,7 @@ package routers import ( + "github.com/cloudreve/Cloudreve/v3/pkg/conf" "net/http" "net/http/httptest" "testing" @@ -19,7 +20,7 @@ func TestPing(t *testing.T) { router.ServeHTTP(w, req) assert.Equal(t, 200, w.Code) - asserts.Contains(w.Body.String(), "Pong") + asserts.Contains(w.Body.String(), conf.BackendVersion) } func TestCaptcha(t *testing.T) {