From 90f82100cf961c0e65957a4f7784638110146639 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Wed, 15 Jan 2020 16:03:23 +0800 Subject: [PATCH] Feat: put / get file in qiniu policy --- models/policy.go | 27 +++--- models/policy_test.go | 3 +- pkg/filesystem/archive.go | 5 +- pkg/filesystem/qiniu/handler_test.go | 42 --------- pkg/filesystem/qiniu/handller.go | 135 +++++++++++++++++++++------ pkg/filesystem/remote/handler.go | 6 ++ pkg/request/request.go | 20 ++-- pkg/request/request_test.go | 12 +++ 8 files changed, 158 insertions(+), 92 deletions(-) delete mode 100644 pkg/filesystem/qiniu/handler_test.go diff --git a/models/policy.go b/models/policy.go index 52a5d17..39d8daa 100644 --- a/models/policy.go +++ b/models/policy.go @@ -126,18 +126,23 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string { } // 部分存储策略可以使用{origin}代表原始文件名 - switch policy.Type { - case "qiniu": - // 七牛会将$(fname)自动替换为原始文件名 - replaceTable["{originname}"] = "$(fname)" - case "local", "remote": + if origin == "" { + // 如果上游未传回原始文件名，则使用占位符，让云存储端替换 + switch policy.Type { + case "qiniu": + // 七牛会将$(fname)自动替换为原始文件名 + replaceTable["{originname}"] = "$(fname)" + case "local", "remote": + replaceTable["{originname}"] = origin + case "oss": + // OSS会将${filename}自动替换为原始文件名 + replaceTable["{originname}"] = "${filename}" + case "upyun": + // Upyun会将{filename}{.suffix}自动替换为原始文件名 + replaceTable["{originname}"] = "{filename}{.suffix}" + } + } else { replaceTable["{originname}"] = origin - case "oss": - // OSS会将${filename}自动替换为原始文件名 - replaceTable["{originname}"] = "${filename}" - case "upyun": - // Upyun会将{filename}{.suffix}自动替换为原始文件名 - replaceTable["{originname}"] = "{filename}{.suffix}" } fileRule = util.Replace(replaceTable, fileRule) diff --git a/models/policy_test.go b/models/policy_test.go index 7fab980..d30d7a4 100644 --- a/models/policy_test.go +++ b/models/policy_test.go @@ -123,7 +123,7 @@ func TestPolicy_GenerateFileName(t *testing.T) { testPolicy.Type = "qiniu" testPolicy.FileNameRule = "{uid}123{originname}" - asserts.Equal("1123$(fname)", testPolicy.GenerateFileName(1, "123.txt")) + asserts.Equal("1123123.txt", testPolicy.GenerateFileName(1, "123.txt")) testPolicy.Type = "oss" testPolicy.FileNameRule = "{uid}123{originname}" @@ -132,6 +132,7 @@ func TestPolicy_GenerateFileName(t *testing.T) { testPolicy.Type = "upyun" testPolicy.FileNameRule = "{uid}123{originname}" asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, "")) + } func TestPolicy_IsDirectlyPreview(t *testing.T) { diff --git a/pkg/filesystem/archive.go b/pkg/filesystem/archive.go index 1527884..df746ed 100644 --- a/pkg/filesystem/archive.go +++ b/pkg/filesystem/archive.go @@ -112,7 +112,10 @@ func (fs *FileSystem) doCompress(ctx context.Context, file *model.File, folder * } // 获取文件内容 - fileToZip, err := fs.Handler.Get(ctx, file.SourceName) + fileToZip, err := fs.Handler.Get( + context.WithValue(ctx, fsctx.FileModelCtx, *file), + file.SourceName, + ) if err != nil { util.Log().Debug("Open%s，%s", file.Name, err) return diff --git a/pkg/filesystem/qiniu/handler_test.go b/pkg/filesystem/qiniu/handler_test.go deleted file mode 100644 index f80a1e3..0000000 --- a/pkg/filesystem/qiniu/handler_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package qiniu - -import ( - "context" - model "github.com/HFO4/cloudreve/models" - "github.com/HFO4/cloudreve/pkg/cache" - "github.com/HFO4/cloudreve/pkg/filesystem/fsctx" - "github.com/stretchr/testify/assert" - "testing" -) - -func TestHandler_Token(t *testing.T) { - asserts := assert.New(t) - handler := Handler{ - Policy: &model.Policy{ - MaxSize: 10, - OptionsSerialized: model.PolicyOption{ - MimeType: "ss", - }, - AccessKey: "ak", - SecretKey: "sk", - Server: "http://test.com", - }, - } - ctx := context.Background() - - // 成功 - { - cache.Set("setting_siteURL", "http://test.cloudreve.org", 0) - ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/123") - _, err := handler.Token(ctx, 10, "123") - asserts.NoError(err) - } - - // 上下文无存储路径 - { - ctx = context.Background() - cache.Set("setting_siteURL", "http://test.cloudreve.org", 0) - _, err := handler.Token(ctx, 10, "123") - asserts.Error(err) - } -} diff --git a/pkg/filesystem/qiniu/handller.go b/pkg/filesystem/qiniu/handller.go index 52365e8..f9cb268 100644 --- a/pkg/filesystem/qiniu/handller.go +++ b/pkg/filesystem/qiniu/handller.go @@ -7,10 +7,12 @@ import ( model "github.com/HFO4/cloudreve/models" "github.com/HFO4/cloudreve/pkg/filesystem/fsctx" "github.com/HFO4/cloudreve/pkg/filesystem/response" + "github.com/HFO4/cloudreve/pkg/request" "github.com/HFO4/cloudreve/pkg/serializer" "github.com/qiniu/api.v7/v7/auth/qbox" "github.com/qiniu/api.v7/v7/storage" "io" + "net/http" "net/url" "time" ) @@ -22,43 +24,118 @@ type Handler struct { // Get 获取文件 func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser, error) { - return nil, errors.New("未实现") + // 给文件名加上随机参数以强制拉取 + path = fmt.Sprintf("%s?v=%d", path, time.Now().UnixNano()) + + // 获取文件源地址 + downloadURL, err := handler.Source( + ctx, + path, + url.URL{}, + int64(model.GetIntSetting("preview_timeout", 60)), + false, + 0, + ) + if err != nil { + return nil, err + } + + // 获取文件数据流 + client := request.HTTPClient{} + resp, err := client.Request( + "GET", + downloadURL, + nil, + request.WithContext(ctx), + request.WithHeader( + http.Header{"Cache-Control": {"no-cache", "no-store", "must-revalidate"}}, + ), + ).CheckHTTPResponse(200).GetRSCloser() + if err != nil { + return nil, err + } + + resp.SetFirstFakeChunk() + + // 尝试自主获取文件大小 + if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok { + resp.SetContentLength(int64(file.Size)) + } + + return resp, nil } // Put 将文件流保存到指定目录 func (handler Handler) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error { - return errors.New("未实现") - //// 凭证生成 - //putPolicy := storage.PutPolicy{ - // Scope: "cloudrevetest", - //} - //mac := auth.New("YNzTBBpDUq4EEiFV0-vyJCZCJ0LvUEI0_WvxtEXE", "Clm9d9M2CH7pZ8vm049ZlGZStQxrRQVRTjU_T5_0") - //upToken := putPolicy.UploadToken(mac) - // - //cfg := storage.Config{} - //// 空间对应的机房 - //cfg.Zone = &storage.ZoneHuadong - //formUploader := storage.NewFormUploader(&cfg) - //ret := storage.PutRet{} - //putExtra := storage.PutExtra{ - // Params: map[string]string{}, - //} - // - //defer file.Close() - // - //err := formUploader.Put(ctx, &ret, upToken, dst, file, int64(size), &putExtra) - //if err != nil { - // fmt.Println(err) - // return err - //} - //fmt.Println(ret.Key, ret.Hash) - //return nil + defer file.Close() + + // 凭证有效期 + credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600) + + // 生成上传策略 + putPolicy := storage.PutPolicy{ + // 指定为覆盖策略 + Scope: fmt.Sprintf("%s:%s", handler.Policy.BucketName, dst), + SaveKey: dst, + ForceSaveKey: true, + FsizeLimit: int64(size), + } + // 是否开启了MIMEType限制 + if handler.Policy.OptionsSerialized.MimeType != "" { + putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType + } + + // 生成上传凭证 + token, err := handler.getUploadCredential(ctx, putPolicy, int64(credentialTTL)) + if err != nil { + return err + } + + // 创建上传表单 + cfg := storage.Config{} + formUploader := storage.NewFormUploader(&cfg) + ret := storage.PutRet{} + putExtra := storage.PutExtra{ + Params: map[string]string{}, + } + + // 开始上传 + err = formUploader.Put(ctx, &ret, token.Token, dst, file, int64(size), &putExtra) + if err != nil { + return err + } + + return nil } // Delete 删除一个或多个文件， -// 返回未删除的文件，及遇到的最后一个错误 +// 返回未删除的文件 func (handler Handler) Delete(ctx context.Context, files []string) ([]string, error) { - return []string{}, errors.New("未实现") + // TODO 大于一千个文件需要分批发送 + deleteOps := make([]string, 0, len(files)) + for _, key := range files { + deleteOps = append(deleteOps, storage.URIDelete(handler.Policy.BucketName, key)) + } + + mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey) + cfg := storage.Config{ + UseHTTPS: true, + } + bucketManager := storage.NewBucketManager(mac, &cfg) + rets, err := bucketManager.Batch(deleteOps) + + // 处理删除结果 + if err != nil { + failed := make([]string, 0, len(rets)) + for k, ret := range rets { + if ret.Code != 200 { + failed = append(failed, files[k]) + } + } + return failed, errors.New("删除失败") + } + + return []string{}, nil } // Thumb 获取文件缩略图 diff --git a/pkg/filesystem/remote/handler.go b/pkg/filesystem/remote/handler.go index ec7403e..882ebf8 100644 --- a/pkg/filesystem/remote/handler.go +++ b/pkg/filesystem/remote/handler.go @@ -76,6 +76,12 @@ func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser, } resp.SetFirstFakeChunk() + + // 尝试获取文件大小 + if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok { + resp.SetContentLength(int64(file.Size)) + } + return resp, nil } diff --git a/pkg/request/request.go b/pkg/request/request.go index d8bb438..9946bf7 100644 --- a/pkg/request/request.go +++ b/pkg/request/request.go @@ -191,7 +191,6 @@ func (resp *Response) DecodeResponse() (*serializer.Response, error) { // NopRSCloser 实现不完整seeker type NopRSCloser struct { body io.ReadCloser - size int64 status *rscStatus } @@ -199,6 +198,8 @@ type rscStatus struct { // http.ServeContent 会读取一小块以决定内容类型， // 但是响应body无法实现seek，所以此项为真时第一个read会返回假数据 IgnoreFirst bool + + Size int64 } // GetRSCloser 返回带有空seeker的RSCloser，供http.ServeContent使用 @@ -208,9 +209,10 @@ func (resp *Response) GetRSCloser() (*NopRSCloser, error) { } return &NopRSCloser{ - body: resp.Response.Body, - size: resp.Response.ContentLength, - status: &rscStatus{}, + body: resp.Response.Body, + status: &rscStatus{ + Size: resp.Response.ContentLength, + }, }, resp.Err } @@ -220,11 +222,13 @@ func (instance NopRSCloser) SetFirstFakeChunk() { instance.status.IgnoreFirst = true } +// SetContentLength 设置数据流大小 +func (instance NopRSCloser) SetContentLength(size int64) { + instance.status.Size = size +} + // Read 实现 NopRSCloser reader func (instance NopRSCloser) Read(p []byte) (n int, err error) { - if instance.status.IgnoreFirst { - return 0, io.EOF - } return instance.body.Read(p) } @@ -244,7 +248,7 @@ func (instance NopRSCloser) Seek(offset int64, whence int) (int64, error) { case io.SeekStart: return 0, nil case io.SeekEnd: - return instance.size, nil + return instance.status.Size, nil } } return 0, errors.New("未实现") diff --git a/pkg/request/request_test.go b/pkg/request/request_test.go index fa4971f..0dd8d37 100644 --- a/pkg/request/request_test.go +++ b/pkg/request/request_test.go @@ -211,3 +211,15 @@ func TestResponse_DecodeResponse(t *testing.T) { asserts.Equal(0, response.Code) } } + +func TestNopRSCloser_SetFirstFakeChunk(t *testing.T) { + asserts := assert.New(t) + rsc := NopRSCloser{ + status: &rscStatus{}, + } + rsc.SetFirstFakeChunk() + asserts.True(rsc.status.IgnoreFirst) + + rsc.SetContentLength(20) + asserts.EqualValues(20, rsc.status.Size) +}