Feat: put / get file in qiniu policy

models/policy.go

@@ -126,6 +126,8 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string {
 	}
 
 	// 部分存储策略可以使用{origin}代表原始文件名
+	if origin == "" {
+		// 如果上游未传回原始文件名，则使用占位符，让云存储端替换
 		switch policy.Type {
 		case "qiniu":
 			// 七牛会将$(fname)自动替换为原始文件名
@@ -139,6 +141,9 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string {
 			// Upyun会将{filename}{.suffix}自动替换为原始文件名
 			replaceTable["{originname}"] = "{filename}{.suffix}"
 		}
+	} else {
+		replaceTable["{originname}"] = origin
+	}
 
 	fileRule = util.Replace(replaceTable, fileRule)
 	return fileRule

models/policy_test.go

@@ -123,7 +123,7 @@ func TestPolicy_GenerateFileName(t *testing.T) {
 
 	testPolicy.Type = "qiniu"
 	testPolicy.FileNameRule = "{uid}123{originname}"
-	asserts.Equal("1123$(fname)", testPolicy.GenerateFileName(1, "123.txt"))
+	asserts.Equal("1123123.txt", testPolicy.GenerateFileName(1, "123.txt"))
 
 	testPolicy.Type = "oss"
 	testPolicy.FileNameRule = "{uid}123{originname}"
@@ -132,6 +132,7 @@ func TestPolicy_GenerateFileName(t *testing.T) {
 	testPolicy.Type = "upyun"
 	testPolicy.FileNameRule = "{uid}123{originname}"
 	asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, ""))
+
 }
 
 func TestPolicy_IsDirectlyPreview(t *testing.T) {

pkg/filesystem/archive.go

@@ -112,7 +112,10 @@ func (fs *FileSystem) doCompress(ctx context.Context, file *model.File, folder *
 		}
 
 		// 获取文件内容
-		fileToZip, err := fs.Handler.Get(ctx, file.SourceName)
+		fileToZip, err := fs.Handler.Get(
+			context.WithValue(ctx, fsctx.FileModelCtx, *file),
+			file.SourceName,
+		)
 		if err != nil {
 			util.Log().Debug("Open%s，%s", file.Name, err)
 			return

pkg/filesystem/qiniu/handler_test.go

@@ -1,42 +0,0 @@
-package qiniu
-
-import (
-	"context"
-	model "github.com/HFO4/cloudreve/models"
-	"github.com/HFO4/cloudreve/pkg/cache"
-	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
-	"github.com/stretchr/testify/assert"
-	"testing"
-)
-
-func TestHandler_Token(t *testing.T) {
-	asserts := assert.New(t)
-	handler := Handler{
-		Policy: &model.Policy{
-			MaxSize: 10,
-			OptionsSerialized: model.PolicyOption{
-				MimeType: "ss",
-			},
-			AccessKey: "ak",
-			SecretKey: "sk",
-			Server:    "http://test.com",
-		},
-	}
-	ctx := context.Background()
-
-	// 成功
-	{
-		cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
-		ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/123")
-		_, err := handler.Token(ctx, 10, "123")
-		asserts.NoError(err)
-	}
-
-	// 上下文无存储路径
-	{
-		ctx = context.Background()
-		cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
-		_, err := handler.Token(ctx, 10, "123")
-		asserts.Error(err)
-	}
-}

pkg/filesystem/qiniu/handller.go

@@ -7,10 +7,12 @@ import (
 	model "github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
 	"github.com/HFO4/cloudreve/pkg/filesystem/response"
+	"github.com/HFO4/cloudreve/pkg/request"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/qiniu/api.v7/v7/auth/qbox"
 	"github.com/qiniu/api.v7/v7/storage"
 	"io"
+	"net/http"
 	"net/url"
 	"time"
 )
@@ -22,43 +24,118 @@ type Handler struct {
 
 // Get 获取文件
 func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser, error) {
-	return nil, errors.New("未实现")
+	// 给文件名加上随机参数以强制拉取
+	path = fmt.Sprintf("%s?v=%d", path, time.Now().UnixNano())
+
+	// 获取文件源地址
+	downloadURL, err := handler.Source(
+		ctx,
+		path,
+		url.URL{},
+		int64(model.GetIntSetting("preview_timeout", 60)),
+		false,
+		0,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// 获取文件数据流
+	client := request.HTTPClient{}
+	resp, err := client.Request(
+		"GET",
+		downloadURL,
+		nil,
+		request.WithContext(ctx),
+		request.WithHeader(
+			http.Header{"Cache-Control": {"no-cache", "no-store", "must-revalidate"}},
+		),
+	).CheckHTTPResponse(200).GetRSCloser()
+	if err != nil {
+		return nil, err
+	}
+
+	resp.SetFirstFakeChunk()
+
+	// 尝试自主获取文件大小
+	if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
+		resp.SetContentLength(int64(file.Size))
+	}
+
+	return resp, nil
 }
 
 // Put 将文件流保存到指定目录
 func (handler Handler) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
-	return errors.New("未实现")
-	//// 凭证生成
-	//putPolicy := storage.PutPolicy{
-	//	Scope: "cloudrevetest",
-	//}
-	//mac := auth.New("YNzTBBpDUq4EEiFV0-vyJCZCJ0LvUEI0_WvxtEXE", "Clm9d9M2CH7pZ8vm049ZlGZStQxrRQVRTjU_T5_0")
-	//upToken := putPolicy.UploadToken(mac)
-	//
-	//cfg := storage.Config{}
-	//// 空间对应的机房
-	//cfg.Zone = &storage.ZoneHuadong
-	//formUploader := storage.NewFormUploader(&cfg)
-	//ret := storage.PutRet{}
-	//putExtra := storage.PutExtra{
-	//	Params: map[string]string{},
-	//}
-	//
-	//defer file.Close()
-	//
-	//err := formUploader.Put(ctx, &ret, upToken, dst, file, int64(size), &putExtra)
-	//if err != nil {
-	//	fmt.Println(err)
-	//	return err
-	//}
-	//fmt.Println(ret.Key, ret.Hash)
-	//return nil
+	defer file.Close()
+
+	// 凭证有效期
+	credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
+
+	// 生成上传策略
+	putPolicy := storage.PutPolicy{
+		// 指定为覆盖策略
+		Scope:        fmt.Sprintf("%s:%s", handler.Policy.BucketName, dst),
+		SaveKey:      dst,
+		ForceSaveKey: true,
+		FsizeLimit:   int64(size),
+	}
+	// 是否开启了MIMEType限制
+	if handler.Policy.OptionsSerialized.MimeType != "" {
+		putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType
+	}
+
+	// 生成上传凭证
+	token, err := handler.getUploadCredential(ctx, putPolicy, int64(credentialTTL))
+	if err != nil {
+		return err
+	}
+
+	// 创建上传表单
+	cfg := storage.Config{}
+	formUploader := storage.NewFormUploader(&cfg)
+	ret := storage.PutRet{}
+	putExtra := storage.PutExtra{
+		Params: map[string]string{},
+	}
+
+	// 开始上传
+	err = formUploader.Put(ctx, &ret, token.Token, dst, file, int64(size), &putExtra)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // Delete 删除一个或多个文件，
-// 返回未删除的文件，及遇到的最后一个错误
+// 返回未删除的文件
 func (handler Handler) Delete(ctx context.Context, files []string) ([]string, error) {
-	return []string{}, errors.New("未实现")
+	// TODO 大于一千个文件需要分批发送
+	deleteOps := make([]string, 0, len(files))
+	for _, key := range files {
+		deleteOps = append(deleteOps, storage.URIDelete(handler.Policy.BucketName, key))
+	}
+
+	mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
+	cfg := storage.Config{
+		UseHTTPS: true,
+	}
+	bucketManager := storage.NewBucketManager(mac, &cfg)
+	rets, err := bucketManager.Batch(deleteOps)
+
+	// 处理删除结果
+	if err != nil {
+		failed := make([]string, 0, len(rets))
+		for k, ret := range rets {
+			if ret.Code != 200 {
+				failed = append(failed, files[k])
+			}
+		}
+		return failed, errors.New("删除失败")
+	}
+
+	return []string{}, nil
 }
 
 // Thumb 获取文件缩略图

pkg/filesystem/remote/handler.go

@@ -76,6 +76,12 @@ func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser,
 	}
 
 	resp.SetFirstFakeChunk()
+
+	// 尝试获取文件大小
+	if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
+		resp.SetContentLength(int64(file.Size))
+	}
+
 	return resp, nil
 }
 

pkg/request/request.go

@@ -191,7 +191,6 @@ func (resp *Response) DecodeResponse() (*serializer.Response, error) {
 // NopRSCloser 实现不完整seeker
 type NopRSCloser struct {
 	body   io.ReadCloser
-	size   int64
 	status *rscStatus
 }
 
@@ -199,6 +198,8 @@ type rscStatus struct {
 	// http.ServeContent 会读取一小块以决定内容类型，
 	// 但是响应body无法实现seek，所以此项为真时第一个read会返回假数据
 	IgnoreFirst bool
+
+	Size int64
 }
 
 // GetRSCloser 返回带有空seeker的RSCloser，供http.ServeContent使用
@@ -209,8 +210,9 @@ func (resp *Response) GetRSCloser() (*NopRSCloser, error) {
 
 	return &NopRSCloser{
 		body: resp.Response.Body,
-		size:   resp.Response.ContentLength,
-		status: &rscStatus{},
+		status: &rscStatus{
+			Size: resp.Response.ContentLength,
+		},
 	}, resp.Err
 }
 
@@ -220,11 +222,13 @@ func (instance NopRSCloser) SetFirstFakeChunk() {
 	instance.status.IgnoreFirst = true
 }
 
+// SetContentLength 设置数据流大小
+func (instance NopRSCloser) SetContentLength(size int64) {
+	instance.status.Size = size
+}
+
 // Read 实现 NopRSCloser reader
 func (instance NopRSCloser) Read(p []byte) (n int, err error) {
-	if instance.status.IgnoreFirst {
-		return 0, io.EOF
-	}
 	return instance.body.Read(p)
 }
 
@@ -244,7 +248,7 @@ func (instance NopRSCloser) Seek(offset int64, whence int) (int64, error) {
 		case io.SeekStart:
 			return 0, nil
 		case io.SeekEnd:
-			return instance.size, nil
+			return instance.status.Size, nil
 		}
 	}
 	return 0, errors.New("未实现")

pkg/request/request_test.go

@@ -211,3 +211,15 @@ func TestResponse_DecodeResponse(t *testing.T) {
 		asserts.Equal(0, response.Code)
 	}
 }
+
+func TestNopRSCloser_SetFirstFakeChunk(t *testing.T) {
+	asserts := assert.New(t)
+	rsc := NopRSCloser{
+		status: &rscStatus{},
+	}
+	rsc.SetFirstFakeChunk()
+	asserts.True(rsc.status.IgnoreFirst)
+
+	rsc.SetContentLength(20)
+	asserts.EqualValues(20, rsc.status.Size)
+}