Feat: upyun download / thumb / sign

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/stretchr/testify v1.4.0
+	github.com/upyun/go-sdk v2.1.0+incompatible
 	golang.org/x/net v0.0.0-20190724013045-ca1201d0de80
 	gopkg.in/go-playground/validator.v8 v8.18.2
 	gopkg.in/ini.v1 v1.51.0 // indirect

go.sum

@@ -170,6 +170,8 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/upyun/go-sdk v2.1.0+incompatible h1:OdjXghQ/TVetWV16Pz3C1/SUpjhGBVPr+cLiqZLLyq0=
+github.com/upyun/go-sdk v2.1.0+incompatible/go.mod h1:eu3F5Uz4b9ZE5bE5QsCL6mgSNWRwfj0zpJ9J626HEqs=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

middleware/auth.go

@@ -8,8 +8,8 @@ import (
 	"github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/auth"
 	"github.com/HFO4/cloudreve/pkg/cache"
-	"github.com/HFO4/cloudreve/pkg/filesystem/oss"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/oss"
-	"github.com/HFO4/cloudreve/pkg/filesystem/upyun"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/upyun"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/HFO4/cloudreve/pkg/util"
 	"github.com/gin-contrib/sessions"

models/policy.go

@@ -37,11 +37,12 @@ type Policy struct {
 
 // PolicyOption 非公有的存储策略属性
 type PolicyOption struct {
-	OPName               string   `json:"op_name"`
+	// Upyun访问Token
-	OPPassword           string   `json:"op_pwd"`
+	Token string `json:"token"`
+	// 允许的文件扩展名
 	FileType []string `json:"file_type"`
+	// MimeType
 	MimeType string `json:"mimetype"`
-	RangeTransferEnabled bool     `json:"range_transfer_enabled"`
 }
 
 func init() {

pkg/filesystem/driver/template/handller.go

@@ -0,0 +1,54 @@
+package template
+
+import (
+	"context"
+	"errors"
+	model "github.com/HFO4/cloudreve/models"
+	"github.com/HFO4/cloudreve/pkg/filesystem/response"
+	"github.com/HFO4/cloudreve/pkg/serializer"
+	"io"
+	"net/url"
+)
+
+// Driver 适配器模板
+type Driver struct {
+	Policy *model.Policy
+}
+
+// Get 获取文件
+func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
+	return nil, errors.New("未实现")
+}
+
+// Put 将文件流保存到指定目录
+func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
+	return errors.New("未实现")
+}
+
+// Delete 删除一个或多个文件，
+// 返回未删除的文件，及遇到的最后一个错误
+func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
+	return []string{}, errors.New("未实现")
+}
+
+// Thumb 获取文件缩略图
+func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
+	return nil, errors.New("未实现")
+}
+
+// Source 获取外链URL
+func (handler Driver) Source(
+	ctx context.Context,
+	path string,
+	baseURL url.URL,
+	ttl int64,
+	isDownload bool,
+	speed int,
+) (string, error) {
+	return "", errors.New("未实现")
+}
+
+// Token 获取上传策略和认证Token
+func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
+	return serializer.UploadCredential{}, errors.New("未实现")
+}

pkg/filesystem/driver/upyun/handller.go

@@ -13,9 +13,12 @@ import (
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
 	"github.com/HFO4/cloudreve/pkg/filesystem/response"
 	"github.com/HFO4/cloudreve/pkg/serializer"
+	"github.com/upyun/go-sdk/upyun"
 	"io"
 	"net/url"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -48,12 +51,88 @@ func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, s
 // Delete 删除一个或多个文件，
 // 返回未删除的文件，及遇到的最后一个错误
 func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
-	return []string{}, errors.New("未实现")
+	up := upyun.NewUpYun(&upyun.UpYunConfig{
+		Bucket:   handler.Policy.BucketName,
+		Operator: handler.Policy.AccessKey,
+		Password: handler.Policy.SecretKey,
+	})
+
+	var (
+		failed       = make([]string, 0, len(files))
+		lastErr      error
+		currentIndex = 0
+		indexLock    sync.Mutex
+		failedLock   sync.Mutex
+		wg           sync.WaitGroup
+		routineNum   = 4
+	)
+	wg.Add(routineNum)
+
+	// upyun不支持批量操作，这里开四个协程并行操作
+	for i := 0; i < routineNum; i++ {
+		go func() {
+			for {
+				// 取得待删除文件
+				indexLock.Lock()
+				if currentIndex >= len(files) {
+					// 所有文件处理完成
+					wg.Done()
+					indexLock.Unlock()
+					return
+				}
+				path := files[currentIndex]
+				currentIndex++
+				indexLock.Unlock()
+
+				// 发送异步删除请求
+				err := up.Delete(&upyun.DeleteObjectConfig{
+					Path:  path,
+					Async: true,
+				})
+
+				// 处理错误
+				if err != nil {
+					failedLock.Lock()
+					lastErr = err
+					failed = append(failed, path)
+					failedLock.Unlock()
+				}
+			}
+		}()
+	}
+
+	wg.Wait()
+
+	return failed, lastErr
 }
 
 // Thumb 获取文件缩略图
 func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
-	return nil, errors.New("未实现")
+	var (
+		thumbSize = [2]uint{400, 300}
+		ok        = false
+	)
+	if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok {
+		return nil, errors.New("无法获取缩略图尺寸设置")
+	}
+
+	thumbParam := fmt.Sprintf("!/fwfh/%dx%d", thumbSize[0], thumbSize[1])
+	thumbURL, err := handler.Source(
+		ctx,
+		path+thumbParam,
+		url.URL{},
+		int64(model.GetIntSetting("preview_timeout", 60)),
+		false,
+		0,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &response.ContentResponse{
+		Redirect: true,
+		URL:      thumbURL,
+	}, nil
 }
 
 // Source 获取外链URL
@@ -65,7 +144,56 @@ func (handler Driver) Source(
 	isDownload bool,
 	speed int,
 ) (string, error) {
-	return "", errors.New("未实现")
+	// 尝试从上下文获取文件名
+	fileName := ""
+	if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
+		fileName = file.Name
+	}
+
+	sourceURL, err := url.Parse(handler.Policy.BaseURL)
+	if err != nil {
+		return "", err
+	}
+
+	fileKey, err := url.Parse(path)
+	if err != nil {
+		return "", err
+	}
+
+	sourceURL = sourceURL.ResolveReference(fileKey)
+
+	// 如果是下载文件URL
+	if isDownload {
+		query := sourceURL.Query()
+		query.Add("_upd", fileName)
+		sourceURL.RawQuery = query.Encode()
+	}
+
+	return handler.signURL(ctx, sourceURL, ttl)
+}
+
+func (handler Driver) signURL(ctx context.Context, path *url.URL, TTL int64) (string, error) {
+	if !handler.Policy.IsPrivate {
+		// 未开启Token防盗链时，直接返回
+		return path.String(), nil
+	}
+
+	etime := time.Now().Add(time.Duration(TTL) * time.Second).Unix()
+	signStr := fmt.Sprintf(
+		"%s&%d&%s",
+		handler.Policy.OptionsSerialized.Token,
+		etime,
+		path.Path,
+	)
+	signMd5 := fmt.Sprintf("%x", md5.Sum([]byte(signStr)))
+	finalSign := signMd5[12:20] + strconv.FormatInt(etime, 10)
+
+	// 将签名添加到URL中
+	query := path.Query()
+	query.Add("_upt", finalSign)
+	path.RawQuery = query.Encode()
+
+	return path.String(), nil
 }
 
 // Token 获取上传策略和认证Token

pkg/filesystem/file_test.go

@@ -6,8 +6,8 @@ import (
 	model "github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/auth"
 	"github.com/HFO4/cloudreve/pkg/cache"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
-	"github.com/HFO4/cloudreve/pkg/filesystem/local"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/jinzhu/gorm"
 	"github.com/stretchr/testify/assert"

pkg/filesystem/filesystem.go

@@ -5,12 +5,12 @@ import (
 	"github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/auth"
 	"github.com/HFO4/cloudreve/pkg/conf"
-	"github.com/HFO4/cloudreve/pkg/filesystem/local"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
-	"github.com/HFO4/cloudreve/pkg/filesystem/oss"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/oss"
-	"github.com/HFO4/cloudreve/pkg/filesystem/qiniu"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/qiniu"
-	"github.com/HFO4/cloudreve/pkg/filesystem/remote"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/remote"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/upyun"
 	"github.com/HFO4/cloudreve/pkg/filesystem/response"
-	"github.com/HFO4/cloudreve/pkg/filesystem/upyun"
 	"github.com/HFO4/cloudreve/pkg/request"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/gin-gonic/gin"

pkg/filesystem/filesystem_test.go

@@ -3,8 +3,8 @@ package filesystem
 import (
 	"github.com/DATA-DOG/go-sqlmock"
 	model "github.com/HFO4/cloudreve/models"
-	"github.com/HFO4/cloudreve/pkg/filesystem/local"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
-	"github.com/HFO4/cloudreve/pkg/filesystem/remote"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/remote"
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/assert"
 	"net/http/httptest"

pkg/filesystem/hooks_test.go

@@ -7,8 +7,8 @@ import (
 	model "github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/cache"
 	"github.com/HFO4/cloudreve/pkg/conf"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
-	"github.com/HFO4/cloudreve/pkg/filesystem/local"
 	"github.com/HFO4/cloudreve/pkg/request"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/jinzhu/gorm"

pkg/filesystem/template/file.go

@@ -1,38 +0,0 @@
-package template
-
-import (
-	"io"
-)
-
-// FileStream 用户传来的文件
-type FileStream struct {
-	File        io.ReadCloser
-	Size        uint64
-	VirtualPath string
-	Name        string
-	MIMEType    string
-}
-
-func (file FileStream) Read(p []byte) (n int, err error) {
-	return file.File.Read(p)
-}
-
-func (file FileStream) GetMIMEType() string {
-	return file.MIMEType
-}
-
-func (file FileStream) GetSize() uint64 {
-	return file.Size
-}
-
-func (file FileStream) Close() error {
-	return file.File.Close()
-}
-
-func (file FileStream) GetFileName() string {
-	return file.Name
-}
-
-func (file FileStream) GetVirtualPath() string {
-	return file.VirtualPath
-}

pkg/filesystem/template/handller.go

@@ -1,82 +0,0 @@
-package template
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	model "github.com/HFO4/cloudreve/models"
-	"github.com/HFO4/cloudreve/pkg/filesystem/response"
-	"github.com/HFO4/cloudreve/pkg/serializer"
-	"github.com/qiniu/api.v7/v7/auth"
-	"github.com/qiniu/api.v7/v7/storage"
-	"io"
-	"net/url"
-)
-
-// Handler 本地策略适配器
-type Handler struct {
-	Policy *model.Policy
-}
-
-// Get 获取文件
-func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser, error) {
-	return nil, errors.New("未实现")
-}
-
-// Put 将文件流保存到指定目录
-func (handler Handler) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
-	return errors.New("未实现")
-	// 凭证生成
-	putPolicy := storage.PutPolicy{
-		Scope: "cloudrevetest",
-	}
-	mac := auth.New("YNzTBBpDUq4EEiFV0-vyJCZCJ0LvUEI0_WvxtEXE", "Clm9d9M2CH7pZ8vm049ZlGZStQxrRQVRTjU_T5_0")
-	upToken := putPolicy.UploadToken(mac)
-
-	cfg := storage.Config{}
-	// 空间对应的机房
-	cfg.Zone = &storage.ZoneHuadong
-	formUploader := storage.NewFormUploader(&cfg)
-	ret := storage.PutRet{}
-	putExtra := storage.PutExtra{
-		Params: map[string]string{},
-	}
-
-	defer file.Close()
-
-	err := formUploader.Put(ctx, &ret, upToken, dst, file, int64(size), &putExtra)
-	if err != nil {
-		fmt.Println(err)
-		return err
-	}
-	fmt.Println(ret.Key, ret.Hash)
-	return nil
-}
-
-// Delete 删除一个或多个文件，
-// 返回未删除的文件，及遇到的最后一个错误
-func (handler Handler) Delete(ctx context.Context, files []string) ([]string, error) {
-	return []string{}, errors.New("未实现")
-}
-
-// Thumb 获取文件缩略图
-func (handler Handler) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
-	return nil, errors.New("未实现")
-}
-
-// Source 获取外链URL
-func (handler Handler) Source(
-	ctx context.Context,
-	path string,
-	baseURL url.URL,
-	ttl int64,
-	isDownload bool,
-	speed int,
-) (string, error) {
-	return "", errors.New("未实现")
-}
-
-// Token 获取上传策略和认证Token
-func (handler Handler) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
-	return serializer.UploadCredential{}, errors.New("未实现")
-}

pkg/filesystem/upload.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	model "github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/cache"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
-	"github.com/HFO4/cloudreve/pkg/filesystem/local"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/HFO4/cloudreve/pkg/util"
 	"github.com/gin-gonic/gin"

pkg/filesystem/upload_test.go

@@ -5,8 +5,8 @@ import (
 	"errors"
 	model "github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/cache"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
-	"github.com/HFO4/cloudreve/pkg/filesystem/local"
 	"github.com/HFO4/cloudreve/pkg/filesystem/response"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/gin-gonic/gin"

pkg/webdav/webdav.go

@@ -10,8 +10,8 @@ import (
 	"errors"
 	"fmt"
 	"github.com/HFO4/cloudreve/pkg/filesystem"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
-	"github.com/HFO4/cloudreve/pkg/filesystem/local"
 	"github.com/HFO4/cloudreve/pkg/util"
 	"net/http"
 	"net/url"

routers/controllers/file.go

@@ -5,8 +5,8 @@ import (
 	"context"
 	"github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/filesystem"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
-	"github.com/HFO4/cloudreve/pkg/filesystem/local"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/HFO4/cloudreve/pkg/util"
 	"github.com/HFO4/cloudreve/service/explorer"

routers/controllers/slave.go

@@ -3,8 +3,8 @@ package controllers
 import (
 	"context"
 	"github.com/HFO4/cloudreve/pkg/filesystem"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
-	"github.com/HFO4/cloudreve/pkg/filesystem/local"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/HFO4/cloudreve/service/explorer"
 	"github.com/gin-gonic/gin"

service/callback/upload.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	model "github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/filesystem"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
-	"github.com/HFO4/cloudreve/pkg/filesystem/local"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/HFO4/cloudreve/pkg/util"
 	"github.com/gin-gonic/gin"

service/explorer/file.go

@@ -8,8 +8,8 @@ import (
 	model "github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/cache"
 	"github.com/HFO4/cloudreve/pkg/filesystem"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
-	"github.com/HFO4/cloudreve/pkg/filesystem/local"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/gin-gonic/gin"
 	"github.com/jinzhu/gorm"