Feat: remote delete file / Fix: broken Filesystem recycle in upload request

pkg/filesystem/filesystem.go

@@ -120,6 +120,9 @@ func NewAnonymousFileSystem() (*FileSystem, error) {
 			return nil, err
 		}
 		fs.User.Group = anonymousGroup
+	} else {
+		// 从机模式下，分配本地策略处理器
+		fs.Handler = local.Handler{}
 	}
 
 	return fs, nil

pkg/filesystem/local/handler.go

@@ -69,7 +69,10 @@ func (handler Handler) Put(ctx context.Context, file io.ReadCloser, dst string,
 		util.Log().Warning("无法创建文件，%s", err)
 		return err
 	}
-	defer out.Close()
+	defer func() {
+		err := out.Close()
+		fmt.Print(err)
+	}()
 
 	// 写入文件内容
 	_, err = io.Copy(out, file)
@@ -138,7 +141,7 @@ func (handler Handler) Source(
 		downloadSessionID := util.RandStringRunes(16)
 		err = cache.Set("download_"+downloadSessionID, file, int(ttl))
 		if err != nil {
-			return "", serializer.NewError(serializer.CodeCacheOperation, "无法创建下載会话", err)
+			return "", serializer.NewError(serializer.CodeCacheOperation, "无法创建下载会话", err)
 		}
 
 		// 签名生成文件记录
@@ -165,7 +168,6 @@ func (handler Handler) Source(
 }
 
 // Token 获取上传策略和认证Token，本地策略直接返回空值
-// TODO 测试
 func (handler Handler) Token(ctx context.Context, ttl int64, key string) (serializer.UploadCredential, error) {
 	return serializer.UploadCredential{}, nil
 }

pkg/filesystem/manage.go

@@ -131,6 +131,7 @@ func (fs *FileSystem) Move(ctx context.Context, dirs, files []uint, src, dst str
 func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint) error {
 	// 已删除的总容量,map用于去重
 	var deletedStorage = make(map[uint]uint64)
+	var totalStorage = make(map[uint]uint64)
 	// 已删除的文件ID
 	var deletedFileIDs = make([]uint, 0, len(fs.FileTarget))
 	// 删除失败的文件的父目录ID
@@ -166,15 +167,18 @@ func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint) error {
 	// 按照存储策略分组删除对象
 	failed := fs.deleteGroupedFile(ctx, policyGroup)
 
+	// 整理删除结果
 	for i := 0; i < len(fs.FileTarget); i++ {
-		if util.ContainsString(failed[fs.FileTarget[i].PolicyID], fs.FileTarget[i].SourceName) {
-			// TODO 删除失败时不删除文件记录及父目录
-		} else {
+		if !util.ContainsString(failed[fs.FileTarget[i].PolicyID], fs.FileTarget[i].SourceName) {
+			// 已成功删除的文件
 			deletedFileIDs = append(deletedFileIDs, fs.FileTarget[i].ID)
+			deletedStorage[fs.FileTarget[i].ID] = fs.FileTarget[i].Size
 		}
-		deletedStorage[fs.FileTarget[i].ID] = fs.FileTarget[i].Size
+		// 全部文件
+		totalStorage[fs.FileTarget[i].ID] = fs.FileTarget[i].Size
 		allFileIDs = append(allFileIDs, fs.FileTarget[i].ID)
 	}
+	// TODO 用户自主选择是否强制删除
 
 	// 删除文件记录
 	err = model.DeleteFileByIDs(allFileIDs)
@@ -184,7 +188,7 @@ func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint) error {
 
 	// 归还容量
 	var total uint64
-	for _, value := range deletedStorage {
+	for _, value := range totalStorage {
 		total += value
 	}
 	fs.User.DeductionStorage(total)

pkg/filesystem/remote/handler.go

@@ -4,24 +4,44 @@ package remote
 import (
 	"context"
 	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	model "github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/auth"
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
 	"github.com/HFO4/cloudreve/pkg/filesystem/response"
+	"github.com/HFO4/cloudreve/pkg/request"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"io"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 )
 
 // Handler 远程存储策略适配器
 type Handler struct {
+	client request.HTTPClient
 	Policy *model.Policy
 }
 
+// getAPI 获取接口请求地址
+func (handler Handler) getAPI(scope string) string {
+	serverURL, err := url.Parse(handler.Policy.Server)
+	if err != nil {
+		return ""
+	}
+	var controller *url.URL
+
+	switch scope {
+	case "delete":
+		controller, _ = url.Parse("/api/v3/slave/delete")
+	}
+
+	return serverURL.ResolveReference(controller).String()
+}
+
 // Get 获取文件内容
 func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser, error) {
 
@@ -35,7 +55,45 @@ func (handler Handler) Put(ctx context.Context, file io.ReadCloser, dst string,
 
 // Delete 删除一个或多个文件，
 // 返回未删除的文件，及遇到的最后一个错误
+// TODO 测试
 func (handler Handler) Delete(ctx context.Context, files []string) ([]string, error) {
+	// 封装接口请求正文
+	reqBody := serializer.RemoteDeleteRequest{
+		Files: files,
+	}
+	reqBodyEncoded, err := json.Marshal(reqBody)
+	if err != nil {
+		return files, err
+	}
+
+	// 发送删除请求
+	bodyReader := strings.NewReader(string(reqBodyEncoded))
+	authInstance := auth.HMACAuth{SecretKey: []byte(handler.Policy.SecretKey)}
+	resp, err := handler.client.Request(
+		"POST",
+		handler.getAPI("delete"),
+		bodyReader,
+		request.WithCredential(authInstance, 60),
+	).GetResponse(200)
+	if err != nil {
+		return files, err
+	}
+
+	// 处理删除结果
+	var reqResp serializer.Response
+	err = json.Unmarshal([]byte(resp), &reqResp)
+	if err != nil {
+		return files, err
+	}
+	if reqResp.Code != 0 {
+		var failedResp serializer.RemoteDeleteRequest
+		err = json.Unmarshal([]byte(reqResp.Data.(string)), &failedResp)
+		if err == nil {
+			return failedResp.Files, errors.New(reqResp.Error)
+		}
+		return files, errors.New("未知的返回结果格式")
+	}
+
 	return []string{}, nil
 }
 

pkg/filesystem/upload.go

@@ -116,6 +116,7 @@ func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file FileHe
 	} else {
 		reqContext = ctx.Value(fsctx.HTTPCtx).(context.Context)
 	}
+	defer fs.Recycle()
 
 	select {
 	case <-reqContext.Done():

pkg/request/request.go

@@ -1,8 +1,10 @@
 package request
 
 import (
+	"fmt"
 	"github.com/HFO4/cloudreve/pkg/auth"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"time"
 )
@@ -104,3 +106,17 @@ func (c HTTPClient) Request(method, target string, body io.Reader, opts ...Optio
 
 	return Response{Err: nil, Response: resp}
 }
+
+// GetResponse 检查响应并获取响应正文
+// todo 测试
+func (resp Response) GetResponse(expectStatus int) (string, error) {
+	if resp.Err != nil {
+		return "", resp.Err
+	}
+	respBody, err := ioutil.ReadAll(resp.Response.Body)
+	if resp.Response.StatusCode != expectStatus {
+		return string(respBody),
+			fmt.Errorf("服务器返回非正常HTTP状态%d", resp.Response.StatusCode)
+	}
+	return string(respBody), err
+}

pkg/request/slave.go

@@ -8,7 +8,6 @@ import (
 	"github.com/HFO4/cloudreve/pkg/conf"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/HFO4/cloudreve/pkg/util"
-	"io/ioutil"
 	"time"
 )
 
@@ -36,20 +35,14 @@ func RemoteCallback(url string, body serializer.RemoteUploadCallback) error {
 	}
 
 	// 检查返回HTTP状态码
-	if resp.Response.StatusCode != 200 {
-		util.Log().Debug("服务端返回非正常状态码：%d", resp.Response.StatusCode)
-		return serializer.NewError(serializer.CodeCallbackError, "服务端返回非正常状态码", nil)
-	}
-
-	// 检查返回API状态码
-	var response serializer.Response
-	rawResp, err := ioutil.ReadAll(resp.Response.Body)
+	rawResp, err := resp.GetResponse(200)
 	if err != nil {
-		return serializer.NewError(serializer.CodeCallbackError, "无法读取响应正文", err)
+		return serializer.NewError(serializer.CodeCallbackError, "服务器返回异常响应", err)
 	}
 
 	// 解析回调服务端响应
-	err = json.Unmarshal(rawResp, &response)
+	var response serializer.Response
+	err = json.Unmarshal([]byte(rawResp), &response)
 	if err != nil {
 		util.Log().Debug("无法解析回调服务端响应：%s", string(rawResp))
 		return serializer.NewError(serializer.CodeCallbackError, "无法解析服务端返回的响应", err)

pkg/serializer/slave.go

@@ -0,0 +1,6 @@
+package serializer
+
+// RemoteDeleteRequest 远程策略删除接口请求正文
+type RemoteDeleteRequest struct {
+	Files []string `json:"files"`
+}

pkg/serializer/upload.go

@@ -59,7 +59,6 @@ func DecodeUploadPolicy(raw string) (*UploadPolicy, error) {
 }
 
 // EncodeUploadPolicy 序列化Header中携带的上传策略
-// TODO 测试
 func (policy *UploadPolicy) EncodeUploadPolicy() (string, error) {
 	jsonRes, err := json.Marshal(policy)
 	if err != nil {

routers/controllers/file.go

@@ -265,7 +265,6 @@ func FileUploadStream(c *gin.Context) {
 		c.JSON(200, serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err))
 		return
 	}
-	defer fs.Recycle()
 
 	// 给文件系统分配钩子
 	fs.Use("BeforeUpload", filesystem.HookValidateFile)

routers/controllers/slave.go

@@ -25,7 +25,6 @@ func SlaveUpload(c *gin.Context) {
 		c.JSON(200, serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err))
 		return
 	}
-	defer fs.Recycle()
 	fs.Handler = local.Handler{}
 
 	// 从请求中取得上传策略
@@ -115,3 +114,18 @@ func SlavePreview(c *gin.Context) {
 		c.JSON(200, ErrorResponse(err))
 	}
 }
+
+// SlaveDelete 从机删除
+func SlaveDelete(c *gin.Context) {
+	// 创建上下文
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var service explorer.SlaveFilesService
+	if err := c.ShouldBindJSON(&service); err == nil {
+		res := service.Delete(ctx, c)
+		c.JSON(200, res)
+	} else {
+		c.JSON(200, ErrorResponse(err))
+	}
+}

routers/router.go

@@ -39,6 +39,8 @@ func InitSlaveRouter() *gin.Engine {
 		v3.GET("download/:speed/:path/:name", controllers.SlaveDownload)
 		// 预览 / 外链
 		v3.GET("source/:speed/:path/:name", controllers.SlavePreview)
+		// 删除文件
+		v3.POST("delete", controllers.SlaveDelete)
 	}
 	return r
 }

service/explorer/file.go

@@ -3,6 +3,8 @@ package explorer
 import (
 	"context"
 	"encoding/base64"
+	"encoding/json"
+	"fmt"
 	model "github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/cache"
 	"github.com/HFO4/cloudreve/pkg/filesystem"
@@ -41,6 +43,11 @@ type SlaveDownloadService struct {
 	Speed       int    `uri:"speed" binding:"min=0"`
 }
 
+// SlaveFilesService 从机多文件相关服务
+type SlaveFilesService struct {
+	Files []string `json:"files" binding:"required,gt=0"`
+}
+
 // DownloadArchived 下載已打包的多文件
 func (service *DownloadService) DownloadArchived(ctx context.Context, c *gin.Context) serializer.Response {
 	// 创建文件系统
@@ -255,7 +262,6 @@ func (service *SingleFileService) PutContent(ctx context.Context, c *gin.Context
 	if err != nil {
 		return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
 	}
-	defer fs.Recycle()
 
 	// 取得现有文件
 	exist, originFile := fs.IsFileExist(service.Path)
@@ -288,7 +294,7 @@ func (service *SingleFileService) PutContent(ctx context.Context, c *gin.Context
 	}
 }
 
-// ServeFile 通过签名URL的文件下载从机文件
+// ServeFile 通过签名的URL下载从机文件
 func (service *SlaveDownloadService) ServeFile(ctx context.Context, c *gin.Context, isDownload bool) serializer.Response {
 	// 创建文件系统
 	fs, err := filesystem.NewAnonymousFileSystem()
@@ -337,3 +343,28 @@ func (service *SlaveDownloadService) ServeFile(ctx context.Context, c *gin.Conte
 		Code: 0,
 	}
 }
+
+// Delete 通过签名的URL删除从机文件
+func (service *SlaveFilesService) Delete(ctx context.Context, c *gin.Context) serializer.Response {
+	// 创建文件系统
+	fs, err := filesystem.NewAnonymousFileSystem()
+	if err != nil {
+		return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
+	}
+	defer fs.Recycle()
+
+	// 删除文件
+	failed, err := fs.Handler.Delete(ctx, service.Files)
+	if err != nil {
+		// 将Data字段写为字符串方便主控端解析
+		data, _ := json.Marshal(serializer.RemoteDeleteRequest{Files: failed})
+
+		return serializer.Response{
+			Code:  serializer.CodeNotFullySuccess,
+			Data:  string(data),
+			Msg:   fmt.Sprintf("有 %d 个文件未能成功删除", len(failed)),
+			Error: err.Error(),
+		}
+	}
+	return serializer.Response{Code: 0}
+}