using afero rclone backend to provide ablitiy of mounting different kinds of service;

.gitignore

@@ -27,3 +27,4 @@ version.lock
 *.ini
 conf/conf.ini
 /statik/
+rclone.conf

go.mod

@@ -5,7 +5,7 @@ go 1.13
 require (
 	github.com/DATA-DOG/go-sqlmock v1.3.3
 	github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible
-	github.com/aws/aws-sdk-go v1.31.5
+	github.com/aws/aws-sdk-go v1.38.22
 	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
 	github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4
 	github.com/fatih/color v1.7.0
@@ -16,13 +16,13 @@ require (
 	github.com/gin-gonic/gin v1.5.0
 	github.com/go-ini/ini v1.50.0
 	github.com/go-mail/mail v2.3.1+incompatible
+	github.com/go-sql-driver/mysql v1.5.0 // indirect
 	github.com/gomodule/redigo v2.0.0+incompatible
-	github.com/google/go-querystring v1.0.0
-	github.com/gorilla/websocket v1.4.1
+	github.com/google/go-querystring v1.1.0
+	github.com/gorilla/websocket v1.4.2
 	github.com/hashicorp/go-version v1.2.0
 	github.com/jinzhu/gorm v1.9.11
 	github.com/juju/ratelimit v1.0.1
-	github.com/mattn/go-colorable v0.1.4 // indirect
 	github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/pkg/errors v0.9.1
@@ -33,13 +33,16 @@ require (
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/speps/go-hashids v2.0.0+incompatible
-	github.com/stretchr/testify v1.5.1
+	github.com/spf13/afero v1.6.0
+	github.com/stretchr/testify v1.7.0
 	github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible
 	github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac
 	github.com/upyun/go-sdk v2.1.0+incompatible
-	golang.org/x/text v0.3.2
+	golang.org/x/text v0.3.6
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/go-playground/validator.v9 v9.29.1
 	gopkg.in/ini.v1 v1.51.0 // indirect
 	gopkg.in/mail.v2 v2.3.1 // indirect
 )
+
+replace github.com/spf13/afero => github.com/chaoqing/afero v1.6.1-0.20210815024925-e47db790683a

pkg/conf/conf.go

@@ -3,7 +3,11 @@ package conf
 import (
 	"github.com/cloudreve/Cloudreve/v3/pkg/util"
 	"github.com/go-ini/ini"
+	"github.com/spf13/afero"
+	"github.com/spf13/afero/rclonefs"
 	"gopkg.in/go-playground/validator.v9"
+	"path/filepath"
+	"strings"
 )
 
 // database 数据库
@@ -84,6 +88,13 @@ type cors struct {
 	ExposeHeaders    []string
 }
 
+// RClone配置
+// Binds /mnt/ibm:ibm
+type rclone struct {
+	Config     string
+	Binds     []string
+}
+
 var cfg *ini.File
 
 const defaultConf = `[System]
@@ -131,6 +142,7 @@ func Init(path string) {
 		"Redis":      RedisConfig,
 		"Thumbnail":  ThumbConfig,
 		"CORS":       CORSConfig,
+		"RClone":     RCloneConfig,
 		"Slave":      SlaveConfig,
 	}
 	for sectionName, sectionStruct := range sections {
@@ -147,6 +159,7 @@ func Init(path string) {
 		util.Log()
 	}
 
+	initRCloneBind()
 }
 
 // mapSection 将配置文件的 Section 映射到结构体上
@@ -165,3 +178,36 @@ func mapSection(section string, confStruct interface{}) error {
 
 	return nil
 }
+
+
+func initRCloneBind(){
+	if RCloneConfig.Binds[0] == "UNSET"{
+		return
+	}
+
+	if ok := util.Exists(RCloneConfig.Config); ok{
+		_ = rclonefs.SetConfigPath(RCloneConfig.Config)
+	}else{
+		util.Log().Warning("未找到RClone配置文件: %s", RCloneConfig.Config)
+		return
+	}
+
+	bindPoints := make(map[string]afero.Fs)
+	bindPoints["/"] = afero.NewOsFs()
+
+	for _, kv := range RCloneConfig.Binds{
+		if bind := strings.SplitN(kv,":", 2); len(bind)==2{
+			util.Log().Info("RClone绑定: '%s' -> '%s'", bind[1], bind[0])
+			if target, err := filepath.Abs(bind[0]); err==nil{
+				bindPoints[target] = rclonefs.NewRCloneFs(bind[1])
+			}else{
+				util.Log().Warning("绑定绝对路径出错: '%s'", bind[0])
+			}
+		}else{
+			util.Log().Warning("RClone绑定不符合格式: %s", kv)
+			return
+		}
+	}
+
+	util.OS = rclonefs.NewBindPathFs(bindPoints)
+}

pkg/conf/defaults.go

@@ -49,6 +49,12 @@ var CORSConfig = &cors{
 	ExposeHeaders:    nil,
 }
 
+// RCloneConfig 配置
+var RCloneConfig = &rclone{
+	Config:		"rclone.conf",
+	Binds:		[]string{"UNSET"},
+}
+
 // ThumbConfig 缩略图配置
 var ThumbConfig = &thumb{
 	MaxWidth:   400,

pkg/filesystem/archive.go

@@ -83,12 +83,17 @@ func (fs *FileSystem) Compress(ctx context.Context, folderIDs, fileIDs []uint, i
 		saveFolder,
 		fmt.Sprintf("archive_%d.zip", time.Now().UnixNano()),
 	)
-	zipFile, err := util.CreatNestedFile(zipFilePath)
+	zipAferoFile, err := util.CreatNestedFile(zipFilePath)
 	if err != nil {
 		util.Log().Warning("%s", err)
 		return "", err
 	}
-	defer zipFile.Close()
+	defer zipAferoFile.Close()
+
+	zipFile, ok := zipAferoFile.(*os.File)
+	if !ok{
+		return "", fmt.Errorf("not implemented for non OS compression")
+	}
 
 	// 创建压缩文件Writer
 	zipWriter := zip.NewWriter(zipFile)

pkg/filesystem/driver/local/handler.go

@@ -17,6 +17,7 @@ import (
 	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
 	"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
 	"github.com/cloudreve/Cloudreve/v3/pkg/util"
+	"github.com/spf13/afero"
 )
 
 // Driver 本地策略适配器
@@ -32,7 +33,7 @@ func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]
 	root := util.RelativePath(filepath.FromSlash(path))
 
 	// 开始遍历路径下的文件、目录
-	err := filepath.Walk(root,
+	err := afero.Walk(util.OS, root,
 		func(path string, info os.FileInfo, err error) error {
 			// 跳过根目录
 			if path == root {
@@ -73,7 +74,7 @@ func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]
 // Get 获取文件内容
 func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
 	// 打开文件
-	file, err := os.Open(util.RelativePath(path))
+	file, err := util.OS.Open(util.RelativePath(path))
 	if err != nil {
 		util.Log().Debug("无法打开文件：%s", err)
 		return nil, err
@@ -111,7 +112,7 @@ func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, s
 	// 如果目标目录不存在，创建
 	basePath := filepath.Dir(dst)
 	if !util.Exists(basePath) {
-		err := os.MkdirAll(basePath, 0744)
+		err := util.OS.MkdirAll(basePath, 0744)
 		if err != nil {
 			util.Log().Warning("无法创建目录，%s", err)
 			return err
@@ -119,7 +120,7 @@ func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, s
 	}
 
 	// 创建目标文件
-	out, err := os.Create(dst)
+	out, err := util.OS.Create(dst)
 	if err != nil {
 		util.Log().Warning("无法创建文件，%s", err)
 		return err
@@ -140,7 +141,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
 	for _, value := range files {
 		filePath := util.RelativePath(filepath.FromSlash(value))
 		if util.Exists(filePath) {
-			err := os.Remove(filePath)
+			err := util.OS.Remove(filePath)
 			if err != nil {
 				util.Log().Warning("无法删除文件，%s", err)
 				retErr = err
@@ -149,7 +150,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
 		}
 
 		// 尝试删除文件的缩略图（如果有）
-		_ = os.Remove(util.RelativePath(value + conf.ThumbConfig.FileSuffix))
+		_ = util.OS.Remove(util.RelativePath(value + conf.ThumbConfig.FileSuffix))
 	}
 
 	return deleteFailed, retErr

pkg/filesystem/driver/local/handler_test.go

@@ -21,7 +21,7 @@ func TestHandler_Put(t *testing.T) {
 	asserts := assert.New(t)
 	handler := Driver{}
 	ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
-	os.Remove(util.RelativePath("test/test/txt"))
+	util.OS.Remove(util.RelativePath("test/test/txt"))
 
 	testCases := []struct {
 		file io.ReadCloser
@@ -62,16 +62,16 @@ func TestHandler_Delete(t *testing.T) {
 	ctx := context.Background()
 	filePath := util.RelativePath("test.file")
 
-	file, err := os.Create(filePath)
+	file, err := util.OS.Create(filePath)
 	asserts.NoError(err)
 	_ = file.Close()
 	list, err := handler.Delete(ctx, []string{"test.file"})
 	asserts.Equal([]string{}, list)
 	asserts.NoError(err)
 
-	file, err = os.Create(filePath)
+	file, err = util.OS.Create(filePath)
 	_ = file.Close()
-	file, _ = os.OpenFile(filePath, os.O_RDWR, os.FileMode(0))
+	file, _ = util.OS.OpenFile(filePath, os.O_RDWR, os.FileMode(0))
 	asserts.NoError(err)
 	list, err = handler.Delete(ctx, []string{"test.file", "test.notexist"})
 	file.Close()
@@ -82,7 +82,7 @@ func TestHandler_Delete(t *testing.T) {
 	asserts.Equal([]string{}, list)
 	asserts.NoError(err)
 
-	file, err = os.Create(filePath)
+	file, err = util.OS.Create(filePath)
 	asserts.NoError(err)
 	list, err = handler.Delete(ctx, []string{"test.file"})
 	_ = file.Close()
@@ -97,7 +97,7 @@ func TestHandler_Get(t *testing.T) {
 	defer cancel()
 
 	// 成功
-	file, err := os.Create(util.RelativePath("TestHandler_Get.txt"))
+	file, err := util.OS.Create(util.RelativePath("TestHandler_Get.txt"))
 	asserts.NoError(err)
 	_ = file.Close()
 
@@ -116,7 +116,7 @@ func TestHandler_Thumb(t *testing.T) {
 	asserts := assert.New(t)
 	handler := Driver{}
 	ctx := context.Background()
-	file, err := os.Create(util.RelativePath("TestHandler_Thumb" + conf.ThumbConfig.FileSuffix))
+	file, err := util.OS.Create(util.RelativePath("TestHandler_Thumb" + conf.ThumbConfig.FileSuffix))
 	asserts.NoError(err)
 	file.Close()
 

pkg/util/io.go

@@ -1,14 +1,19 @@
 package util
 
 import (
+	"github.com/spf13/afero"
 	"io"
 	"os"
 	"path/filepath"
 )
 
+var (
+	OS = afero.NewOsFs()
+)
+
 // Exists reports whether the named file or directory exists.
 func Exists(name string) bool {
-	if _, err := os.Stat(name); err != nil {
+	if _, err := OS.Stat(name); err != nil {
 		if os.IsNotExist(err) {
 			return false
 		}
@@ -17,22 +22,22 @@ func Exists(name string) bool {
 }
 
 // CreatNestedFile 给定path创建文件，如果目录不存在就递归创建
-func CreatNestedFile(path string) (*os.File, error) {
+func CreatNestedFile(path string) (afero.File, error) {
 	basePath := filepath.Dir(path)
 	if !Exists(basePath) {
-		err := os.MkdirAll(basePath, 0700)
+		err := OS.MkdirAll(basePath, 0700)
 		if err != nil {
 			Log().Warning("无法创建目录，%s", err)
 			return nil, err
 		}
 	}
 
-	return os.Create(path)
+	return OS.Create(path)
 }
 
 // IsEmpty 返回给定目录是否为空目录
 func IsEmpty(name string) (bool, error) {
-	f, err := os.Open(name)
+	f, err := OS.Open(name)
 	if err != nil {
 		return false, err
 	}