msb-public
/
cloudreve
mirror of https://github.com/cloudreve/Cloudreve
1
0
Code Issues Projects Releases Wiki Activity

using afero rclone backend to provide ablitiy of mounting different kinds of service;

Browse Source
pull/984/head
chaoqing 4 years ago
parent 65c4367689
commit af17e661df
9 changed files with 1028 additions and 26 deletions
Show Stats Download Patch File Download Diff File

1
.gitignore vendored
Unescape View File

@ -27,3 +27,4 @@ version.lock
*.ini
conf/conf.ini
/statik/
rclone.conf

15
go.mod
Unescape View File

@ -5,7 +5,7 @@ go 1.13
require (
github.com/DATA-DOG/go-sqlmock v1.3.3
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible
github.com/aws/aws-sdk-go v1.31.5
github.com/aws/aws-sdk-go v1.38.22
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4
github.com/fatih/color v1.7.0
@ -16,13 +16,13 @@ require (
github.com/gin-gonic/gin v1.5.0
github.com/go-ini/ini v1.50.0
github.com/go-mail/mail v2.3.1+incompatible
github.com/go-sql-driver/mysql v1.5.0 // indirect
github.com/gomodule/redigo v2.0.0+incompatible
github.com/google/go-querystring v1.0.0
github.com/gorilla/websocket v1.4.1
github.com/google/go-querystring v1.1.0
github.com/gorilla/websocket v1.4.2
github.com/hashicorp/go-version v1.2.0
github.com/jinzhu/gorm v1.9.11
github.com/juju/ratelimit v1.0.1
github.com/mattn/go-colorable v0.1.4 // indirect
github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/pkg/errors v0.9.1
@ -33,13 +33,16 @@ require (
github.com/robfig/cron/v3 v3.0.1
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/speps/go-hashids v2.0.0+incompatible
github.com/stretchr/testify v1.5.1
github.com/spf13/afero v1.6.0
github.com/stretchr/testify v1.7.0
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac
github.com/upyun/go-sdk v2.1.0+incompatible
golang.org/x/text v0.3.2
golang.org/x/text v0.3.6
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/go-playground/validator.v9 v9.29.1
gopkg.in/ini.v1 v1.51.0 // indirect
gopkg.in/mail.v2 v2.3.1 // indirect
)
replace github.com/spf13/afero => github.com/chaoqing/afero v1.6.1-0.20210815024925-e47db790683a

935
go.sum
View File

File diff suppressed because it is too large Load Diff

46
pkg/conf/conf.go
Unescape View File

@ -3,7 +3,11 @@ package conf
import (
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/go-ini/ini"
"github.com/spf13/afero"
"github.com/spf13/afero/rclonefs"
"gopkg.in/go-playground/validator.v9"
"path/filepath"
"strings"
)
// database 数据库
@ -84,6 +88,13 @@ type cors struct {
ExposeHeaders []string
}
// RClone配置
// Binds /mnt/ibm:ibm
type rclone struct {
Config string
Binds []string
}
var cfg *ini.File
const defaultConf = `[System]
@ -131,6 +142,7 @@ func Init(path string) {
"Redis": RedisConfig,
"Thumbnail": ThumbConfig,
"CORS": CORSConfig,
"RClone": RCloneConfig,
"Slave": SlaveConfig,
}
for sectionName, sectionStruct := range sections {
@ -147,6 +159,7 @@ func Init(path string) {
util.Log()
}
initRCloneBind()
}
// mapSection 将配置文件的 Section 映射到结构体上
@ -165,3 +178,36 @@ func mapSection(section string, confStruct interface{}) error {
return nil
}
func initRCloneBind(){
if RCloneConfig.Binds[0] == "UNSET"{
return
}
if ok := util.Exists(RCloneConfig.Config); ok{
_ = rclonefs.SetConfigPath(RCloneConfig.Config)
}else{
util.Log().Warning("未找到RClone配置文件: %s", RCloneConfig.Config)
return
}
bindPoints := make(map[string]afero.Fs)
bindPoints["/"] = afero.NewOsFs()
for _, kv := range RCloneConfig.Binds{
if bind := strings.SplitN(kv,":", 2); len(bind)==2{
util.Log().Info("RClone绑定: '%s' -> '%s'", bind[1], bind[0])
if target, err := filepath.Abs(bind[0]); err==nil{
bindPoints[target] = rclonefs.NewRCloneFs(bind[1])
}else{
util.Log().Warning("绑定绝对路径出错: '%s'", bind[0])
}
}else{
util.Log().Warning("RClone绑定不符合格式: %s", kv)
return
}
}
util.OS = rclonefs.NewBindPathFs(bindPoints)
}

6
pkg/conf/defaults.go
Unescape View File

@ -49,6 +49,12 @@ var CORSConfig = &cors{
ExposeHeaders: nil,
}
// RCloneConfig 配置
var RCloneConfig = &rclone{
Config: "rclone.conf",
Binds: []string{"UNSET"},
}
// ThumbConfig 缩略图配置
var ThumbConfig = &thumb{
MaxWidth: 400,

9
pkg/filesystem/archive.go
Unescape View File

@ -83,12 +83,17 @@ func (fs *FileSystem) Compress(ctx context.Context, folderIDs, fileIDs []uint, i
saveFolder,
fmt.Sprintf("archive_%d.zip", time.Now().UnixNano()),
)
zipFile, err := util.CreatNestedFile(zipFilePath)
zipAferoFile, err := util.CreatNestedFile(zipFilePath)
if err != nil {
util.Log().Warning("%s", err)
return "", err
}
defer zipFile.Close()
defer zipAferoFile.Close()
zipFile, ok := zipAferoFile.(*os.File)
if !ok{
return "", fmt.Errorf("not implemented for non OS compression")
}
// 创建压缩文件Writer
zipWriter := zip.NewWriter(zipFile)

13
pkg/filesystem/driver/local/handler.go
Unescape View File

@ -17,6 +17,7 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/spf13/afero"
)
// Driver 本地策略适配器
@ -32,7 +33,7 @@ func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]
root := util.RelativePath(filepath.FromSlash(path))
// 开始遍历路径下的文件、目录
err := filepath.Walk(root,
err := afero.Walk(util.OS, root,
func(path string, info os.FileInfo, err error) error {
// 跳过根目录
if path == root {
@ -73,7 +74,7 @@ func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]
// Get 获取文件内容
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 打开文件
file, err := os.Open(util.RelativePath(path))
file, err := util.OS.Open(util.RelativePath(path))
if err != nil {
util.Log().Debug("无法打开文件：%s", err)
return nil, err
@ -111,7 +112,7 @@ func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, s
// 如果目标目录不存在，创建
basePath := filepath.Dir(dst)
if !util.Exists(basePath) {
err := os.MkdirAll(basePath, 0744)
err := util.OS.MkdirAll(basePath, 0744)
if err != nil {
util.Log().Warning("无法创建目录，%s", err)
return err
@ -119,7 +120,7 @@ func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, s
}
// 创建目标文件
out, err := os.Create(dst)
out, err := util.OS.Create(dst)
if err != nil {
util.Log().Warning("无法创建文件，%s", err)
return err
@ -140,7 +141,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
for _, value := range files {
filePath := util.RelativePath(filepath.FromSlash(value))
if util.Exists(filePath) {
err := os.Remove(filePath)
err := util.OS.Remove(filePath)
if err != nil {
util.Log().Warning("无法删除文件，%s", err)
retErr = err
@ -149,7 +150,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// 尝试删除文件的缩略图（如果有）
_ = os.Remove(util.RelativePath(value + conf.ThumbConfig.FileSuffix))
_ = util.OS.Remove(util.RelativePath(value + conf.ThumbConfig.FileSuffix))
}
return deleteFailed, retErr

14
pkg/filesystem/driver/local/handler_test.go
Unescape View File

@ -21,7 +21,7 @@ func TestHandler_Put(t *testing.T) {
asserts := assert.New(t)
handler := Driver{}
ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
os.Remove(util.RelativePath("test/test/txt"))
util.OS.Remove(util.RelativePath("test/test/txt"))
testCases := []struct {
file io.ReadCloser
@ -62,16 +62,16 @@ func TestHandler_Delete(t *testing.T) {
ctx := context.Background()
filePath := util.RelativePath("test.file")
file, err := os.Create(filePath)
file, err := util.OS.Create(filePath)
asserts.NoError(err)
_ = file.Close()
list, err := handler.Delete(ctx, []string{"test.file"})
asserts.Equal([]string{}, list)
asserts.NoError(err)
file, err = os.Create(filePath)
file, err = util.OS.Create(filePath)
_ = file.Close()
file, _ = os.OpenFile(filePath, os.O_RDWR, os.FileMode(0))
file, _ = util.OS.OpenFile(filePath, os.O_RDWR, os.FileMode(0))
asserts.NoError(err)
list, err = handler.Delete(ctx, []string{"test.file", "test.notexist"})
file.Close()
@ -82,7 +82,7 @@ func TestHandler_Delete(t *testing.T) {
asserts.Equal([]string{}, list)
asserts.NoError(err)
file, err = os.Create(filePath)
file, err = util.OS.Create(filePath)
asserts.NoError(err)
list, err = handler.Delete(ctx, []string{"test.file"})
_ = file.Close()
@ -97,7 +97,7 @@ func TestHandler_Get(t *testing.T) {
defer cancel()
// 成功
file, err := os.Create(util.RelativePath("TestHandler_Get.txt"))
file, err := util.OS.Create(util.RelativePath("TestHandler_Get.txt"))
asserts.NoError(err)
_ = file.Close()
@ -116,7 +116,7 @@ func TestHandler_Thumb(t *testing.T) {
asserts := assert.New(t)
handler := Driver{}
ctx := context.Background()
file, err := os.Create(util.RelativePath("TestHandler_Thumb" + conf.ThumbConfig.FileSuffix))
file, err := util.OS.Create(util.RelativePath("TestHandler_Thumb" + conf.ThumbConfig.FileSuffix))
asserts.NoError(err)
file.Close()

15
pkg/util/io.go
Unescape View File

@ -1,14 +1,19 @@
package util
import (
"github.com/spf13/afero"
"io"
"os"
"path/filepath"
)
var (
OS = afero.NewOsFs()
)
// Exists reports whether the named file or directory exists.
func Exists(name string) bool {
if _, err := os.Stat(name); err != nil {
if _, err := OS.Stat(name); err != nil {
if os.IsNotExist(err) {
return false
}
@ -17,22 +22,22 @@ func Exists(name string) bool {
}
// CreatNestedFile 给定path创建文件，如果目录不存在就递归创建
func CreatNestedFile(path string) (*os.File, error) {
func CreatNestedFile(path string) (afero.File, error) {
basePath := filepath.Dir(path)
if !Exists(basePath) {
err := os.MkdirAll(basePath, 0700)
err := OS.MkdirAll(basePath, 0700)
if err != nil {
Log().Warning("无法创建目录，%s", err)
return nil, err
}
}
return os.Create(path)
return OS.Create(path)
}
// IsEmpty 返回给定目录是否为空目录
func IsEmpty(name string) (bool, error) {
f, err := os.Open(name)
f, err := OS.Open(name)
if err != nil {
return false, err
}

Write Preview
Loading…
Cancel
Save