msb-public
/
cloudreve
mirror of https://github.com/cloudreve/Cloudreve
1
0
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse Source
pull/1203/head
vvisionnn 3 years ago
parent db11d90045 23dc7e370e
commit 321090bff4
89 changed files with 3173 additions and 3363 deletions
Show Stats Download Patch File Download Diff File

61
.github/stale.yml vendored
Unescape View File

@ -0,0 +1,61 @@
# Configuration for probot-stale - https://github.com/probot/stale
# Number of days of inactivity before an Issue or Pull Request becomes stale
daysUntilStale: 360
# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
daysUntilClose: 30
# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
onlyLabels: []
# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
exemptLabels:
- pinned
- security
- "[Status] Maybe Later"
# Set to true to ignore issues in a project (defaults to false)
exemptProjects: true
# Set to true to ignore issues in a milestone (defaults to false)
exemptMilestones: true
# Set to true to ignore issues with an assignee (defaults to false)
exemptAssignees: true
# Label to use when marking as stale
staleLabel: wontfix
# Comment to post when marking as stale. Set to `false` to disable
markComment: >
This issue has been automatically marked as stale because it has not had
recent activity. It will be closed if no further activity occurs. Thank you
for your contributions.
# Comment to post when removing the stale label.
# unmarkComment: >
# Your comment here.
# Comment to post when closing a stale Issue or Pull Request.
# closeComment: >
# Your comment here.
# Limit the number of actions per hour, from 1-30. Default is 30
limitPerRun: 30
# Limit to only `issues` or `pulls`
# only: issues
# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
# pulls:
# daysUntilStale: 30
# markComment: >
# This pull request has been automatically marked as stale because it has not had
# recent activity. It will be closed if no further activity occurs. Thank you
# for your contributions.
# issues:
# exemptLabels:
# - confirmed

96
.github/workflows/build.yml vendored
Unescape View File

@ -9,53 +9,49 @@ jobs:
name: Build
runs-on: ubuntu-18.04
steps:
- name: Set up Golang
uses: actions/setup-go@v1
with:
go-version: 1.17
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
clean: false
submodules: 'recursive'
- run: |
git fetch --prune --unshallow --tags
- name: Get dependencies and build
run: |
go install github.com/rakyll/statik
export PATH=$PATH:~/go/bin/
statik -src=models -f
sudo apt-get update
sudo apt-get -y install gcc-mingw-w64-x86-64
sudo apt-get -y install gcc-arm-linux-gnueabihf libc6-dev-armhf-cross
sudo apt-get -y install gcc-aarch64-linux-gnu libc6-dev-arm64-cross
chmod +x ./build.sh
./build.sh -r b
- name: Upload binary files (windows_amd64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_windows_amd64
path: release/cloudreve*windows_amd64.*
- name: Upload binary files (linux_amd64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_amd64
path: release/cloudreve*linux_amd64.*
- name: Upload binary files (linux_arm)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm
path: release/cloudreve*linux_arm.*
- name: Upload binary files (linux_arm64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm64
path: release/cloudreve*linux_arm64.*
- name: Set up Go 1.17
uses: actions/setup-go@v2
with:
go-version: "1.17"
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
clean: false
submodules: "recursive"
- run: |
git fetch --prune --unshallow --tags
- name: Get dependencies and build
run: |
sudo apt-get update
sudo apt-get -y install gcc-mingw-w64-x86-64
sudo apt-get -y install gcc-arm-linux-gnueabihf libc6-dev-armhf-cross
sudo apt-get -y install gcc-aarch64-linux-gnu libc6-dev-arm64-cross
chmod +x ./build.sh
./build.sh -r b
- name: Upload binary files (windows_amd64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_windows_amd64
path: release/cloudreve*windows_amd64.*
- name: Upload binary files (linux_amd64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_amd64
path: release/cloudreve*linux_amd64.*
- name: Upload binary files (linux_arm)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm
path: release/cloudreve*linux_arm.*
- name: Upload binary files (linux_arm64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm64
path: release/cloudreve*linux_arm64.*

67
.github/workflows/test.yml vendored
Unescape View File

@ -2,46 +2,43 @@ name: Test
on:
pull_request:
branches:
branches:
- master
push:
branches: [ master ]
branches: [master]
jobs:
test:
name: Test
runs-on: ubuntu-18.04
steps:
- name: Set up Golang
uses: actions/setup-go@v1
with:
go-version: 1.17
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
submodules: 'recursive'
- name: Get dependencies
run: |
go get github.com/rakyll/statik
export PATH=$PATH:~/go/bin/
statik -src=models -f
- name: Test
run: go test -coverprofile=coverage.txt -covermode=atomic ./...
- name: Upload binary files (linux_arm)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm
path: release/cloudreve*linux_arm.*
- name: Upload binary files (linux_arm64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm64
path: release/cloudreve*linux_arm64.*
- name: Set up Go 1.17
uses: actions/setup-go@v2
with:
go-version: "1.17"
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
submodules: "recursive"
- name: Build static files
run: |
mkdir assets/build
touch assets/build/test.html
- name: Test
run: go test -coverprofile=coverage.txt -covermode=atomic ./...
- name: Upload binary files (linux_arm)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm
path: release/cloudreve*linux_arm.*
- name: Upload binary files (linux_arm64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm64
path: release/cloudreve*linux_arm64.*

7
.travis.yml
Unescape View File

@ -4,10 +4,9 @@ go:
node_js: "12.16.3"
git:
depth: 1
install:
- go get github.com/rakyll/statik
before_script:
- statik -src=models -f
- mkdir assets/build
- touch assets/build/test.html
script:
- go test -coverprofile=coverage.txt -covermode=atomic ./...
after_success:
@ -27,4 +26,4 @@ deploy:
draft: true
skip_cleanup: true
on:
tags: true
tags: true

16
README.md
Unescape View File

@ -68,7 +68,7 @@ chmod +x ./cloudreve
## :gear: 构建
自行构建前需要拥有 `Go >= 1.13`、`yarn`等必要依赖。
自行构建前需要拥有 `Go >= 1.17`、`yarn`等必要依赖。
#### 克隆代码
@ -85,19 +85,7 @@ cd assets
yarn install
# 开始构建
yarn run build
```
#### 嵌入静态资源
```shell
# 回到项目主目录
cd ../
# 安装 statik, 用于嵌入静态资源
go get github.com/rakyll/statik
# 开始嵌入
statik -src=assets/build/ -include=*.html,*.js,*.json,*.css,*.png,*.svg,*.ico -f
```
#### 编译项目
@ -108,7 +96,7 @@ export COMMIT_SHA=$(git rev-parse --short HEAD)
export VERSION=$(git describe --tags)
# 开始编译
go build -a -o cloudreve -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
go build -a -o cloudreve -ldflags "-s -w -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
```
你也可以使用项目根目录下的`build.sh`快速开始构建：

2
bootstrap/init.go
Unescape View File

@ -37,7 +37,7 @@ func Init(path string) {
{
"both",
func() {
cache.Init()
cache.Init(conf.SystemConfig.Mode == "slave")
},
},
{

141
bootstrap/static.go
Unescape View File

@ -1,21 +1,26 @@
package bootstrap
import (
"bufio"
"embed"
"encoding/json"
"io"
"io/ioutil"
"io/fs"
"net/http"
"path"
"path/filepath"
"github.com/pkg/errors"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
_ "github.com/cloudreve/Cloudreve/v3/statik"
"github.com/gin-contrib/static"
"github.com/rakyll/statik/fs"
)
const StaticFolder = "statics"
var StaticEmbed embed.FS
type GinFS struct {
FS http.FileSystem
}
@ -35,124 +40,100 @@ func (b *GinFS) Open(name string) (http.File, error) {
// Exists 文件是否存在
func (b *GinFS) Exists(prefix string, filepath string) bool {
if _, err := b.FS.Open(filepath); err != nil {
return false
}
return true
}
// InitStatic 初始化静态资源文件
func InitStatic() {
var err error
if util.Exists(util.RelativePath(StaticFolder)) {
util.Log().Info("检测到 statics 目录存在，将使用此目录下的静态资源文件")
StaticFS = static.LocalFile(util.RelativePath("statics"), false)
// 检查静态资源的版本
f, err := StaticFS.Open("version.json")
if err != nil {
util.Log().Warning("静态资源版本标识文件不存在，请重新构建或删除 statics 目录")
return
}
b, err := ioutil.ReadAll(f)
} else {
// 初始化静态资源
embedFS, err := fs.Sub(StaticEmbed, "assets/build")
if err != nil {
util.Log().Warning("无法读取静态资源文件版本，请重新构建或删除 statics 目录")
return
util.Log().Panic("无法初始化静态资源, %s", err)
}
var v staticVersion
if err := json.Unmarshal(b, &v); err != nil {
util.Log().Warning("无法解析静态资源文件版本, %s", err)
return
StaticFS = &GinFS{
FS: http.FS(embedFS),
}
}
// 检查静态资源的版本
f, err := StaticFS.Open("version.json")
if err != nil {
util.Log().Warning("静态资源版本标识文件不存在，请重新构建或删除 statics 目录")
return
}
staticName := "cloudreve-frontend"
if conf.IsPro == "true" {
staticName += "-pro"
}
b, err := io.ReadAll(f)
if err != nil {
util.Log().Warning("无法读取静态资源文件版本，请重新构建或删除 statics 目录")
return
}
if v.Name != staticName {
util.Log().Warning("静态资源版本不匹配，请重新构建或删除 statics 目录")
return
}
var v staticVersion
if err := json.Unmarshal(b, &v); err != nil {
util.Log().Warning("无法解析静态资源文件版本, %s", err)
return
}
if v.Version != conf.RequiredStaticVersion {
util.Log().Warning("静态资源版本不匹配 [当前 %s, 需要: %s]，请重新构建或删除 statics 目录", v.Version, conf.RequiredStaticVersion)
return
}
staticName := "cloudreve-frontend"
if conf.IsPro == "true" {
staticName += "-pro"
}
} else {
StaticFS = &GinFS{}
StaticFS.(*GinFS).FS, err = fs.New()
if err != nil {
util.Log().Panic("无法初始化静态资源, %s", err)
}
if v.Name != staticName {
util.Log().Warning("静态资源版本不匹配，请重新构建或删除 statics 目录")
return
}
if v.Version != conf.RequiredStaticVersion {
util.Log().Warning("静态资源版本不匹配 [当前 %s, 需要: %s]，请重新构建或删除 statics 目录", v.Version, conf.RequiredStaticVersion)
return
}
}
// Eject 抽离内置静态资源
func Eject() {
staticFS, err := fs.New()
// 初始化静态资源
embedFS, err := fs.Sub(StaticEmbed, "assets/build")
if err != nil {
util.Log().Panic("无法初始化静态资源, %s", err)
}
root, err := staticFS.Open("/")
if err != nil {
util.Log().Panic("根目录不存在, %s", err)
}
var walk func(relPath string, object http.File)
walk = func(relPath string, object http.File) {
stat, err := object.Stat()
var walk func(relPath string, d fs.DirEntry, err error) error
walk = func(relPath string, d fs.DirEntry, err error) error {
if err != nil {
util.Log().Error("无法获取[%s]的信息, %s, 跳过...", relPath, err)
return
return errors.Errorf("无法获取[%s]的信息, %s, 跳过...", relPath, err)
}
if !stat.IsDir() {
if !d.IsDir() {
// 写入文件
out, err := util.CreatNestedFile(util.RelativePath(StaticFolder + relPath))
out, err := util.CreatNestedFile(filepath.Join(util.RelativePath(""), StaticFolder, relPath))
defer out.Close()
if err != nil {
util.Log().Error("无法创建文件[%s], %s, 跳过...", relPath, err)
return
return errors.Errorf("无法创建文件[%s], %s, 跳过...", relPath, err)
}
util.Log().Info("导出 [%s]...", relPath)
if _, err := io.Copy(out, object); err != nil {
util.Log().Error("无法写入文件[%s], %s, 跳过...", relPath, err)
return
obj, _ := embedFS.Open(relPath)
if _, err := io.Copy(out, bufio.NewReader(obj)); err != nil {
return errors.Errorf("无法写入文件[%s], %s, 跳过...", relPath, err)
}
} else {
// 列出目录
objects, err := object.Readdir(0)
if err != nil {
util.Log().Error("无法步入子目录[%s], %s, 跳过...", relPath, err)
return
}
// 递归遍历子目录
for _, newObject := range objects {
newPath := path.Join(relPath, newObject.Name())
newRoot, err := staticFS.Open(newPath)
if err != nil {
util.Log().Error("无法打开对象[%s], %s, 跳过...", newPath, err)
continue
}
walk(newPath, newRoot)
}
}
return nil
}
util.Log().Info("开始导出内置静态资源...")
walk("/", root)
// util.Log().Info("开始导出内置静态资源...")
err = fs.WalkDir(embedFS, ".", walk)
if err != nil {
util.Log().Error("导出内置静态资源遇到错误：%s", err)
return
}
util.Log().Info("内置静态资源导出完成")
}

130
build.sh
Unescape View File

@ -1,13 +1,16 @@
#!/bin/bash
REPO=$(cd $(dirname $0); pwd)
REPO=$(
cd $(dirname $0)
pwd
)
COMMIT_SHA=$(git rev-parse --short HEAD)
VERSION=$(git describe --tags)
ASSETS="false"
BINARY="false"
RELEASE="false"
debugInfo () {
debugInfo() {
echo "Repo: $REPO"
echo "Build assets: $ASSETS"
echo "Build binary: $BINARY"
@ -16,10 +19,9 @@ debugInfo () {
echo "Commit: $COMMIT_SHA"
}
buildAssets () {
buildAssets() {
cd $REPO
rm -rf assets/build
rm -f statik/statik.go
export CI=false
@ -27,94 +29,88 @@ buildAssets () {
yarn install
yarn run build
if ! [ -x "$(command -v statik)" ]; then
export CGO_ENABLED=0
go get github.com/rakyll/statik
fi
cd $REPO
statik -src=assets/build/ -include=*.html,*.js,*.json,*.css,*.png,*.svg,*.ico,*.ttf -f
cd build
rm -rf *.map
}
buildBinary () {
buildBinary() {
cd $REPO
go build -a -o cloudreve -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
}
_build() {
local osarch=$1
IFS=/ read -r -a arr <<<"$osarch"
os="${arr[0]}"
arch="${arr[1]}"
gcc="${arr[2]}"
# Go build to build the binary.
export GOOS=$os
export GOARCH=$arch
export CC=$gcc
export CGO_ENABLED=1
if [ -n "$VERSION" ]; then
out="release/cloudreve_${VERSION}_${os}_${arch}"
else
out="release/cloudreve_${COMMIT_SHA}_${os}_${arch}"
fi
go build -a -o "${out}" -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
if [ "$os" = "windows" ]; then
mv $out release/cloudreve.exe
zip -j -q "${out}.zip" release/cloudreve.exe
rm -f "release/cloudreve.exe"
else
mv $out release/cloudreve
tar -zcvf "${out}.tar.gz" -C release cloudreve
rm -f "release/cloudreve"
fi
local osarch=$1
IFS=/ read -r -a arr <<<"$osarch"
os="${arr[0]}"
arch="${arr[1]}"
gcc="${arr[2]}"
# Go build to build the binary.
export GOOS=$os
export GOARCH=$arch
export CC=$gcc
export CGO_ENABLED=1
if [ -n "$VERSION" ]; then
out="release/cloudreve_${VERSION}_${os}_${arch}"
else
out="release/cloudreve_${COMMIT_SHA}_${os}_${arch}"
fi
go build -a -o "${out}" -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
if [ "$os" = "windows" ]; then
mv $out release/cloudreve.exe
zip -j -q "${out}.zip" release/cloudreve.exe
rm -f "release/cloudreve.exe"
else
mv $out release/cloudreve
tar -zcvf "${out}.tar.gz" -C release cloudreve
rm -f "release/cloudreve"
fi
}
release(){
release() {
cd $REPO
## List of architectures and OS to test coss compilation.
SUPPORTED_OSARCH="linux/amd64/gcc linux/arm/arm-linux-gnueabihf-gcc windows/amd64/x86_64-w64-mingw32-gcc linux/arm64/aarch64-linux-gnu-gcc"
echo "Release builds for OS/Arch/CC: ${SUPPORTED_OSARCH}"
for each_osarch in ${SUPPORTED_OSARCH}; do
_build "${each_osarch}"
_build "${each_osarch}"
done
}
usage() {
echo "Usage: $0 [-a] [-c] [-b] [-r]" 1>&2;
exit 1;
echo "Usage: $0 [-a] [-c] [-b] [-r]" 1>&2
exit 1
}
while getopts "bacr:d" o; do
case "${o}" in
b)
ASSETS="true"
BINARY="true"
;;
a)
ASSETS="true"
;;
c)
BINARY="true"
;;
r)
ASSETS="true"
RELEASE="true"
;;
d)
DEBUG="true"
;;
*)
usage
;;
b)
ASSETS="true"
BINARY="true"
;;
a)
ASSETS="true"
;;
c)
BINARY="true"
;;
r)
ASSETS="true"
RELEASE="true"
;;
d)
DEBUG="true"
;;
*)
usage
;;
esac
done
shift $((OPTIND-1))
shift $((OPTIND - 1))
if [ "$DEBUG" = "true" ]; then
debugInfo

69
go.mod
Unescape View File

@ -1,47 +1,90 @@
module github.com/cloudreve/Cloudreve/v3
go 1.13
go 1.17
require (
github.com/DATA-DOG/go-sqlmock v1.3.3
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible
github.com/HFO4/aliyun-oss-go-sdk v2.2.3+incompatible
github.com/aws/aws-sdk-go v1.31.5
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4
github.com/fatih/color v1.7.0
github.com/gin-contrib/cors v1.3.0
github.com/gin-contrib/gzip v0.0.2-0.20200226035851-25bef2ef21e8
github.com/gin-contrib/sessions v0.0.1
github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2
github.com/gin-gonic/gin v1.5.0
github.com/gin-gonic/gin v1.7.0
github.com/go-ini/ini v1.50.0
github.com/go-mail/mail v2.3.1+incompatible
github.com/gofrs/uuid v4.0.0+incompatible
github.com/gomodule/redigo v2.0.0+incompatible
github.com/google/go-querystring v1.0.0
github.com/gorilla/websocket v1.4.1
github.com/hashicorp/go-version v1.2.0
github.com/hashicorp/go-version v1.3.0
github.com/jinzhu/gorm v1.9.11
github.com/juju/ratelimit v1.0.1
github.com/mattn/go-colorable v0.1.4 // indirect
github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/pkg/errors v0.9.1
github.com/pquerna/otp v1.2.0
github.com/qiniu/api.v7/v7 v7.4.0
github.com/qiniu/go-sdk/v7 v7.11.1
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1
github.com/rakyll/statik v0.1.7
github.com/robfig/cron/v3 v3.0.1
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/speps/go-hashids v2.0.0+incompatible
github.com/stretchr/testify v1.5.1
github.com/stretchr/testify v1.6.1
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac
github.com/upyun/go-sdk v2.1.0+incompatible
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
golang.org/x/text v0.3.6
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
golang.org/x/text v0.3.7
gopkg.in/go-playground/validator.v9 v9.29.1
)
require (
cloud.google.com/go v0.37.4 // indirect
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-playground/validator/v10 v10.8.0 // indirect
github.com/go-sql-driver/mysql v1.5.0 // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
github.com/golang/protobuf v1.3.3 // indirect
github.com/google/certificate-transparency-go v1.0.21 // indirect
github.com/gorilla/context v1.1.1 // indirect
github.com/gorilla/securecookie v1.1.1 // indirect
github.com/gorilla/sessions v1.1.3 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.3.0 // indirect
github.com/json-iterator/go v1.1.9 // indirect
github.com/katzenpost/core v0.0.7 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lib/pq v1.1.1 // indirect
github.com/mattn/go-colorable v0.1.4 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/mattn/go-sqlite3 v1.11.0 // indirect
github.com/mitchellh/mapstructure v1.1.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/mozillazg/go-httpheader v0.2.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438 // indirect
github.com/satori/go.uuid v1.2.0 // indirect
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/stretchr/objx v0.2.0 // indirect
github.com/ugorji/go/codec v1.1.7 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
golang.org/x/sys v0.0.0-20211020174200-9d6173849985 // indirect
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/ini.v1 v1.51.0 // indirect
gopkg.in/mail.v2 v2.3.1 // indirect
gopkg.in/yaml.v2 v2.2.8 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)

97
go.sum
Unescape View File

@ -5,6 +5,8 @@ cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7h
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/HFO4/aliyun-oss-go-sdk v2.2.3+incompatible h1:aX/+gJM2dAMDDy3JqWS0DJn3JfOUchf4k37P5TbBKU8=
github.com/HFO4/aliyun-oss-go-sdk v2.2.3+incompatible/go.mod h1:8KDiKVrHK/UbXAhj+iQGp1m40rQa+UAvzBi7m22KywI=
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
@ -12,9 +14,6 @@ github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7I
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/aliyun/aliyun-oss-go-sdk v2.0.0/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible h1:A3oZlWPD/Poa19FvNbw+Zu4yKAurDBTjlRDilYGBiS4=
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/aws/aws-sdk-go v1.31.5 h1:DFA7BzTydO4etqsTja+x7UfkOKQUv1xzEluLvNk81L0=
github.com/aws/aws-sdk-go v1.31.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
@ -31,6 +30,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7 h1:Puu1hUwfps3+1CUzYdAZXijuvLuRMirgiXdf3zsM2Ig=
github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -61,8 +61,9 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm
github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2 h1:xLG16iua01X7Gzms9045s2Y2niNpvSY/Zb1oBwgNYZY=
github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2/go.mod h1:VhW/Ch/3FhimwZb8Oj+qJmdMmoB8r7lmJ5auRjm50oQ=
github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
github.com/gin-gonic/gin v1.5.0 h1:fi+bqFAx/oLK54somfCtEZs9HeH1LHVoEPUgARpTqyc=
github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do=
github.com/gin-gonic/gin v1.7.0 h1:jGB9xAJQ12AIGNB4HguylppmDK1Am9ppF7XnGXXJuoU=
github.com/gin-gonic/gin v1.7.0/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/go-ini/ini v1.50.0 h1:ogX6RS8VstVN8MJcwhEP78hHhWaI3klN02+97bByabY=
github.com/go-ini/ini v1.50.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
@ -70,10 +71,19 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-mail/mail v2.3.1+incompatible h1:UzNOn0k5lpfVtO31cK3hn6I4VEVGhe3lX8AJBAxXExM=
github.com/go-mail/mail v2.3.1+incompatible/go.mod h1:VPWjmmNyRsWXQZHVHT3g0YbIINUkSmuKOiLIDkWbL6M=
github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc=
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM=
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-playground/validator/v10 v10.8.0 h1:1kAa0fCrnpv+QYdkdcRzrRM7AyYs5o8+jZdJCz9xj6k=
github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
@ -89,8 +99,9 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
@ -119,8 +130,8 @@ github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9R
github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w=
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw=
github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/jinzhu/gorm v1.9.11 h1:gaHGvE+UnWGlbWG4Y3FUwY1EcZ5n6S9WtqBA/uySMLE=
@ -132,8 +143,9 @@ github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
@ -146,22 +158,28 @@ github.com/kidstuff/mongostore v0.0.0-20181113001930-e650cd85ee4b/go.mod h1:g2nV
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@ -179,13 +197,12 @@ github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2/go.mod h1:wAQ
github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -200,17 +217,20 @@ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/qiniu/api.v7/v7 v7.4.0 h1:9dZMVQifh31QGFLVaHls6akCaS2rlj3du8MnEFd7XjQ=
github.com/qiniu/api.v7/v7 v7.4.0/go.mod h1:VE5oC5rkE1xul0u1S2N0b2Uxq9/6hZzhyqjgK25XDcM=
github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk=
github.com/qiniu/go-sdk/v7 v7.11.1 h1:/LZ9rvFS4p6SnszhGv11FNB1+n4OZvBCwFg7opH5Ovs=
github.com/qiniu/go-sdk/v7 v7.11.1/go.mod h1:btsaOc8CA3hdVloULfFdDgDc+g4f3TDZEFsDY0BLE+w=
github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=
github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438 h1:jnz/4VenymvySjE+Ez511s0pqVzkUOmr1fwCVytNNWk=
github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438/go.mod h1:wTPjTepVu7uJBYgZ0SdWHQlIas582j6cn2jgk4DDdlg=
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1 h1:leEwA4MD1ew0lNgzz6Q4G76G3AEfeci+TMggN6WuFRs=
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY=
github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@ -227,8 +247,9 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible h1:dqpmYaez7VBT7PCRBcBxkzlDOiTk7Td8ATiia1b1GuE=
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac h1:PSBhZblOjdwH7SIVgcue+7OlnLHkM45KuScLZ+PiVbQ=
@ -244,10 +265,12 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/image v0.0.0-20190501045829-6d32002ffd75 h1:TbGuee8sSq15Iguxu4deQ7+Bqq/d2rsQejGcEtADAMQ=
golang.org/x/image v0.0.0-20190501045829-6d32002ffd75/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
@ -266,8 +289,9 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -275,6 +299,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -285,14 +311,21 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211020174200-9d6173849985 h1:LOlKVhfDyahgmqa97awczplwkjzNaELFg3zRIJ13RYo=
golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@ -314,8 +347,10 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
@ -328,8 +363,12 @@ gopkg.in/mail.v2 v2.3.1 h1:WYFn/oANrAGP2C0dcV6/pbkPzv8yGzqTjPmTeO7qoXk=
gopkg.in/mail.v2 v2.3.1/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

4
main.go
Unescape View File

@ -1,6 +1,7 @@
package main
import (
"embed"
"flag"
"github.com/cloudreve/Cloudreve/v3/bootstrap"
@ -15,6 +16,9 @@ var (
scriptName string
)
//go:embed assets/build
var StaticEmbed embed.FS
func init() {
flag.StringVar(&confPath, "c", util.RelativePath("conf.ini"), "配置文件路径")
flag.BoolVar(&isEject, "eject", false, "导出内置静态资源")

199
middleware/auth.go
Unescape View File

@ -1,7 +1,17 @@
package middleware
import (
"bytes"
"context"
"crypto/md5"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/oss"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/upyun"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"io/ioutil"
"net/http"
model "github.com/cloudreve/Cloudreve/v3/models"
@ -178,28 +188,23 @@ func RemoteCallbackAuth() gin.HandlerFunc {
// QiniuCallbackAuth 七牛回调签名验证
func QiniuCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
//// 验证key并查找用户
//resp, user := uploadCallbackCheck(c)
//if resp.Code != 0 {
// c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
// c.Abort()
// return
//}
//
//// 验证回调是否来自qiniu
//mac := qbox.NewMac(user.Policy.AccessKey, user.Policy.SecretKey)
//ok, err := mac.VerifyCallback(c.Request)
//if err != nil {
// util.Log().Debug("无法验证回调请求，%s", err)
// c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "无法验证回调请求"})
// c.Abort()
// return
//}
//if !ok {
// c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名无效"})
// c.Abort()
// return
//}
session := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
// 验证回调是否来自qiniu
mac := qbox.NewMac(session.Policy.AccessKey, session.Policy.SecretKey)
ok, err := mac.VerifyCallback(c.Request)
if err != nil {
util.Log().Debug("无法验证回调请求，%s", err)
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "无法验证回调请求"})
c.Abort()
return
}
if !ok {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名无效"})
c.Abort()
return
}
c.Next()
}
@ -208,21 +213,13 @@ func QiniuCallbackAuth() gin.HandlerFunc {
// OSSCallbackAuth 阿里云OSS回调签名验证
func OSSCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
//// 验证key并查找用户
//resp, _ := uploadCallbackCheck(c)
//if resp.Code != 0 {
// c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
// c.Abort()
// return
//}
//
//err := oss.VerifyCallbackSignature(c.Request)
//if err != nil {
// util.Log().Debug("回调签名验证失败，%s", err)
// c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名验证失败"})
// c.Abort()
// return
//}
err := oss.VerifyCallbackSignature(c.Request)
if err != nil {
util.Log().Debug("回调签名验证失败，%s", err)
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名验证失败"})
c.Abort()
return
}
c.Next()
}
@ -231,103 +228,57 @@ func OSSCallbackAuth() gin.HandlerFunc {
// UpyunCallbackAuth 又拍云回调签名验证
func UpyunCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
//// 验证key并查找用户
//resp, user := uploadCallbackCheck(c)
//if resp.Code != 0 {
// c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
// c.Abort()
// return
//}
//
//// 获取请求正文
//body, err := ioutil.ReadAll(c.Request.Body)
//c.Request.Body.Close()
//if err != nil {
// c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: err.Error()})
// c.Abort()
// return
//}
//
//c.Request.Body = ioutil.NopCloser(bytes.NewReader(body))
//
//// 准备验证Upyun回调签名
//handler := upyun.Driver{Policy: &user.Policy}
//contentMD5 := c.Request.Header.Get("Content-Md5")
//date := c.Request.Header.Get("Date")
//actualSignature := c.Request.Header.Get("Authorization")
//
//// 计算正文MD5
//actualContentMD5 := fmt.Sprintf("%x", md5.Sum(body))
//if actualContentMD5 != contentMD5 {
// c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "MD5不一致"})
// c.Abort()
// return
//}
//
//// 计算理论签名
//signature := handler.Sign(context.Background(), []string{
// "POST",
// c.Request.URL.Path,
// date,
// contentMD5,
//})
//
//// 对比签名
//if signature != actualSignature {
// c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "鉴权失败"})
// c.Abort()
// return
//}
session := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
c.Next()
}
}
// 获取请求正文
body, err := ioutil.ReadAll(c.Request.Body)
c.Request.Body.Close()
if err != nil {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: err.Error()})
c.Abort()
return
}
// OneDriveCallbackAuth OneDrive回调签名验证
// TODO 解耦
func OneDriveCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
//// 验证key并查找用户
//resp, _ := uploadCallbackCheck(c)
//if resp.Code != 0 {
// c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
// c.Abort()
// return
//}
//
//// 发送回调结束信号
//onedrive.FinishCallback(c.Param("key"))
c.Request.Body = ioutil.NopCloser(bytes.NewReader(body))
c.Next()
}
}
// 准备验证Upyun回调签名
handler := upyun.Driver{Policy: &session.Policy}
contentMD5 := c.Request.Header.Get("Content-Md5")
date := c.Request.Header.Get("Date")
actualSignature := c.Request.Header.Get("Authorization")
// COSCallbackAuth 腾讯云COS回调签名验证
// TODO 解耦 测试
func COSCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
//// 验证key并查找用户
//resp, _ := uploadCallbackCheck(c)
//if resp.Code != 0 {
// c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
// c.Abort()
// return
//}
// 计算正文MD5
actualContentMD5 := fmt.Sprintf("%x", md5.Sum(body))
if actualContentMD5 != contentMD5 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "MD5不一致"})
c.Abort()
return
}
// 计算理论签名
signature := handler.Sign(context.Background(), []string{
"POST",
c.Request.URL.Path,
date,
contentMD5,
})
// 对比签名
if signature != actualSignature {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "鉴权失败"})
c.Abort()
return
}
c.Next()
}
}
// S3CallbackAuth Amazon S3回调签名验证
func S3CallbackAuth() gin.HandlerFunc {
// OneDriveCallbackAuth OneDrive回调签名验证
func OneDriveCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
//// 验证key并查找用户
//resp, _ := uploadCallbackCheck(c)
//if resp.Code != 0 {
// c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
// c.Abort()
// return
//}
// 发送回调结束信号
mq.GlobalMQ.Publish(c.Param("sessionID"), mq.Message{})
c.Next()
}

521
middleware/auth_test.go
Unescape View File

@ -3,21 +3,24 @@ package middleware
import (
"database/sql"
"errors"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
"github.com/qiniu/api.v7/v7/auth/qbox"
"github.com/stretchr/testify/assert"
)
@ -223,19 +226,31 @@ func TestWebDAVAuth(t *testing.T) {
}
func TestRemoteCallbackAuth(t *testing.T) {
func TestUseUploadSession(t *testing.T) {
asserts := assert.New(t)
rec := httptest.NewRecorder()
AuthFunc := RemoteCallbackAuth()
AuthFunc := UseUploadSession("local")
// sessionID 为空
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/sessionID", nil)
authInstance := auth.HMACAuth{SecretKey: []byte("123")}
auth.SignRequest(authInstance, c.Request, 0)
AuthFunc(c)
asserts.True(c.IsAborted())
}
// 成功
{
cache.Set(
"callback_testCallBackRemote",
filesystem.UploadSessionCachePrefix+"testCallBackRemote",
serializer.UploadSession{
UID: 1,
PolicyID: 513,
VirtualPath: "/",
Policy: model.Policy{Type: "local"},
},
0,
)
@ -248,7 +263,7 @@ func TestRemoteCallbackAuth(t *testing.T) {
WillReturnRows(sqlmock.NewRows([]string{"id", "secret_key"}).AddRow(2, "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackRemote"},
{"sessionID", "testCallBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil)
authInstance := auth.HMACAuth{SecretKey: []byte("123")}
@ -257,79 +272,95 @@ func TestRemoteCallbackAuth(t *testing.T) {
asserts.NoError(mock.ExpectationsWereMet())
asserts.False(c.IsAborted())
}
}
// Callback Key 不存在
{
func TestUploadCallbackCheck(t *testing.T) {
a := assert.New(t)
rec := httptest.NewRecorder()
// 上传会话不存在
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackRemote"},
{"sessionID", "testSessionNotExist"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil)
authInstance := auth.HMACAuth{SecretKey: []byte("123")}
auth.SignRequest(authInstance, c.Request, 0)
AuthFunc(c)
asserts.True(c.IsAborted())
res := uploadCallbackCheck(c, "local")
a.Contains("上传会话不存在或已过期", res.Msg)
}
// 用户不存在
// 上传策略不一致
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"sessionID", "testPolicyNotMatch"},
}
cache.Set(
"callback_testCallBackRemote",
filesystem.UploadSessionCachePrefix+"testPolicyNotMatch",
serializer.UploadSession{
UID: 1,
PolicyID: 550,
VirtualPath: "/",
Policy: model.Policy{Type: "remote"},
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil)
authInstance := auth.HMACAuth{SecretKey: []byte("123")}
auth.SignRequest(authInstance, c.Request, 0)
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.True(c.IsAborted())
res := uploadCallbackCheck(c, "local")
a.Contains("Policy not supported", res.Msg)
}
// 签名错误
// 用户不存在
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"sessionID", "testUserNotExist"},
}
cache.Set(
"callback_testCallBackRemote",
filesystem.UploadSessionCachePrefix+"testUserNotExist",
serializer.UploadSession{
UID: 1,
PolicyID: 514,
UID: 313,
VirtualPath: "/",
Policy: model.Policy{Type: "remote"},
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[514]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "secret_key"}).AddRow(2, "123"))
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}))
res := uploadCallbackCheck(c, "remote")
a.Contains("找不到用户", res.Msg)
a.NoError(mock.ExpectationsWereMet())
_, ok := cache.Get(filesystem.UploadSessionCachePrefix + "testUserNotExist")
a.False(ok)
}
}
func TestRemoteCallbackAuth(t *testing.T) {
asserts := assert.New(t)
rec := httptest.NewRecorder()
AuthFunc := RemoteCallbackAuth()
// 成功
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackRemote"},
}
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{SecretKey: "123"},
})
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil)
authInstance := auth.HMACAuth{SecretKey: []byte("123")}
auth.SignRequest(authInstance, c.Request, 0)
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.True(c.IsAborted())
asserts.False(c.IsAborted())
}
// Callback Key 为空
// 签名错误
{
c, _ := gin.CreateTestContext(rec)
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote", nil)
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{SecretKey: "123"},
})
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil)
AuthFunc(c)
asserts.True(c.IsAborted())
}
@ -340,39 +371,17 @@ func TestQiniuCallbackAuth(t *testing.T) {
rec := httptest.NewRecorder()
AuthFunc := QiniuCallbackAuth()
// Callback Key 相关验证失败
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testQiniuBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testQiniuBackRemote", nil)
AuthFunc(c)
asserts.True(c.IsAborted())
}
// 成功
{
cache.Set(
"callback_testCallBackQiniu",
serializer.UploadSession{
UID: 1,
PolicyID: 515,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[515]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackQiniu"},
}
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
},
})
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/qiniu/testCallBackQiniu", nil)
mac := qbox.NewMac("123", "123")
token, err := mac.SignRequest(c.Request)
@ -385,33 +394,21 @@ func TestQiniuCallbackAuth(t *testing.T) {
// 验证失败
{
cache.Set(
"callback_testCallBackQiniu",
serializer.UploadSession{
UID: 1,
PolicyID: 516,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[516]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackQiniu"},
}
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
},
})
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/qiniu/testCallBackQiniu", nil)
mac := qbox.NewMac("123", "123")
mac := qbox.NewMac("123", "1213")
token, err := mac.SignRequest(c.Request)
asserts.NoError(err)
c.Request.Header["Authorization"] = []string{"QBox " + token + " "}
c.Request.Header["Authorization"] = []string{"QBox " + token}
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.True(c.IsAborted())
}
}
@ -421,76 +418,41 @@ func TestOSSCallbackAuth(t *testing.T) {
rec := httptest.NewRecorder()
AuthFunc := OSSCallbackAuth()
// Callback Key 相关验证失败
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testOSSBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/testQiniuBackRemote", nil)
AuthFunc(c)
asserts.True(c.IsAborted())
}
// 签名验证失败
{
cache.Set(
"callback_testCallBackOSS",
serializer.UploadSession{
UID: 1,
PolicyID: 517,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[517]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackOSS"},
}
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
},
})
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/testCallBackOSS", nil)
mac := qbox.NewMac("123", "123")
token, err := mac.SignRequest(c.Request)
asserts.NoError(err)
c.Request.Header["Authorization"] = []string{"QBox " + token}
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.True(c.IsAborted())
}
// 成功
{
cache.Set(
"callback_TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH",
serializer.UploadSession{
UID: 1,
PolicyID: 518,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[518]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
}
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
},
})
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH", ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)))
c.Request.Header["Authorization"] = []string{"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="}
c.Request.Header["X-Oss-Pub-Key-Url"] = []string{"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="}
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.False(c.IsAborted())
}
@ -507,130 +469,71 @@ func TestUpyunCallbackAuth(t *testing.T) {
rec := httptest.NewRecorder()
AuthFunc := UpyunCallbackAuth()
// Callback Key 相关验证失败
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testUpyunBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil)
AuthFunc(c)
asserts.True(c.IsAborted())
}
// 无法获取请求正文
{
cache.Set(
"callback_testCallBackUpyun",
serializer.UploadSession{
UID: 1,
PolicyID: 509,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[519]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackUpyun"},
}
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
},
})
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(fakeRead("")))
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.True(c.IsAborted())
}
// 正文MD5不一致
{
cache.Set(
"callback_testCallBackUpyun",
serializer.UploadSession{
UID: 1,
PolicyID: 510,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[520]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackUpyun"},
}
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
},
})
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
c.Request.Header["Content-Md5"] = []string{"123"}
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.True(c.IsAborted())
}
// 签名不一致
{
cache.Set(
"callback_testCallBackUpyun",
serializer.UploadSession{
UID: 1,
PolicyID: 511,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[521]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackUpyun"},
}
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
},
})
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
c.Request.Header["Content-Md5"] = []string{"c4ca4238a0b923820dcc509a6f75849b"}
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.True(c.IsAborted())
}
// 成功
{
cache.Set(
"callback_testCallBackUpyun",
serializer.UploadSession{
UID: 1,
PolicyID: 512,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[522]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackUpyun"},
}
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
},
})
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
c.Request.Header["Content-Md5"] = []string{"c4ca4238a0b923820dcc509a6f75849b"}
c.Request.Header["Authorization"] = []string{"UPYUN 123:GWueK9x493BKFFk5gmfdO2Mn6EM="}
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.False(c.IsAborted())
}
}
@ -640,87 +543,28 @@ func TestOneDriveCallbackAuth(t *testing.T) {
rec := httptest.NewRecorder()
AuthFunc := OneDriveCallbackAuth()
// Callback Key 相关验证失败
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testUpyunBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil)
AuthFunc(c)
asserts.True(c.IsAborted())
}
// 成功
{
cache.Set(
"callback_testCallBackUpyun",
serializer.UploadSession{
UID: 1,
PolicyID: 512,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[657]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackUpyun"},
{"sessionID", "TestOneDriveCallbackAuth"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.False(c.IsAborted())
}
}
func TestCOSCallbackAuth(t *testing.T) {
asserts := assert.New(t)
rec := httptest.NewRecorder()
AuthFunc := COSCallbackAuth()
// Callback Key 相关验证失败
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testUpyunBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil)
AuthFunc(c)
asserts.True(c.IsAborted())
}
// 成功
{
cache.Set(
"callback_testCallBackUpyun",
serializer.UploadSession{
UID: 1,
PolicyID: 512,
VirtualPath: "/",
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[702]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackUpyun"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
})
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/TestOneDriveCallbackAuth", ioutil.NopCloser(strings.NewReader("1")))
res := mq.GlobalMQ.Subscribe("TestOneDriveCallbackAuth", 1)
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
select {
case <-res:
case <-time.After(time.Millisecond * 500):
asserts.Fail("mq message should be published")
}
asserts.False(c.IsAborted())
}
}
@ -759,46 +603,3 @@ func TestIsAdmin(t *testing.T) {
asserts.False(c.IsAborted())
}
}
func TestS3CallbackAuth(t *testing.T) {
asserts := assert.New(t)
rec := httptest.NewRecorder()
AuthFunc := S3CallbackAuth()
// Callback Key 相关验证失败
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testUpyunBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil)
AuthFunc(c)
asserts.True(c.IsAborted())
}
// 成功
{
cache.Set(
"callback_testCallBackUpyun",
serializer.UploadSession{
UID: 1,
PolicyID: 512,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[702]"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackUpyun"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
AuthFunc(c)
asserts.False(c.IsAborted())
asserts.NoError(mock.ExpectationsWereMet())
}
}

117
models/defaults.go
Unescape View File

@ -0,0 +1,117 @@
package model
import (
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gofrs/uuid"
)
var defaultSettings = []Setting{
{Name: "siteURL", Value: `http://localhost`, Type: "basic"},
{Name: "siteName", Value: `Cloudreve`, Type: "basic"},
{Name: "siteICPId", Value: ``, Type: "basic"},
{Name: "register_enabled", Value: `1`, Type: "register"},
{Name: "default_group", Value: `2`, Type: "register"},
{Name: "siteKeywords", Value: `网盘，网盘`, Type: "basic"},
{Name: "siteDes", Value: `Cloudreve`, Type: "basic"},
{Name: "siteTitle", Value: `平步云端`, Type: "basic"},
{Name: "siteScript", Value: ``, Type: "basic"},
{Name: "siteID", Value: uuid.Must(uuid.NewV4()).String(), Type: "basic"},
{Name: "fromName", Value: `Cloudreve`, Type: "mail"},
{Name: "mail_keepalive", Value: `30`, Type: "mail"},
{Name: "fromAdress", Value: `no-reply@acg.blue`, Type: "mail"},
{Name: "smtpHost", Value: `smtp.mxhichina.com`, Type: "mail"},
{Name: "smtpPort", Value: `25`, Type: "mail"},
{Name: "replyTo", Value: `abslant@126.com`, Type: "mail"},
{Name: "smtpUser", Value: `no-reply@acg.blue`, Type: "mail"},
{Name: "smtpPass", Value: ``, Type: "mail"},
{Name: "smtpEncryption", Value: `0`, Type: "mail"},
{Name: "maxEditSize", Value: `4194304`, Type: "file_edit"},
{Name: "archive_timeout", Value: `60`, Type: "timeout"},
{Name: "download_timeout", Value: `60`, Type: "timeout"},
{Name: "preview_timeout", Value: `60`, Type: "timeout"},
{Name: "doc_preview_timeout", Value: `60`, Type: "timeout"},
{Name: "upload_session_timeout", Value: `86400`, Type: "timeout"},
{Name: "slave_api_timeout", Value: `60`, Type: "timeout"},
{Name: "slave_node_retry", Value: `3`, Type: "slave"},
{Name: "slave_ping_interval", Value: `60`, Type: "slave"},
{Name: "slave_recover_interval", Value: `120`, Type: "slave"},
{Name: "slave_transfer_timeout", Value: `172800`, Type: "timeout"},
{Name: "onedrive_monitor_timeout", Value: `600`, Type: "timeout"},
{Name: "share_download_session_timeout", Value: `2073600`, Type: "timeout"},
{Name: "onedrive_callback_check", Value: `20`, Type: "timeout"},
{Name: "folder_props_timeout", Value: `300`, Type: "timeout"},
{Name: "chunk_retries", Value: `5`, Type: "retry"},
{Name: "onedrive_source_timeout", Value: `1800`, Type: "timeout"},
{Name: "reset_after_upload_failed", Value: `0`, Type: "upload"},
{Name: "use_temp_chunk_buffer", Value: `1`, Type: "upload"},
{Name: "login_captcha", Value: `0`, Type: "login"},
{Name: "reg_captcha", Value: `0`, Type: "login"},
{Name: "email_active", Value: `0`, Type: "register"},
{Name: "mail_activation_template", Value: `<!DOCTYPE html PUBLIC"-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box;
font-size: 14px; margin: 0;"><head><meta name="viewport"content="width=device-width"/><meta http-equiv="Content-Type"content="text/html; charset=UTF-8"/><title>激活您的账户</title><style type="text/css">img{max-width:100%}body{-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:none;width:100%!important;height:100%;line-height:1.6em}body{background-color:#f6f6f6}@media only screen and(max-width:640px){body{padding:0!important}h1{font-weight:800!important;margin:20px 0 5px!important}h2{font-weight:800!important;margin:20px 0 5px!important}h3{font-weight:800!important;margin:20px 0 5px!important}h4{font-weight:800!important;margin:20px 0 5px!important}h1{font-size:22px!important}h2{font-size:18px!important}h3{font-size:16px!important}.container{padding:0!important;width:100%!important}.content{padding:0!important}.content-wrap{padding:10px!important}.invoice{width:100%!important}}</style></head><body itemscope itemtype="http://schema.org/EmailMessage"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><table class="body-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; margin: 0;"><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td><td class="container"width="600"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; margin: 0 auto;"valign="top"><div class="content"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 20px;"><table class="main"width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px
solid #e9e9e9;"bgcolor="#fff"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
14px; margin: 0;"><td class="alert alert-warning"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #009688; margin: 0; padding: 20px;"align="center"bgcolor="#FF9F00"valign="top">激活{siteTitle}账户</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 20px;"valign="top"><table width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">亲爱的<strong style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{userName}</strong>：</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您注册{siteTitle},请点击下方按钮完成账户激活。</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top"><a href="{activationUrl}"class="btn-primary"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color: #FFF; text-decoration: none; line-height: 2em; font-weight: bold; text-align: center; cursor: pointer; display: inline-block; border-radius: 5px; text-transform: capitalize; background-color: #009688; margin: 0; border-color: #009688; border-style: solid; border-width: 10px 20px;">激活账户</a></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您选择{siteTitle}。</td></tr></table></td></tr></table><div class="footer"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;"><table width="100%"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="aligncenter content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; color: #999; text-align: center; margin: 0; padding: 0 0 20px;"align="center"valign="top">此邮件由系统自动发送，请不要直接回复。</td></tr></table></div></div></td><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td></tr></table></body></html>`, Type: "mail_template"},
{Name: "forget_captcha", Value: `0`, Type: "login"},
{Name: "mail_reset_pwd_template", Value: `<!DOCTYPE html PUBLIC"-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box;
font-size: 14px; margin: 0;"><head><meta name="viewport"content="width=device-width"/><meta http-equiv="Content-Type"content="text/html; charset=UTF-8"/><title>重设密码</title><style type="text/css">img{max-width:100%}body{-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:none;width:100%!important;height:100%;line-height:1.6em}body{background-color:#f6f6f6}@media only screen and(max-width:640px){body{padding:0!important}h1{font-weight:800!important;margin:20px 0 5px!important}h2{font-weight:800!important;margin:20px 0 5px!important}h3{font-weight:800!important;margin:20px 0 5px!important}h4{font-weight:800!important;margin:20px 0 5px!important}h1{font-size:22px!important}h2{font-size:18px!important}h3{font-size:16px!important}.container{padding:0!important;width:100%!important}.content{padding:0!important}.content-wrap{padding:10px!important}.invoice{width:100%!important}}</style></head><body itemscope itemtype="http://schema.org/EmailMessage"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><table class="body-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; margin: 0;"><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td><td class="container"width="600"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; margin: 0 auto;"valign="top"><div class="content"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 20px;"><table class="main"width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px
solid #e9e9e9;"bgcolor="#fff"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
14px; margin: 0;"><td class="alert alert-warning"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #2196F3; margin: 0; padding: 20px;"align="center"bgcolor="#FF9F00"valign="top">重设{siteTitle}密码</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 20px;"valign="top"><table width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">亲爱的<strong style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{userName}</strong>：</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">请点击下方按钮完成密码重设。如果非你本人操作，请忽略此邮件。</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top"><a href="{resetUrl}"class="btn-primary"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color: #FFF; text-decoration: none; line-height: 2em; font-weight: bold; text-align: center; cursor: pointer; display: inline-block; border-radius: 5px; text-transform: capitalize; background-color: #2196F3; margin: 0; border-color: #2196F3; border-style: solid; border-width: 10px 20px;">重设密码</a></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您选择{siteTitle}。</td></tr></table></td></tr></table><div class="footer"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;"><table width="100%"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="aligncenter content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; color: #999; text-align: center; margin: 0; padding: 0 0 20px;"align="center"valign="top">此邮件由系统自动发送，请不要直接回复。</td></tr></table></div></div></td><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td></tr></table></body></html>`, Type: "mail_template"},
{Name: "db_version_" + conf.RequiredDBVersion, Value: `installed`, Type: "version"},
{Name: "hot_share_num", Value: `10`, Type: "share"},
{Name: "gravatar_server", Value: `https://www.gravatar.com/`, Type: "avatar"},
{Name: "defaultTheme", Value: `#3f51b5`, Type: "basic"},
{Name: "themes", Value: `{"#3f51b5":{"palette":{"primary":{"main":"#3f51b5"},"secondary":{"main":"#f50057"}}},"#2196f3":{"palette":{"primary":{"main":"#2196f3"},"secondary":{"main":"#FFC107"}}},"#673AB7":{"palette":{"primary":{"main":"#673AB7"},"secondary":{"main":"#2196F3"}}},"#E91E63":{"palette":{"primary":{"main":"#E91E63"},"secondary":{"main":"#42A5F5","contrastText":"#fff"}}},"#FF5722":{"palette":{"primary":{"main":"#FF5722"},"secondary":{"main":"#3F51B5"}}},"#FFC107":{"palette":{"primary":{"main":"#FFC107"},"secondary":{"main":"#26C6DA"}}},"#8BC34A":{"palette":{"primary":{"main":"#8BC34A","contrastText":"#fff"},"secondary":{"main":"#FF8A65","contrastText":"#fff"}}},"#009688":{"palette":{"primary":{"main":"#009688"},"secondary":{"main":"#4DD0E1","contrastText":"#fff"}}},"#607D8B":{"palette":{"primary":{"main":"#607D8B"},"secondary":{"main":"#F06292"}}},"#795548":{"palette":{"primary":{"main":"#795548"},"secondary":{"main":"#4CAF50","contrastText":"#fff"}}}}`, Type: "basic"},
{Name: "max_worker_num", Value: `10`, Type: "task"},
{Name: "max_parallel_transfer", Value: `4`, Type: "task"},
{Name: "secret_key", Value: util.RandStringRunes(256), Type: "auth"},
{Name: "temp_path", Value: "temp", Type: "path"},
{Name: "avatar_path", Value: "avatar", Type: "path"},
{Name: "avatar_size", Value: "2097152", Type: "avatar"},
{Name: "avatar_size_l", Value: "200", Type: "avatar"},
{Name: "avatar_size_m", Value: "130", Type: "avatar"},
{Name: "avatar_size_s", Value: "50", Type: "avatar"},
{Name: "home_view_method", Value: "icon", Type: "view"},
{Name: "share_view_method", Value: "list", Type: "view"},
{Name: "cron_garbage_collect", Value: "@hourly", Type: "cron"},
{Name: "cron_recycle_upload_session", Value: "@every 1h30m", Type: "cron"},
{Name: "authn_enabled", Value: "0", Type: "authn"},
{Name: "captcha_type", Value: "normal", Type: "captcha"},
{Name: "captcha_height", Value: "60", Type: "captcha"},
{Name: "captcha_width", Value: "240", Type: "captcha"},
{Name: "captcha_mode", Value: "3", Type: "captcha"},
{Name: "captcha_ComplexOfNoiseText", Value: "0", Type: "captcha"},
{Name: "captcha_ComplexOfNoiseDot", Value: "0", Type: "captcha"},
{Name: "captcha_IsShowHollowLine", Value: "0", Type: "captcha"},
{Name: "captcha_IsShowNoiseDot", Value: "1", Type: "captcha"},
{Name: "captcha_IsShowNoiseText", Value: "0", Type: "captcha"},
{Name: "captcha_IsShowSlimeLine", Value: "1", Type: "captcha"},
{Name: "captcha_IsShowSineLine", Value: "0", Type: "captcha"},
{Name: "captcha_CaptchaLen", Value: "6", Type: "captcha"},
{Name: "captcha_ReCaptchaKey", Value: "defaultKey", Type: "captcha"},
{Name: "captcha_ReCaptchaSecret", Value: "defaultSecret", Type: "captcha"},
{Name: "captcha_TCaptcha_CaptchaAppId", Value: "", Type: "captcha"},
{Name: "captcha_TCaptcha_AppSecretKey", Value: "", Type: "captcha"},
{Name: "captcha_TCaptcha_SecretId", Value: "", Type: "captcha"},
{Name: "captcha_TCaptcha_SecretKey", Value: "", Type: "captcha"},
{Name: "thumb_width", Value: "400", Type: "thumb"},
{Name: "thumb_height", Value: "300", Type: "thumb"},
{Name: "thumb_file_suffix", Value: "._thumb", Type: "thumb"},
{Name: "thumb_max_task_count", Value: "-1", Type: "thumb"},
{Name: "thumb_encode_method", Value: "jpg", Type: "thumb"},
{Name: "thumb_gc_after_gen", Value: "false", Type: "thumb"},
{Name: "thumb_encode_quality", Value: "85", Type: "thumb"},
{Name: "pwa_small_icon", Value: "/static/img/favicon.ico", Type: "pwa"},
{Name: "pwa_medium_icon", Value: "/static/img/logo192.png", Type: "pwa"},
{Name: "pwa_large_icon", Value: "/static/img/logo512.png", Type: "pwa"},
{Name: "pwa_display", Value: "standalone", Type: "pwa"},
{Name: "pwa_theme_color", Value: "#000000", Type: "pwa"},
{Name: "pwa_background_color", Value: "#ffffff", Type: "pwa"},
{Name: "office_preview_service", Value: "https://view.officeapps.live.com/op/view.aspx?src={$src}", Type: "preview"},
}

215
models/file_test.go
Unescape View File

@ -15,22 +15,62 @@ func TestFile_Create(t *testing.T) {
Name: "123",
}
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1))
mock.ExpectCommit()
fileID, err := file.Create()
asserts.NoError(err)
asserts.Equal(uint(5), fileID)
asserts.Equal(uint(5), file.ID)
asserts.NoError(mock.ExpectationsWereMet())
// 无法插入文件记录
{
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
err := file.Create()
asserts.Error(err)
asserts.NoError(mock.ExpectationsWereMet())
}
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
fileID, err = file.Create()
asserts.Error(err)
asserts.NoError(mock.ExpectationsWereMet())
// 无法更新用户容量
{
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1))
mock.ExpectExec("UPDATE(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
err := file.Create()
asserts.Error(err)
asserts.NoError(mock.ExpectationsWereMet())
}
// 成功
{
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
err := file.Create()
asserts.NoError(err)
asserts.Equal(uint(5), file.ID)
asserts.NoError(mock.ExpectationsWereMet())
}
}
func TestFile_AfterFind(t *testing.T) {
a := assert.New(t)
file := File{
Name: "123",
Metadata: "{\"name\":\"123\"}",
}
a.NoError(file.AfterFind())
a.Equal("123", file.MetadataSerialized["name"])
}
func TestFile_BeforeSave(t *testing.T) {
a := assert.New(t)
file := File{
Name: "123",
MetadataSerialized: map[string]string{
"name": "123",
},
}
a.NoError(file.BeforeSave())
a.Equal("{\"name\":\"123\"}", file.Metadata)
}
func TestFolder_GetChildFile(t *testing.T) {
@ -175,6 +215,17 @@ func TestGetChildFilesOfFolders(t *testing.T) {
}
}
func TestGetUploadPlaceholderFiles(t *testing.T) {
a := assert.New(t)
mock.ExpectQuery("SELECT(.+)upload_session_id(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "1"))
files := GetUploadPlaceholderFiles(1)
a.NoError(mock.ExpectationsWereMet())
a.Len(files, 1)
}
func TestFile_GetPolicy(t *testing.T) {
asserts := assert.New(t)
@ -282,28 +333,50 @@ func TestRemoveFilesWithSoftLinks(t *testing.T) {
}
}
func TestDeleteFileByIDs(t *testing.T) {
asserts := assert.New(t)
func TestDeleteFiles(t *testing.T) {
a := assert.New(t)
// 出错
// uid 不一致
{
err := DeleteFiles([]*File{{}}, 1)
a.Contains("User id not consistent", err.Error())
}
// 删除失败
{
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
WillReturnError(errors.New("error"))
mock.ExpectRollback()
err := DeleteFileByIDs([]uint{1, 2, 3})
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
err := DeleteFiles([]*File{{}}, 0)
a.NoError(mock.ExpectationsWereMet())
a.Error(err)
}
// 成功
// 无法变更用户容量
{
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(0, 3))
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
err := DeleteFiles([]*File{{}}, 0)
a.NoError(mock.ExpectationsWereMet())
a.Error(err)
}
// 成功，其中一个文件已经不存在
{
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(1, 0))
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(2, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := DeleteFileByIDs([]uint{1, 2, 3})
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
err := DeleteFiles([]*File{{Size: 1}, {Size: 2}}, 0)
a.NoError(mock.ExpectationsWereMet())
a.NoError(err)
}
}
@ -324,6 +397,19 @@ func TestGetFilesByParentIDs(t *testing.T) {
asserts.Len(files, 3)
}
func TestGetFilesByUploadSession(t *testing.T) {
a := assert.New(t)
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, "sessionID").
WillReturnRows(
sqlmock.NewRows([]string{"id", "name"}).AddRow(4, "4.txt"))
files, err := GetFilesByUploadSession("sessionID", 1)
a.NoError(err)
a.NoError(mock.ExpectationsWereMet())
a.Equal("4.txt", files.Name)
}
func TestFile_Updates(t *testing.T) {
asserts := assert.New(t)
file := File{Model: gorm.Model{ID: 1}}
@ -340,24 +426,93 @@ func TestFile_Updates(t *testing.T) {
// UpdatePicInfo
{
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WithArgs(10, sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)").WithArgs("1,1", 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := file.UpdateSize(10)
err := file.UpdatePicInfo("1,1")
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
// UpdatePicInfo
// UpdateSourceName
{
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WithArgs("1,1", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)").WithArgs("newName", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := file.UpdatePicInfo("1,1")
err := file.UpdateSourceName("newName")
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
}
func TestFile_UpdateSize(t *testing.T) {
a := assert.New(t)
// 增加成功
{
file := File{Size: 10}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(11, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)+(.+)").WithArgs(uint64(1), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
a.NoError(file.UpdateSize(11))
a.NoError(mock.ExpectationsWereMet())
}
// 减少成功
{
file := File{Size: 10}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)-(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
a.NoError(file.UpdateSize(8))
a.NoError(mock.ExpectationsWereMet())
}
// 文件更新失败
{
file := File{Size: 10}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnError(errors.New("error"))
mock.ExpectRollback()
a.Error(file.UpdateSize(8))
a.NoError(mock.ExpectationsWereMet())
}
// 用户容量更新失败
{
file := File{Size: 10}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)-(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnError(errors.New("error"))
mock.ExpectRollback()
a.Error(file.UpdateSize(8))
a.NoError(mock.ExpectationsWereMet())
}
}
func TestFile_PopChunkToFile(t *testing.T) {
a := assert.New(t)
timeNow := time.Now()
file := File{}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
a.NoError(file.PopChunkToFile(&timeNow, "1,1"))
}
func TestFile_CanCopy(t *testing.T) {
a := assert.New(t)
file := File{}
a.True(file.CanCopy())
file.UploadSessionID = &file.Name
a.False(file.CanCopy())
}
func TestFile_FileInfoInterface(t *testing.T) {
asserts := assert.New(t)
file := File{

19
models/folder_test.go
Unescape View File

@ -212,12 +212,14 @@ func TestFolder_MoveOrCopyFileTo(t *testing.T) {
WithArgs(
1,
2,
3,
1,
1,
).WillReturnRows(
sqlmock.NewRows([]string{"id", "size"}).
AddRow(1, 10).
AddRow(2, 20),
sqlmock.NewRows([]string{"id", "size", "upload_session_id"}).
AddRow(1, 10, nil).
AddRow(2, 20, nil).
AddRow(2, 20, &folder.Name),
)
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
@ -226,7 +228,7 @@ func TestFolder_MoveOrCopyFileTo(t *testing.T) {
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
storage, err := folder.MoveOrCopyFileTo(
[]uint{1, 2},
[]uint{1, 2, 3},
&dstFolder,
true,
)
@ -335,7 +337,7 @@ func TestFolder_CopyFolderTo(t *testing.T) {
// 测试复制目录结构
// test(2)(5)
// 1(3)(6) 2.txt
// 3(4)(7) 4.txt
// 3(4)(7) 4.txt 5.txt(上传中)
// 正常情况 成功
{
@ -360,9 +362,10 @@ func TestFolder_CopyFolderTo(t *testing.T) {
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 2, 3, 4).
WillReturnRows(
sqlmock.NewRows([]string{"id", "name", "folder_id", "size"}).
AddRow(1, "2.txt", 2, 10).
AddRow(2, "3.txt", 3, 20),
sqlmock.NewRows([]string{"id", "name", "folder_id", "size", "upload_session_id"}).
AddRow(1, "2.txt", 2, 10, nil).
AddRow(2, "3.txt", 3, 20, nil).
AddRow(3, "5.txt", 3, 20, &dstFolder.Name),
)
// 复制子文件

109
models/migration.go
Unescape View File

@ -7,7 +7,6 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/fatih/color"
"github.com/gofrs/uuid"
"github.com/hashicorp/go-version"
"github.com/jinzhu/gorm"
"sort"
@ -86,114 +85,6 @@ func addDefaultPolicy() {
}
func addDefaultSettings() {
siteID, _ := uuid.NewV4()
defaultSettings := []Setting{
{Name: "siteURL", Value: `http://localhost`, Type: "basic"},
{Name: "siteName", Value: `Cloudreve`, Type: "basic"},
{Name: "siteICPId", Value: ``, Type: "basic"},
{Name: "register_enabled", Value: `1`, Type: "register"},
{Name: "default_group", Value: `2`, Type: "register"},
{Name: "siteKeywords", Value: `网盘，网盘`, Type: "basic"},
{Name: "siteDes", Value: `Cloudreve`, Type: "basic"},
{Name: "siteTitle", Value: `平步云端`, Type: "basic"},
{Name: "siteScript", Value: ``, Type: "basic"},
{Name: "siteID", Value: siteID.String(), Type: "basic"},
{Name: "fromName", Value: `Cloudreve`, Type: "mail"},
{Name: "mail_keepalive", Value: `30`, Type: "mail"},
{Name: "fromAdress", Value: `no-reply@acg.blue`, Type: "mail"},
{Name: "smtpHost", Value: `smtp.mxhichina.com`, Type: "mail"},
{Name: "smtpPort", Value: `25`, Type: "mail"},
{Name: "replyTo", Value: `abslant@126.com`, Type: "mail"},
{Name: "smtpUser", Value: `no-reply@acg.blue`, Type: "mail"},
{Name: "smtpPass", Value: ``, Type: "mail"},
{Name: "smtpEncryption", Value: `0`, Type: "mail"},
{Name: "maxEditSize", Value: `4194304`, Type: "file_edit"},
{Name: "archive_timeout", Value: `60`, Type: "timeout"},
{Name: "download_timeout", Value: `60`, Type: "timeout"},
{Name: "preview_timeout", Value: `60`, Type: "timeout"},
{Name: "doc_preview_timeout", Value: `60`, Type: "timeout"},
{Name: "upload_credential_timeout", Value: `1800`, Type: "timeout"},
{Name: "upload_session_timeout", Value: `86400`, Type: "timeout"},
{Name: "slave_api_timeout", Value: `60`, Type: "timeout"},
{Name: "slave_node_retry", Value: `3`, Type: "slave"},
{Name: "slave_ping_interval", Value: `60`, Type: "slave"},
{Name: "slave_recover_interval", Value: `120`, Type: "slave"},
{Name: "slave_transfer_timeout", Value: `172800`, Type: "timeout"},
{Name: "onedrive_monitor_timeout", Value: `600`, Type: "timeout"},
{Name: "share_download_session_timeout", Value: `2073600`, Type: "timeout"},
{Name: "onedrive_callback_check", Value: `20`, Type: "timeout"},
{Name: "folder_props_timeout", Value: `300`, Type: "timeout"},
{Name: "onedrive_chunk_retries", Value: `1`, Type: "retry"},
{Name: "slave_chunk_retries", Value: `1`, Type: "retry"},
{Name: "onedrive_source_timeout", Value: `1800`, Type: "timeout"},
{Name: "reset_after_upload_failed", Value: `0`, Type: "upload"},
{Name: "login_captcha", Value: `0`, Type: "login"},
{Name: "reg_captcha", Value: `0`, Type: "login"},
{Name: "email_active", Value: `0`, Type: "register"},
{Name: "mail_activation_template", Value: `<!DOCTYPE html PUBLIC"-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box;
font-size: 14px; margin: 0;"><head><meta name="viewport"content="width=device-width"/><meta http-equiv="Content-Type"content="text/html; charset=UTF-8"/><title>激活您的账户</title><style type="text/css">img{max-width:100%}body{-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:none;width:100%!important;height:100%;line-height:1.6em}body{background-color:#f6f6f6}@media only screen and(max-width:640px){body{padding:0!important}h1{font-weight:800!important;margin:20px 0 5px!important}h2{font-weight:800!important;margin:20px 0 5px!important}h3{font-weight:800!important;margin:20px 0 5px!important}h4{font-weight:800!important;margin:20px 0 5px!important}h1{font-size:22px!important}h2{font-size:18px!important}h3{font-size:16px!important}.container{padding:0!important;width:100%!important}.content{padding:0!important}.content-wrap{padding:10px!important}.invoice{width:100%!important}}</style></head><body itemscope itemtype="http://schema.org/EmailMessage"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><table class="body-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; margin: 0;"><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td><td class="container"width="600"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; margin: 0 auto;"valign="top"><div class="content"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 20px;"><table class="main"width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px
solid #e9e9e9;"bgcolor="#fff"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
14px; margin: 0;"><td class="alert alert-warning"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #009688; margin: 0; padding: 20px;"align="center"bgcolor="#FF9F00"valign="top">激活{siteTitle}账户</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 20px;"valign="top"><table width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">亲爱的<strong style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{userName}</strong>：</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您注册{siteTitle},请点击下方按钮完成账户激活。</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top"><a href="{activationUrl}"class="btn-primary"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color: #FFF; text-decoration: none; line-height: 2em; font-weight: bold; text-align: center; cursor: pointer; display: inline-block; border-radius: 5px; text-transform: capitalize; background-color: #009688; margin: 0; border-color: #009688; border-style: solid; border-width: 10px 20px;">激活账户</a></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您选择{siteTitle}。</td></tr></table></td></tr></table><div class="footer"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;"><table width="100%"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="aligncenter content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; color: #999; text-align: center; margin: 0; padding: 0 0 20px;"align="center"valign="top">此邮件由系统自动发送，请不要直接回复。</td></tr></table></div></div></td><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td></tr></table></body></html>`, Type: "mail_template"},
{Name: "forget_captcha", Value: `0`, Type: "login"},
{Name: "mail_reset_pwd_template", Value: `<!DOCTYPE html PUBLIC"-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box;
font-size: 14px; margin: 0;"><head><meta name="viewport"content="width=device-width"/><meta http-equiv="Content-Type"content="text/html; charset=UTF-8"/><title>重设密码</title><style type="text/css">img{max-width:100%}body{-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:none;width:100%!important;height:100%;line-height:1.6em}body{background-color:#f6f6f6}@media only screen and(max-width:640px){body{padding:0!important}h1{font-weight:800!important;margin:20px 0 5px!important}h2{font-weight:800!important;margin:20px 0 5px!important}h3{font-weight:800!important;margin:20px 0 5px!important}h4{font-weight:800!important;margin:20px 0 5px!important}h1{font-size:22px!important}h2{font-size:18px!important}h3{font-size:16px!important}.container{padding:0!important;width:100%!important}.content{padding:0!important}.content-wrap{padding:10px!important}.invoice{width:100%!important}}</style></head><body itemscope itemtype="http://schema.org/EmailMessage"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><table class="body-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; margin: 0;"><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td><td class="container"width="600"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; margin: 0 auto;"valign="top"><div class="content"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 20px;"><table class="main"width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px
solid #e9e9e9;"bgcolor="#fff"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
14px; margin: 0;"><td class="alert alert-warning"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #2196F3; margin: 0; padding: 20px;"align="center"bgcolor="#FF9F00"valign="top">重设{siteTitle}密码</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 20px;"valign="top"><table width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">亲爱的<strong style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{userName}</strong>：</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">请点击下方按钮完成密码重设。如果非你本人操作，请忽略此邮件。</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top"><a href="{resetUrl}"class="btn-primary"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color: #FFF; text-decoration: none; line-height: 2em; font-weight: bold; text-align: center; cursor: pointer; display: inline-block; border-radius: 5px; text-transform: capitalize; background-color: #2196F3; margin: 0; border-color: #2196F3; border-style: solid; border-width: 10px 20px;">重设密码</a></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您选择{siteTitle}。</td></tr></table></td></tr></table><div class="footer"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;"><table width="100%"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="aligncenter content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; color: #999; text-align: center; margin: 0; padding: 0 0 20px;"align="center"valign="top">此邮件由系统自动发送，请不要直接回复。</td></tr></table></div></div></td><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td></tr></table></body></html>`, Type: "mail_template"},
{Name: "db_version_" + conf.RequiredDBVersion, Value: `installed`, Type: "version"},
{Name: "hot_share_num", Value: `10`, Type: "share"},
{Name: "gravatar_server", Value: `https://www.gravatar.com/`, Type: "avatar"},
{Name: "defaultTheme", Value: `#3f51b5`, Type: "basic"},
{Name: "themes", Value: `{"#3f51b5":{"palette":{"primary":{"main":"#3f51b5"},"secondary":{"main":"#f50057"}}},"#2196f3":{"palette":{"primary":{"main":"#2196f3"},"secondary":{"main":"#FFC107"}}},"#673AB7":{"palette":{"primary":{"main":"#673AB7"},"secondary":{"main":"#2196F3"}}},"#E91E63":{"palette":{"primary":{"main":"#E91E63"},"secondary":{"main":"#42A5F5","contrastText":"#fff"}}},"#FF5722":{"palette":{"primary":{"main":"#FF5722"},"secondary":{"main":"#3F51B5"}}},"#FFC107":{"palette":{"primary":{"main":"#FFC107"},"secondary":{"main":"#26C6DA"}}},"#8BC34A":{"palette":{"primary":{"main":"#8BC34A","contrastText":"#fff"},"secondary":{"main":"#FF8A65","contrastText":"#fff"}}},"#009688":{"palette":{"primary":{"main":"#009688"},"secondary":{"main":"#4DD0E1","contrastText":"#fff"}}},"#607D8B":{"palette":{"primary":{"main":"#607D8B"},"secondary":{"main":"#F06292"}}},"#795548":{"palette":{"primary":{"main":"#795548"},"secondary":{"main":"#4CAF50","contrastText":"#fff"}}}}`, Type: "basic"},
{Name: "max_worker_num", Value: `10`, Type: "task"},
{Name: "max_parallel_transfer", Value: `4`, Type: "task"},
{Name: "secret_key", Value: util.RandStringRunes(256), Type: "auth"},
{Name: "temp_path", Value: "temp", Type: "path"},
{Name: "avatar_path", Value: "avatar", Type: "path"},
{Name: "avatar_size", Value: "2097152", Type: "avatar"},
{Name: "avatar_size_l", Value: "200", Type: "avatar"},
{Name: "avatar_size_m", Value: "130", Type: "avatar"},
{Name: "avatar_size_s", Value: "50", Type: "avatar"},
{Name: "home_view_method", Value: "icon", Type: "view"},
{Name: "share_view_method", Value: "list", Type: "view"},
{Name: "cron_garbage_collect", Value: "@hourly", Type: "cron"},
{Name: "cron_recycle_upload_session", Value: "@every 1h30m", Type: "cron"},
{Name: "authn_enabled", Value: "0", Type: "authn"},
{Name: "captcha_type", Value: "normal", Type: "captcha"},
{Name: "captcha_height", Value: "60", Type: "captcha"},
{Name: "captcha_width", Value: "240", Type: "captcha"},
{Name: "captcha_mode", Value: "3", Type: "captcha"},
{Name: "captcha_ComplexOfNoiseText", Value: "0", Type: "captcha"},
{Name: "captcha_ComplexOfNoiseDot", Value: "0", Type: "captcha"},
{Name: "captcha_IsShowHollowLine", Value: "0", Type: "captcha"},
{Name: "captcha_IsShowNoiseDot", Value: "1", Type: "captcha"},
{Name: "captcha_IsShowNoiseText", Value: "0", Type: "captcha"},
{Name: "captcha_IsShowSlimeLine", Value: "1", Type: "captcha"},
{Name: "captcha_IsShowSineLine", Value: "0", Type: "captcha"},
{Name: "captcha_CaptchaLen", Value: "6", Type: "captcha"},
{Name: "captcha_ReCaptchaKey", Value: "defaultKey", Type: "captcha"},
{Name: "captcha_ReCaptchaSecret", Value: "defaultSecret", Type: "captcha"},
{Name: "captcha_TCaptcha_CaptchaAppId", Value: "", Type: "captcha"},
{Name: "captcha_TCaptcha_AppSecretKey", Value: "", Type: "captcha"},
{Name: "captcha_TCaptcha_SecretId", Value: "", Type: "captcha"},
{Name: "captcha_TCaptcha_SecretKey", Value: "", Type: "captcha"},
{Name: "thumb_width", Value: "400", Type: "thumb"},
{Name: "thumb_height", Value: "300", Type: "thumb"},
{Name: "pwa_small_icon", Value: "/static/img/favicon.ico", Type: "pwa"},
{Name: "pwa_medium_icon", Value: "/static/img/logo192.png", Type: "pwa"},
{Name: "pwa_large_icon", Value: "/static/img/logo512.png", Type: "pwa"},
{Name: "pwa_display", Value: "standalone", Type: "pwa"},
{Name: "pwa_theme_color", Value: "#000000", Type: "pwa"},
{Name: "pwa_background_color", Value: "#ffffff", Type: "pwa"},
{Name: "office_preview_service", Value: "https://view.officeapps.live.com/op/view.aspx?src={$src}", Type: "preview"},
}
for _, value := range defaultSettings {
DB.Where(Setting{Name: value.Name}).Create(&value)
}

76
models/policy.go
Unescape View File

@ -3,8 +3,7 @@ package model
import (
"encoding/gob"
"encoding/json"
"fmt"
"net/url"
"github.com/gofrs/uuid"
"path"
"path/filepath"
"strconv"
@ -60,6 +59,8 @@ type PolicyOption struct {
ServerSideEndpoint string `json:"server_side_endpoint,omitempty"`
// 分片上传的分片大小
ChunkSize uint64 `json:"chunk_size,omitempty"`
// 分片上传时是否需要预留空间
PlaceholderWithSize bool `json:"placeholder_with_size,omitempty"`
}
var thumbSuffix = map[string][]string{
@ -150,7 +151,7 @@ func (policy *Policy) GeneratePath(uid uint, origin string) string {
func (policy *Policy) GenerateFileName(uid uint, origin string) string {
// 未开启自动重命名时，直接返回原始文件名
if !policy.AutoRename {
return policy.getOriginNameRule(origin)
return origin
}
fileRule := policy.FileNameRule
@ -169,35 +170,15 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string {
"{hour}": time.Now().Format("15"),
"{minute}": time.Now().Format("04"),
"{second}": time.Now().Format("05"),
"{originname}": origin,
"{ext}": filepath.Ext(origin),
"{uuid}": uuid.Must(uuid.NewV4()).String(),
}
replaceTable["{originname}"] = policy.getOriginNameRule(origin)
fileRule = util.Replace(replaceTable, fileRule)
return fileRule
}
func (policy Policy) getOriginNameRule(origin string) string {
// 部分存储策略可以使用{origin}代表原始文件名
if origin == "" {
// 如果上游未传回原始文件名，则使用占位符，让云存储端替换
switch policy.Type {
case "qiniu":
// 七牛会将$(fname)自动替换为原始文件名
return "$(fname)"
case "local", "remote":
return origin
case "oss", "cos":
// OSS会将${filename}自动替换为原始文件名
return "${filename}"
case "upyun":
// Upyun会将{filename}{.suffix}自动替换为原始文件名
return "{filename}{.suffix}"
}
}
return origin
}
// IsDirectlyPreview 返回此策略下文件是否可以直接预览（不需要重定向）
func (policy *Policy) IsDirectlyPreview() bool {
return policy.Type == "local"
@ -226,7 +207,15 @@ func (policy *Policy) IsThumbGenerateNeeded() bool {
// IsUploadPlaceholderWithSize 返回此策略创建上传会话时是否需要预留空间
func (policy *Policy) IsUploadPlaceholderWithSize() bool {
return policy.Type == "remote"
if policy.Type == "remote" {
return true
}
if util.ContainsString([]string{"onedrive", "oss", "qiniu", "cos", "s3"}, policy.Type) {
return policy.OptionsSerialized.PlaceholderWithSize
}
return false
}
// CanStructureBeListed 返回存储策略是否能被前台列物理目录
@ -234,39 +223,6 @@ func (policy *Policy) CanStructureBeListed() bool {
return policy.Type != "local" && policy.Type != "remote"
}
// GetUploadURL 获取文件上传服务API地址
func (policy *Policy) GetUploadURL() string {
server, err := url.Parse(policy.Server)
if err != nil {
return policy.Server
}
controller, _ := url.Parse("")
switch policy.Type {
case "local", "onedrive":
return "/api/v3/file/upload"
case "remote":
controller, _ = url.Parse("/api/v3/slave/upload")
case "oss":
return "https://" + policy.BucketName + "." + policy.Server
case "cos":
return policy.Server
case "upyun":
return "https://v0.api.upyun.com/" + policy.BucketName
case "s3":
if policy.Server == "" {
return fmt.Sprintf("https://%s.s3.%s.amazonaws.com/", policy.BucketName,
policy.OptionsSerialized.Region)
}
if !strings.Contains(policy.Server, policy.BucketName) {
controller, _ = url.Parse("/" + policy.BucketName)
}
}
return server.ResolveReference(controller).String()
}
// SaveAndClearCache 更新并清理缓存
func (policy *Policy) SaveAndClearCache() error {
err := DB.Save(policy).Error

93
models/policy_test.go
Unescape View File

@ -104,7 +104,7 @@ func TestPolicy_GenerateFileName(t *testing.T) {
asserts.Equal("123.txt", testPolicy.GenerateFileName(1, "123.txt"))
testPolicy.Type = "oss"
asserts.Equal("${filename}", testPolicy.GenerateFileName(1, ""))
asserts.Equal("origin", testPolicy.GenerateFileName(1, "origin"))
}
// 重命名开启
@ -145,19 +145,23 @@ func TestPolicy_GenerateFileName(t *testing.T) {
testPolicy.Type = "oss"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123${filename}", testPolicy.GenerateFileName(1, ""))
asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321"))
testPolicy.Type = "upyun"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, ""))
asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321"))
testPolicy.Type = "qiniu"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123$(fname)", testPolicy.GenerateFileName(1, ""))
asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321"))
testPolicy.Type = "local"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123", testPolicy.GenerateFileName(1, ""))
testPolicy.Type = "local"
testPolicy.FileNameRule = "{ext}123{uuid}"
asserts.Contains(testPolicy.GenerateFileName(1, "123.txt"), ".txt123")
}
}
@ -170,78 +174,6 @@ func TestPolicy_IsDirectlyPreview(t *testing.T) {
asserts.False(policy.IsDirectlyPreview())
}
func TestPolicy_GetUploadURL(t *testing.T) {
asserts := assert.New(t)
// 本地
{
cache.Set("setting_siteURL", "http://127.0.0.1", 0)
policy := Policy{Type: "local", Server: "http://127.0.0.1"}
asserts.Equal("/api/v3/file/upload", policy.GetUploadURL())
}
// 远程
{
policy := Policy{Type: "remote", Server: "http://127.0.0.1"}
asserts.Equal("http://127.0.0.1/api/v3/slave/upload", policy.GetUploadURL())
}
// OSS
{
policy := Policy{Type: "oss", BucketName: "base", Server: "127.0.0.1"}
asserts.Equal("https://base.127.0.0.1", policy.GetUploadURL())
}
// cos
{
policy := Policy{Type: "cos", BaseURL: "base", Server: "http://127.0.0.1"}
asserts.Equal("http://127.0.0.1", policy.GetUploadURL())
}
// upyun
{
policy := Policy{Type: "upyun", BucketName: "base", Server: "http://127.0.0.1"}
asserts.Equal("https://v0.api.upyun.com/base", policy.GetUploadURL())
}
// 未知
{
policy := Policy{Type: "unknown", Server: "http://127.0.0.1"}
asserts.Equal("http://127.0.0.1", policy.GetUploadURL())
}
// S3 未填写自动生成
{
policy := Policy{
Type: "s3",
Server: "",
BucketName: "bucket",
OptionsSerialized: PolicyOption{Region: "us-east"},
}
asserts.Equal("https://bucket.s3.us-east.amazonaws.com/", policy.GetUploadURL())
}
// s3 自己指定
{
policy := Policy{
Type: "s3",
Server: "https://s3.us-east.amazonaws.com/",
BucketName: "bucket",
OptionsSerialized: PolicyOption{Region: "us-east"},
}
asserts.Equal("https://s3.us-east.amazonaws.com/bucket", policy.GetUploadURL())
}
}
func TestPolicy_IsPathGenerateNeeded(t *testing.T) {
asserts := assert.New(t)
policy := Policy{Type: "qiniu"}
asserts.True(policy.IsPathGenerateNeeded())
policy.Type = "remote"
asserts.False(policy.IsPathGenerateNeeded())
}
func TestPolicy_ClearCache(t *testing.T) {
asserts := assert.New(t)
cache.Set("policy_202", 1, 0)
@ -266,15 +198,18 @@ func TestPolicy_UpdateAccessKey(t *testing.T) {
func TestPolicy_Props(t *testing.T) {
asserts := assert.New(t)
policy := Policy{Type: "onedrive"}
policy.OptionsSerialized.PlaceholderWithSize = true
asserts.False(policy.IsThumbGenerateNeeded())
asserts.True(policy.IsPathGenerateNeeded())
asserts.True(policy.IsTransitUpload(4))
asserts.False(policy.IsTransitUpload(4))
asserts.False(policy.IsTransitUpload(5 * 1024 * 1024))
asserts.True(policy.CanStructureBeListed())
asserts.True(policy.IsUploadPlaceholderWithSize())
policy.Type = "local"
asserts.True(policy.IsThumbGenerateNeeded())
asserts.True(policy.IsPathGenerateNeeded())
asserts.False(policy.CanStructureBeListed())
asserts.False(policy.IsUploadPlaceholderWithSize())
policy.Type = "remote"
asserts.True(policy.IsUploadPlaceholderWithSize())
}
func TestPolicy_IsThumbExist(t *testing.T) {

9
models/setting.go
Unescape View File

@ -43,6 +43,15 @@ func GetSettingByName(name string) string {
return ""
}
// GetSettingByNameWithDefault 用 Name 获取设置值, 取不到时使用缺省值
func GetSettingByNameWithDefault(name, fallback string) string {
res := GetSettingByName(name)
if res == "" {
return fallback
}
return res
}
// GetSettingByNames 用多个 Name 获取设置值
func GetSettingByNames(names ...string) map[string]string {
var queryRes []Setting

9
models/setting_test.go
Unescape View File

@ -59,6 +59,15 @@ func TestGetSettingByType(t *testing.T) {
asserts.Equal(map[string]string{}, settings)
}
func TestGetSettingByNameWithDefault(t *testing.T) {
a := assert.New(t)
rows := sqlmock.NewRows([]string{"name", "value", "type"})
mock.ExpectQuery("^SELECT \\* FROM `(.+)` WHERE `(.+)`\\.`deleted_at` IS NULL AND(.+)$").WillReturnRows(rows)
settings := GetSettingByNameWithDefault("123", "123321")
a.Equal("123321", settings)
}
func TestGetSettingByNames(t *testing.T) {
cache.Store = cache.NewMemoStore()
asserts := assert.New(t)

4
models/tag_test.go
Unescape View File

@ -56,8 +56,8 @@ func TestGetTagsByUID(t *testing.T) {
func TestGetTagsByID(t *testing.T) {
asserts := assert.New(t)
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"name"}).AddRow("tag"))
res, err := GetTagsByUID(1)
res, err := GetTagsByID(1, 1)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
asserts.EqualValues("tag", res[0].Name)
asserts.EqualValues("tag", res.Name)
}

12
pkg/cache/driver.go vendored
Unescape View File

@ -2,6 +2,7 @@ package cache
import (
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin"
)
@ -9,9 +10,7 @@ import (
var Store Driver = NewMemoStore()
// Init 初始化缓存
func Init() {
//Store = NewRedisStore(10, "tcp", "127.0.0.1:6379", "", "0")
//return
func Init(isSlave bool) {
if conf.RedisConfig.Server != "" && gin.Mode() != gin.TestMode {
Store = NewRedisStore(
10,
@ -21,6 +20,13 @@ func Init() {
conf.RedisConfig.DB,
)
}
if isSlave {
err := Store.Sets(conf.OptionOverwrite, "setting_")
if err != nil {
util.Log().Warning("无法覆盖数据库设置: %s", err)
}
}
}
// Driver 键值缓存存储容器

6
pkg/cache/driver_test.go vendored
Unescape View File

@ -56,6 +56,10 @@ func TestInit(t *testing.T) {
asserts := assert.New(t)
asserts.NotPanics(func() {
Init()
Init(false)
})
asserts.NotPanics(func() {
Init(true)
})
}

1
pkg/cluster/slave.go
Unescape View File

@ -413,7 +413,6 @@ func getAria2RequestBody(body *serializer.SlaveAria2Call) (io.Reader, error) {
return strings.NewReader(string(reqBodyEncoded)), nil
}
// TODO: move to slave pkg
// RemoteCallback 发送远程存储策略上传回调请求
func RemoteCallback(url string, body serializer.UploadCallback) error {
callbackBody, err := json.Marshal(struct {

236
pkg/cluster/slave_test.go
Unescape View File

@ -1,8 +1,12 @@
package cluster
import (
"bytes"
"encoding/json"
"errors"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/stretchr/testify/assert"
@ -442,124 +446,114 @@ func TestSlaveCaller_DeleteTempFile(t *testing.T) {
}
}
//func TestRemoteCallback(t *testing.T) {
// asserts := assert.New(t)
//
// // 回调成功
// {
// clientMock := request.ClientMock{}
// mockResp, _ := json.Marshal(serializer.Response{Code: 0})
// clientMock.On(
// "Request",
// "POST",
// "http://test/test/url",
// testMock.Anything,
// testMock.Anything,
// ).Return(&request.Response{
// Err: nil,
// Response: &http.Response{
// StatusCode: 200,
// Body: ioutil.NopCloser(bytes.NewReader(mockResp)),
// },
// })
// request.GeneralClient = clientMock
// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
// SourceName: "source",
// })
// asserts.NoError(resp)
// clientMock.AssertExpectations(t)
// }
//
// // 服务端返回业务错误
// {
// clientMock := request.ClientMock{}
// mockResp, _ := json.Marshal(serializer.Response{Code: 401})
// clientMock.On(
// "Request",
// "POST",
// "http://test/test/url",
// testMock.Anything,
// testMock.Anything,
// ).Return(&request.Response{
// Err: nil,
// Response: &http.Response{
// StatusCode: 200,
// Body: ioutil.NopCloser(bytes.NewReader(mockResp)),
// },
// })
// request.GeneralClient = clientMock
// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
// SourceName: "source",
// })
// asserts.EqualValues(401, resp.(serializer.AppError).Code)
// clientMock.AssertExpectations(t)
// }
//
// // 无法解析回调响应
// {
// clientMock := request.ClientMock{}
// clientMock.On(
// "Request",
// "POST",
// "http://test/test/url",
// testMock.Anything,
// testMock.Anything,
// ).Return(&request.Response{
// Err: nil,
// Response: &http.Response{
// StatusCode: 200,
// Body: ioutil.NopCloser(strings.NewReader("mockResp")),
// },
// })
// request.GeneralClient = clientMock
// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
// SourceName: "source",
// })
// asserts.Error(resp)
// clientMock.AssertExpectations(t)
// }
//
// // HTTP状态码非200
// {
// clientMock := request.ClientMock{}
// clientMock.On(
// "Request",
// "POST",
// "http://test/test/url",
// testMock.Anything,
// testMock.Anything,
// ).Return(&request.Response{
// Err: nil,
// Response: &http.Response{
// StatusCode: 404,
// Body: ioutil.NopCloser(strings.NewReader("mockResp")),
// },
// })
// request.GeneralClient = clientMock
// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
// SourceName: "source",
// })
// asserts.Error(resp)
// clientMock.AssertExpectations(t)
// }
//
// // 无法发起回调
// {
// clientMock := request.ClientMock{}
// clientMock.On(
// "Request",
// "POST",
// "http://test/test/url",
// testMock.Anything,
// testMock.Anything,
// ).Return(&request.Response{
// Err: errors.New("error"),
// })
// request.GeneralClient = clientMock
// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
// SourceName: "source",
// })
// asserts.Error(resp)
// clientMock.AssertExpectations(t)
// }
//}
func TestRemoteCallback(t *testing.T) {
asserts := assert.New(t)
// 回调成功
{
clientMock := requestmock.RequestMock{}
mockResp, _ := json.Marshal(serializer.Response{Code: 0})
clientMock.On(
"Request",
"POST",
"http://test/test/url",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewReader(mockResp)),
},
})
request.GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
asserts.NoError(resp)
clientMock.AssertExpectations(t)
}
// 服务端返回业务错误
{
clientMock := requestmock.RequestMock{}
mockResp, _ := json.Marshal(serializer.Response{Code: 401})
clientMock.On(
"Request",
"POST",
"http://test/test/url",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewReader(mockResp)),
},
})
request.GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
asserts.EqualValues(401, resp.(serializer.AppError).Code)
clientMock.AssertExpectations(t)
}
// 无法解析回调响应
{
clientMock := requestmock.RequestMock{}
clientMock.On(
"Request",
"POST",
"http://test/test/url",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader("mockResp")),
},
})
request.GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
asserts.Error(resp)
clientMock.AssertExpectations(t)
}
// HTTP状态码非200
{
clientMock := requestmock.RequestMock{}
clientMock.On(
"Request",
"POST",
"http://test/test/url",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 404,
Body: ioutil.NopCloser(strings.NewReader("mockResp")),
},
})
request.GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
asserts.Error(resp)
clientMock.AssertExpectations(t)
}
// 无法发起回调
{
clientMock := requestmock.RequestMock{}
clientMock.On(
"Request",
"POST",
"http://test/test/url",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: errors.New("error"),
})
request.GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
asserts.Error(resp)
clientMock.AssertExpectations(t)
}
}

34
pkg/conf/conf.go
Unescape View File

@ -45,21 +45,6 @@ type slave struct {
SignatureTTL int `validate:"omitempty,gte=1"`
}
// captcha 验证码配置
type captcha struct {
Height int `validate:"gte=0"`
Width int `validate:"gte=0"`
Mode int `validate:"gte=0,lte=3"`
ComplexOfNoiseText int `validate:"gte=0,lte=2"`
ComplexOfNoiseDot int `validate:"gte=0,lte=2"`
IsShowHollowLine bool
IsShowNoiseDot bool
IsShowNoiseText bool
IsShowSlimeLine bool
IsShowSineLine bool
CaptchaLen int `validate:"gt=0"`
}
// redis 配置
type redis struct {
Network string
@ -68,17 +53,6 @@ type redis struct {
DB string
}
// 缩略图 配置
type thumb struct {
MaxWidth uint
MaxHeight uint
FileSuffix string `validate:"min=1"`
MaxTaskCount int
EncodeMethod string `validate:"eq=jpg|eq=png"`
EncodeQuality int `validate:"gte=1,lte=100"`
GCAfterGen bool
}
// 跨域配置
type cors struct {
AllowOrigins []string
@ -91,6 +65,7 @@ type cors struct {
var cfg *ini.File
const defaultConf = `[System]
Debug = false
Mode = master
Listen = :5212
SessionSecret = {SessionSecret}
@ -131,9 +106,7 @@ func Init(path string) {
"System": SystemConfig,
"SSL": SSLConfig,
"UnixSocket": UnixConfig,
"Captcha": CaptchaConfig,
"Redis": RedisConfig,
"Thumbnail": ThumbConfig,
"CORS": CORSConfig,
"Slave": SlaveConfig,
}
@ -144,6 +117,11 @@ func Init(path string) {
}
}
// 映射数据库配置覆盖
for _, key := range cfg.Section("OptionOverwrite").Keys() {
OptionOverwrite[key.Name()] = key.Value()
}
// 重设log等级
if !SystemConfig.Debug {
util.Level = util.LevelInformational

7
pkg/conf/conf_test.go
Unescape View File

@ -56,7 +56,11 @@ User = root
Password = root
Host = 127.0.0.1:3306
Name = v3
TablePrefix = v3_`
TablePrefix = v3_
[OptionOverwrite]
key=value
`
err := ioutil.WriteFile("testConf.ini", []byte(testCase), 0644)
defer func() { err = os.Remove("testConf.ini") }()
if err != nil {
@ -65,6 +69,7 @@ TablePrefix = v3_`
asserts.NotPanics(func() {
Init("testConf.ini")
})
asserts.Equal(OptionOverwrite["key"], "value")
}
func TestMapSection(t *testing.T) {

30
pkg/conf/defaults.go
Unescape View File

@ -1,7 +1,5 @@
package conf
import "github.com/mojocn/base64Captcha"
// RedisConfig Redis服务器配置
var RedisConfig = &redis{
Network: "tcp",
@ -25,21 +23,6 @@ var SystemConfig = &system{
Listen: ":5212",
}
// CaptchaConfig 验证码配置
var CaptchaConfig = &captcha{
Height: 60,
Width: 240,
Mode: 3,
ComplexOfNoiseText: base64Captcha.CaptchaComplexLower,
ComplexOfNoiseDot: base64Captcha.CaptchaComplexLower,
IsShowHollowLine: false,
IsShowNoiseDot: false,
IsShowNoiseText: false,
IsShowSlimeLine: false,
IsShowSineLine: false,
CaptchaLen: 6,
}
// CORSConfig 跨域配置
var CORSConfig = &cors{
AllowOrigins: []string{"UNSET"},
@ -49,17 +32,6 @@ var CORSConfig = &cors{
ExposeHeaders: nil,
}
// ThumbConfig 缩略图配置
var ThumbConfig = &thumb{
MaxWidth: 400,
MaxHeight: 300,
FileSuffix: "._thumb",
MaxTaskCount: -1,
EncodeMethod: "jpg",
GCAfterGen: false,
EncodeQuality: 85,
}
// SlaveConfig 从机配置
var SlaveConfig = &slave{
CallbackTimeout: 20,
@ -75,3 +47,5 @@ var SSLConfig = &ssl{
var UnixConfig = &unix{
Listen: "",
}
var OptionOverwrite = map[string]interface{}{}

6
pkg/filesystem/archive_test.go
Unescape View File

@ -3,6 +3,9 @@ package filesystem
import (
"context"
"errors"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
testMock "github.com/stretchr/testify/mock"
"io"
"os"
"strings"
@ -12,11 +15,8 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
)
func TestFileSystem_Compress(t *testing.T) {

22
pkg/filesystem/chunk/backoff/backoff_test.go
Unescape View File

@ -0,0 +1,22 @@
package backoff
import (
"github.com/stretchr/testify/assert"
"testing"
"time"
)
func TestConstantBackoff_Next(t *testing.T) {
a := assert.New(t)
b := &ConstantBackoff{Sleep: time.Duration(0), Max: 3}
a.True(b.Next())
a.True(b.Next())
a.True(b.Next())
a.False(b.Next())
b.Reset()
a.True(b.Next())
a.True(b.Next())
a.True(b.Next())
a.False(b.Next())
}

103
pkg/filesystem/chunk/chunk.go
Unescape View File

@ -7,50 +7,100 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"io"
"os"
)
const bufferTempPattern = "cdChunk.*.tmp"
// ChunkProcessFunc callback function for processing a chunk
type ChunkProcessFunc func(c *ChunkGroup, chunk io.Reader) error
// ChunkGroup manage groups of chunks
type ChunkGroup struct {
file fsctx.FileHeader
chunkSize uint64
backoff backoff.Backoff
file fsctx.FileHeader
chunkSize uint64
backoff backoff.Backoff
enableRetryBuffer bool
fileInfo *fsctx.UploadTaskInfo
currentIndex int
chunkNum uint64
bufferTemp *os.File
}
func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Backoff) *ChunkGroup {
func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Backoff, useBuffer bool) *ChunkGroup {
c := &ChunkGroup{
file: file,
chunkSize: chunkSize,
backoff: backoff,
fileInfo: file.Info(),
currentIndex: -1,
file: file,
chunkSize: chunkSize,
backoff: backoff,
fileInfo: file.Info(),
currentIndex: -1,
enableRetryBuffer: useBuffer,
}
if c.chunkSize == 0 {
c.chunkSize = c.fileInfo.Size
}
c.chunkNum = c.fileInfo.Size / c.chunkSize
if c.fileInfo.Size%c.chunkSize != 0 || c.fileInfo.Size == 0 {
c.chunkNum++
if c.fileInfo.Size == 0 {
c.chunkNum = 1
} else {
c.chunkNum = c.fileInfo.Size / c.chunkSize
if c.fileInfo.Size%c.chunkSize != 0 {
c.chunkNum++
}
}
return c
}
// TempAvailable returns if current chunk temp file is available to be read
func (c *ChunkGroup) TempAvailable() bool {
if c.bufferTemp != nil {
state, _ := c.bufferTemp.Stat()
return state != nil && state.Size() == c.Length()
}
return false
}
// Process a chunk with retry logic
func (c *ChunkGroup) Process(processor ChunkProcessFunc) error {
err := processor(c, io.LimitReader(c.file, int64(c.chunkSize)))
reader := io.LimitReader(c.file, int64(c.chunkSize))
// If useBuffer is enabled, tee the reader to a temp file
if c.enableRetryBuffer && c.bufferTemp == nil && !c.file.Seekable() {
c.bufferTemp, _ = os.CreateTemp("", bufferTempPattern)
reader = io.TeeReader(reader, c.bufferTemp)
}
if c.bufferTemp != nil {
defer func() {
if c.bufferTemp != nil {
c.bufferTemp.Close()
os.Remove(c.bufferTemp.Name())
c.bufferTemp = nil
}
}()
// if temp buffer file is available, use it
if c.TempAvailable() {
if _, err := c.bufferTemp.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek temp file back to chunk start: %w", err)
}
util.Log().Debug("Chunk %d will be read from temp file %q.", c.Index(), c.bufferTemp.Name())
reader = c.bufferTemp
}
}
err := processor(c, reader)
if err != nil {
if err != context.Canceled && c.file.Seekable() && c.backoff.Next() {
if _, seekErr := c.file.Seek(c.Start(), io.SeekStart); seekErr != nil {
return fmt.Errorf("failed to seek back to chunk start: %w, last error: %w", seekErr, err)
if err != context.Canceled && (c.file.Seekable() || c.TempAvailable()) && c.backoff.Next() {
if c.file.Seekable() {
if _, seekErr := c.file.Seek(c.Start(), io.SeekStart); seekErr != nil {
return fmt.Errorf("failed to seek back to chunk start: %w, last error: %s", seekErr, err)
}
}
util.Log().Debug("Retrying chunk %d, last error: %s", c.currentIndex, err)
@ -60,6 +110,7 @@ func (c *ChunkGroup) Process(processor ChunkProcessFunc) error {
return err
}
util.Log().Debug("Chunk %d processed", c.currentIndex)
return nil
}
@ -68,6 +119,21 @@ func (c *ChunkGroup) Start() int64 {
return int64(uint64(c.Index()) * c.chunkSize)
}
// Total returns the total length
func (c *ChunkGroup) Total() int64 {
return int64(c.fileInfo.Size)
}
// Num returns the total chunk number
func (c *ChunkGroup) Num() int {
return int(c.chunkNum)
}
// RangeHeader returns header value of Content-Range
func (c *ChunkGroup) RangeHeader() string {
return fmt.Sprintf("bytes %d-%d/%d", c.Start(), c.Start()+c.Length()-1, c.Total())
}
// Index returns current chunk index, starts from 0
func (c *ChunkGroup) Index() int {
return c.currentIndex
@ -89,3 +155,8 @@ func (c *ChunkGroup) Length() int64 {
return int64(contentLength)
}
// IsLast returns if current chunk is the last one
func (c *ChunkGroup) IsLast() bool {
return c.Index() == int(c.chunkNum-1)
}

250
pkg/filesystem/chunk/chunk_test.go
Unescape View File

@ -0,0 +1,250 @@
package chunk
import (
"errors"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/stretchr/testify/assert"
"io"
"os"
"strings"
"testing"
)
func TestNewChunkGroup(t *testing.T) {
a := assert.New(t)
testCases := []struct {
fileSize uint64
chunkSize uint64
expectedInnerChunkSize uint64
expectedChunkNum uint64
expectedInfo [][2]int //Start, Index,Length
}{
{10, 0, 10, 1, [][2]int{{0, 10}}},
{0, 0, 0, 1, [][2]int{{0, 0}}},
{0, 10, 10, 1, [][2]int{{0, 0}}},
{50, 10, 10, 5, [][2]int{
{0, 10},
{10, 10},
{20, 10},
{30, 10},
{40, 10},
}},
{50, 50, 50, 1, [][2]int{
{0, 50},
}},
{50, 15, 15, 4, [][2]int{
{0, 15},
{15, 15},
{30, 15},
{45, 5},
}},
}
for index, testCase := range testCases {
file := &fsctx.FileStream{Size: testCase.fileSize}
chunkGroup := NewChunkGroup(file, testCase.chunkSize, &backoff.ConstantBackoff{}, true)
a.EqualValues(testCase.expectedChunkNum, chunkGroup.Num(),
"TestCase:%d,ChunkNum()", index)
a.EqualValues(testCase.expectedInnerChunkSize, chunkGroup.chunkSize,
"TestCase:%d,InnerChunkSize()", index)
a.EqualValues(testCase.expectedChunkNum, chunkGroup.Num(),
"TestCase:%d,len(Chunks)", index)
a.EqualValues(testCase.fileSize, chunkGroup.Total())
for cIndex, info := range testCase.expectedInfo {
a.True(chunkGroup.Next())
a.EqualValues(info[1], chunkGroup.Length(),
"TestCase:%d,Chunks[%d].Length()", index, cIndex)
a.EqualValues(info[0], chunkGroup.Start(),
"TestCase:%d,Chunks[%d].Start()", index, cIndex)
a.Equal(cIndex == len(testCase.expectedInfo)-1, chunkGroup.IsLast(),
"TestCase:%d,Chunks[%d].IsLast()", index, cIndex)
a.NotEmpty(chunkGroup.RangeHeader())
}
a.False(chunkGroup.Next())
}
}
func TestChunkGroup_TempAvailablet(t *testing.T) {
a := assert.New(t)
file := &fsctx.FileStream{Size: 1}
c := NewChunkGroup(file, 0, &backoff.ConstantBackoff{}, true)
a.False(c.TempAvailable())
f, err := os.CreateTemp("", "TestChunkGroup_TempAvailablet.*")
defer func() {
f.Close()
os.Remove(f.Name())
}()
a.NoError(err)
c.bufferTemp = f
a.False(c.TempAvailable())
f.Write([]byte("1"))
a.True(c.TempAvailable())
}
func TestChunkGroup_Process(t *testing.T) {
a := assert.New(t)
file := &fsctx.FileStream{Size: 10}
// success
{
file.File = io.NopCloser(strings.NewReader("1234567890"))
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{}, true)
count := 0
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("12345", string(res))
return nil
}))
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("67890", string(res))
return nil
}))
a.False(c.Next())
a.Equal(2, count)
}
// retry, read from buffer file
{
file.File = io.NopCloser(strings.NewReader("1234567890"))
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, true)
count := 0
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("12345", string(res))
return nil
}))
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("67890", string(res))
if count == 2 {
return errors.New("error")
}
return nil
}))
a.False(c.Next())
a.Equal(3, count)
}
// retry, read from seeker
{
f, _ := os.CreateTemp("", "TestChunkGroup_Process.*")
f.Write([]byte("1234567890"))
f.Seek(0, 0)
defer func() {
f.Close()
os.Remove(f.Name())
}()
file.File = f
file.Seeker = f
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false)
count := 0
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("12345", string(res))
return nil
}))
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("67890", string(res))
if count == 2 {
return errors.New("error")
}
return nil
}))
a.False(c.Next())
a.Equal(3, count)
}
// retry, seek error
{
f, _ := os.CreateTemp("", "TestChunkGroup_Process.*")
f.Write([]byte("1234567890"))
f.Seek(0, 0)
defer func() {
f.Close()
os.Remove(f.Name())
}()
file.File = f
file.Seeker = f
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false)
count := 0
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("12345", string(res))
return nil
}))
a.True(c.Next())
f.Close()
a.Error(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
if count == 2 {
return errors.New("error")
}
return nil
}))
a.False(c.Next())
a.Equal(2, count)
}
// retry, finally error
{
f, _ := os.CreateTemp("", "TestChunkGroup_Process.*")
f.Write([]byte("1234567890"))
f.Seek(0, 0)
defer func() {
f.Close()
os.Remove(f.Name())
}()
file.File = f
file.Seeker = f
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false)
count := 0
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("12345", string(res))
return nil
}))
a.True(c.Next())
a.Error(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
return errors.New("error")
}))
a.False(c.Next())
a.Equal(4, count)
}
}

13
pkg/filesystem/driver/cos/handler.go
Unescape View File

@ -357,8 +357,9 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
res, err := handler.getUploadCredential(ctx, postPolicy, keyTime, savePath)
if err == nil {
res.SessionID = uploadSession.Key
res.Callback = apiURL
res.Key = uploadSession.Key
res.UploadURLs = []string{handler.Policy.Server}
}
return res, err
@ -415,10 +416,10 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli
signature := hmacFinalSign.Sum(nil)
return &serializer.UploadCredential{
Policy: policyEncoded,
Path: savePath,
AccessKey: handler.Policy.AccessKey,
Token: fmt.Sprintf("%x", signature),
KeyTime: keyTime,
Policy: policyEncoded,
Path: savePath,
AccessKey: handler.Policy.AccessKey,
Credential: fmt.Sprintf("%x", signature),
KeyTime: keyTime,
}, nil
}

7
pkg/filesystem/driver/local/handler.go
Unescape View File

@ -12,7 +12,6 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
@ -162,7 +161,7 @@ func (handler Driver) Truncate(ctx context.Context, src string, size uint64) err
util.Log().Warning("截断文件 [%s] 至 [%d]", src, size)
out, err := os.OpenFile(src, os.O_WRONLY, Perm)
if err != nil {
util.Log().Warning("无法打开或创建文件，%s", err)
util.Log().Warning("无法打开文件，%s", err)
return err
}
@ -188,7 +187,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// 尝试删除文件的缩略图（如果有）
_ = os.Remove(util.RelativePath(value + conf.ThumbConfig.FileSuffix))
_ = os.Remove(util.RelativePath(value + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")))
}
return deleteFailed, retErr
@ -196,7 +195,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
file, err := handler.Get(ctx, path+conf.ThumbConfig.FileSuffix)
file, err := handler.Get(ctx, path+model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"))
if err != nil {
return nil, err
}

123
pkg/filesystem/driver/local/handler_test.go
Unescape View File

@ -4,13 +4,12 @@ import (
"context"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
"io"
"io/ioutil"
"net/url"
"os"
"strings"
@ -20,52 +19,74 @@ import (
func TestHandler_Put(t *testing.T) {
asserts := assert.New(t)
handler := Driver{}
ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
os.Remove(util.RelativePath("test/test/txt"))
defer func() {
os.Remove(util.RelativePath("TestHandler_Put.txt"))
os.Remove(util.RelativePath("inner/TestHandler_Put.txt"))
}()
testCases := []struct {
file io.ReadCloser
dst string
err bool
file fsctx.FileHeader
errContains string
}{
{
file: ioutil.NopCloser(strings.NewReader("test input file")),
dst: "test/test/txt",
err: false,
},
{
file: ioutil.NopCloser(strings.NewReader("test input file")),
dst: "test/test/txt",
err: true,
},
{
file: ioutil.NopCloser(strings.NewReader("test input file")),
dst: "/notexist:/S.TXT",
err: true,
},
{&fsctx.FileStream{
SavePath: "TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("")),
}, ""},
{&fsctx.FileStream{
SavePath: "TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("")),
}, "物理同名文件已存在或不可用"},
{&fsctx.FileStream{
SavePath: "inner/TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("")),
}, ""},
{&fsctx.FileStream{
Mode: fsctx.Append | fsctx.Overwrite,
SavePath: "inner/TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("123")),
}, ""},
{&fsctx.FileStream{
AppendStart: 10,
Mode: fsctx.Append | fsctx.Overwrite,
SavePath: "inner/TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("123")),
}, "未上传完成的文件分片与预期大小不一致"},
{&fsctx.FileStream{
Mode: fsctx.Append | fsctx.Overwrite,
SavePath: "inner/TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("123")),
}, ""},
}
for _, testCase := range testCases {
err := handler.Put(ctx, testCase.file, testCase.dst, 15)
if testCase.err {
err := handler.Put(context.Background(), testCase.file)
if testCase.errContains != "" {
asserts.Error(err)
asserts.Contains(err.Error(), testCase.errContains)
} else {
asserts.NoError(err)
asserts.True(util.Exists(util.RelativePath(testCase.dst)))
asserts.True(util.Exists(util.RelativePath(testCase.file.Info().SavePath)))
}
}
}
func TestDriver_TruncateFailed(t *testing.T) {
a := assert.New(t)
h := Driver{}
a.Error(h.Truncate(context.Background(), "TestDriver_TruncateFailed", 0))
}
func TestHandler_Delete(t *testing.T) {
asserts := assert.New(t)
handler := Driver{}
ctx := context.Background()
filePath := util.RelativePath("test.file")
filePath := util.RelativePath("TestHandler_Delete.file")
file, err := os.Create(filePath)
asserts.NoError(err)
_ = file.Close()
list, err := handler.Delete(ctx, []string{"test.file"})
list, err := handler.Delete(ctx, []string{"TestHandler_Delete.file"})
asserts.Equal([]string{}, list)
asserts.NoError(err)
@ -73,7 +94,7 @@ func TestHandler_Delete(t *testing.T) {
_ = file.Close()
file, _ = os.OpenFile(filePath, os.O_RDWR, os.FileMode(0))
asserts.NoError(err)
list, err = handler.Delete(ctx, []string{"test.file", "test.notexist"})
list, err = handler.Delete(ctx, []string{"TestHandler_Delete.file", "test.notexist"})
file.Close()
asserts.Equal([]string{}, list)
asserts.NoError(err)
@ -84,7 +105,7 @@ func TestHandler_Delete(t *testing.T) {
file, err = os.Create(filePath)
asserts.NoError(err)
list, err = handler.Delete(ctx, []string{"test.file"})
list, err = handler.Delete(ctx, []string{"TestHandler_Delete.file"})
_ = file.Close()
asserts.Equal([]string{}, list)
asserts.NoError(err)
@ -116,7 +137,7 @@ func TestHandler_Thumb(t *testing.T) {
asserts := assert.New(t)
handler := Driver{}
ctx := context.Background()
file, err := os.Create(util.RelativePath("TestHandler_Thumb" + conf.ThumbConfig.FileSuffix))
file, err := os.Create(util.RelativePath("TestHandler_Thumb._thumb"))
asserts.NoError(err)
file.Close()
@ -160,6 +181,25 @@ func TestHandler_Source(t *testing.T) {
asserts.Contains(sourceURL, "https://cloudreve.org")
}
// 下载
{
file := model.File{
Model: gorm.Model{
ID: 1,
},
Name: "test.jpg",
}
ctx := context.WithValue(ctx, fsctx.FileModelCtx, file)
baseURL, err := url.Parse("https://cloudreve.org")
asserts.NoError(err)
sourceURL, err := handler.Source(ctx, "", *baseURL, 0, true, 0)
asserts.NoError(err)
asserts.NotEmpty(sourceURL)
asserts.Contains(sourceURL, "sign=")
asserts.Contains(sourceURL, "download")
asserts.Contains(sourceURL, "https://cloudreve.org")
}
// 无法获取上下文
{
baseURL, err := url.Parse("https://cloudreve.org")
@ -241,10 +281,29 @@ func TestHandler_GetDownloadURL(t *testing.T) {
func TestHandler_Token(t *testing.T) {
asserts := assert.New(t)
handler := Driver{}
handler := Driver{
Policy: &model.Policy{},
}
ctx := context.Background()
_, err := handler.Token(ctx, 10, "123")
upSession := &serializer.UploadSession{SavePath: "TestHandler_Token"}
_, err := handler.Token(ctx, 10, upSession, &fsctx.FileStream{})
asserts.NoError(err)
file, _ := os.Create("TestHandler_Token")
defer func() {
file.Close()
os.Remove("TestHandler_Token")
}()
_, err = handler.Token(ctx, 10, upSession, &fsctx.FileStream{})
asserts.Error(err)
asserts.Contains(err.Error(), "already exist")
}
func TestDriver_CancelToken(t *testing.T) {
a := assert.New(t)
handler := Driver{}
a.NoError(handler.CancelToken(context.Background(), &serializer.UploadSession{}))
}
func TestDriver_List(t *testing.T) {

109
pkg/filesystem/driver/onedrive/api.go
Unescape View File

@ -1,7 +1,6 @@
package onedrive
import (
"bytes"
"context"
"encoding/json"
"errors"
@ -18,7 +17,10 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)
@ -29,7 +31,8 @@ const (
// ChunkSize 服务端中转分片上传分片大小
ChunkSize uint64 = 10 * 1024 * 1024
// ListRetry 列取请求重试次数
ListRetry = 1
ListRetry = 1
chunkRetrySleep = time.Second * 5
)
// GetSourcePath 获取文件的绝对路径
@ -219,28 +222,21 @@ func (client *Client) GetUploadSessionStatus(ctx context.Context, uploadURL stri
}
// UploadChunk 上传分片
func (client *Client) UploadChunk(ctx context.Context, uploadURL string, chunk *Chunk) (*UploadSessionResponse, error) {
func (client *Client) UploadChunk(ctx context.Context, uploadURL string, content io.Reader, current *chunk.ChunkGroup) (*UploadSessionResponse, error) {
res, err := client.request(
ctx, "PUT", uploadURL, bytes.NewReader(chunk.Data[0:chunk.ChunkSize]),
request.WithContentLength(int64(chunk.ChunkSize)),
ctx, "PUT", uploadURL, content,
request.WithContentLength(current.Length()),
request.WithHeader(http.Header{
"Content-Range": {fmt.Sprintf("bytes %d-%d/%d", chunk.Offset, chunk.Offset+chunk.ChunkSize-1, chunk.Total)},
"Content-Range": {current.RangeHeader()},
}),
request.WithoutHeader([]string{"Authorization", "Content-Type"}),
request.WithTimeout(time.Duration(300)*time.Second),
)
if err != nil {
// 如果重试次数小于限制，5秒后重试
if chunk.Retried < model.GetIntSetting("onedrive_chunk_retries", 1) {
chunk.Retried++
util.Log().Debug("分片偏移%d上传失败[%s]，5秒钟后重试", chunk.Offset, err)
time.Sleep(time.Duration(5) * time.Second)
return client.UploadChunk(ctx, uploadURL, chunk)
}
return nil, err
return nil, fmt.Errorf("failed to upload OneDrive chunk #%d: %w", current.Index(), err)
}
if chunk.IsLast() {
if current.IsLast() {
return nil, nil
}
@ -281,46 +277,24 @@ func (client *Client) Upload(ctx context.Context, file fsctx.FileHeader) error {
return err
}
offset := 0
chunkNum := size / int(ChunkSize)
if size%int(ChunkSize) != 0 {
chunkNum++
}
chunkData := make([]byte, ChunkSize)
for i := 0; i < chunkNum; i++ {
select {
case <-ctx.Done():
util.Log().Debug("OneDrive 客户端取消")
return ErrClientCanceled
default:
// 分块
chunkSize := int(ChunkSize)
if size-offset < chunkSize {
chunkSize = size - offset
}
// 因为后面需要错误重试，这里要把分片内容读到内存中
chunkContent := chunkData[:chunkSize]
_, err := io.ReadFull(file, chunkContent)
// Initial chunk groups
chunks := chunk.NewChunkGroup(file, client.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
Max: model.GetIntSetting("chunk_retries", 5),
Sleep: chunkRetrySleep,
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
chunk := Chunk{
Offset: offset,
ChunkSize: chunkSize,
Total: size,
Data: chunkContent,
}
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
_, err := client.UploadChunk(ctx, uploadURL, content, current)
return err
}
// 上传
_, err = client.UploadChunk(ctx, uploadURL, &chunk)
if err != nil {
return err
}
offset += chunkSize
// upload chunks
for chunks.Next() {
if err := chunks.Process(uploadFunc); err != nil {
return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
}
}
return nil
}
@ -349,16 +323,6 @@ func (client *Client) SimpleUpload(ctx context.Context, dst string, body io.Read
request.WithTimeout(time.Duration(150)*time.Second),
)
if err != nil {
retried := 0
if v, ok := ctx.Value(fsctx.RetryCtx).(int); ok {
retried = v
}
if retried < model.GetIntSetting("onedrive_chunk_retries", 1) {
retried++
util.Log().Debug("文件[%s]上传失败[%s]，5秒钟后重试", dst, err)
time.Sleep(time.Duration(5) * time.Second)
return client.SimpleUpload(context.WithValue(ctx, fsctx.RetryCtx, retried), dst, body, size, opts...)
}
return nil, err
}
@ -487,9 +451,9 @@ func (client *Client) GetThumbURL(ctx context.Context, dst string, w, h uint) (s
// MonitorUpload 监控客户端分片上传进度
func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size uint64, ttl int64) {
// 回调完成通知chan
callbackChan := make(chan bool)
callbackSignal.Store(callbackKey, callbackChan)
defer callbackSignal.Delete(callbackKey)
callbackChan := mq.GlobalMQ.Subscribe(callbackKey, 1)
defer mq.GlobalMQ.Unsubscribe(callbackKey, callbackChan)
timeout := model.GetIntSetting("onedrive_monitor_timeout", 600)
interval := model.GetIntSetting("onedrive_callback_check", 20)
@ -514,16 +478,16 @@ func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size ui
if resErr, ok := err.(*RespError); ok {
if resErr.APIError.Code == "itemNotFound" {
util.Log().Debug("上传会话已完成，稍后检查回调")
time.Sleep(time.Duration(interval) * time.Second)
util.Log().Debug("开始检查回调")
_, ok := cache.Get("callback_" + callbackKey)
if ok {
select {
case <-time.After(time.Duration(interval) * time.Second):
util.Log().Warning("未发送回调，删除文件")
cache.Deletes([]string{callbackKey}, "callback_")
_, err = client.Delete(context.Background(), []string{path})
if err != nil {
util.Log().Warning("无法删除未回调的文件，%s", err)
}
case <-callbackChan:
util.Log().Debug("客户端完成回调")
}
return
}
@ -560,15 +524,6 @@ func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size ui
}
}
// FinishCallback 向Monitor发送回调结束信号
func FinishCallback(key string) {
if signal, ok := callbackSignal.Load(key); ok {
if signalChan, ok := signal.(chan bool); ok {
close(signalChan)
}
}
}
func sysError(err error) *RespError {
return &RespError{APIError: APIError{
Code: "system",

140
pkg/filesystem/driver/onedrive/api_test.go
Unescape View File

@ -4,6 +4,11 @@ import (
"context"
"errors"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"io"
"io/ioutil"
"net/http"
"strings"
@ -12,7 +17,6 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
@ -307,6 +311,31 @@ func TestClient_Meta(t *testing.T) {
asserts.NotNil(res)
asserts.Equal("123321", res.Name)
}
// 返回正常, 使用资源id
{
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
clientMock.On(
"Request",
"GET",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"name":"123321"}`)),
},
})
client.Request = clientMock
res, err := client.Meta(context.Background(), "123321", "123")
clientMock.AssertExpectations(t)
asserts.NoError(err)
asserts.NotNil(res)
asserts.Equal("123321", res.Name)
}
}
func TestClient_CreateUploadSession(t *testing.T) {
@ -442,9 +471,11 @@ func TestClient_UploadChunk(t *testing.T) {
client, _ := NewClient(&model.Policy{})
client.Credential.AccessToken = "AccessToken"
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
cg := chunk.NewChunkGroup(&fsctx.FileStream{Size: 15}, 10, &backoff.ConstantBackoff{}, false)
// 非最后分片，正常
{
cg.Next()
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
clientMock.On(
@ -453,6 +484,10 @@ func TestClient_UploadChunk(t *testing.T) {
"http://dev.com",
testMock.Anything,
testMock.Anything,
testMock.Anything,
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
@ -461,13 +496,7 @@ func TestClient_UploadChunk(t *testing.T) {
},
})
client.Request = clientMock
res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{
Offset: 0,
ChunkSize: 10,
Total: 100,
Retried: 0,
Data: []byte("12313121231312"),
})
res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("1234567890"), cg)
clientMock.AssertExpectations(t)
asserts.NoError(err)
asserts.Equal("http://dev.com/2", res.UploadURL)
@ -491,13 +520,7 @@ func TestClient_UploadChunk(t *testing.T) {
},
})
client.Request = clientMock
res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{
Offset: 0,
ChunkSize: 10,
Total: 100,
Retried: 0,
Data: []byte("12313112313122"),
})
res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("1234567890"), cg)
clientMock.AssertExpectations(t)
asserts.Error(err)
asserts.Nil(res)
@ -505,6 +528,7 @@ func TestClient_UploadChunk(t *testing.T) {
// 最后分片，正常
{
cg.Next()
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
clientMock.On(
@ -521,53 +545,26 @@ func TestClient_UploadChunk(t *testing.T) {
},
})
client.Request = clientMock
res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{
Offset: 95,
ChunkSize: 5,
Total: 100,
Retried: 0,
Data: []byte("1231312"),
})
res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("12345"), cg)
clientMock.AssertExpectations(t)
asserts.NoError(err)
asserts.Nil(res)
}
// 最后分片，第一次失败，重试后成功
// 最后分片，失败
{
cache.Set("setting_onedrive_chunk_retries", "1", 0)
cache.Set("setting_chunk_retries", "1", 0)
client.Credential.ExpiresIn = 0
go func() {
time.Sleep(time.Duration(2) * time.Second)
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
}()
clientMock := ClientMock{}
clientMock.On(
"Request",
"PUT",
"http://dev.com",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`???`)),
},
})
client.Request = clientMock
chunk := &Chunk{
Offset: 95,
ChunkSize: 5,
Total: 100,
Retried: 0,
Data: []byte("1231312"),
}
res, err := client.UploadChunk(context.Background(), "http://dev.com", chunk)
res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("12345"), cg)
clientMock.AssertExpectations(t)
asserts.NoError(err)
asserts.Error(err)
asserts.Nil(res)
asserts.EqualValues(1, chunk.Retried)
}
}
@ -576,16 +573,21 @@ func TestClient_Upload(t *testing.T) {
client, _ := NewClient(&model.Policy{})
client.Credential.AccessToken = "AccessToken"
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
ctx := context.Background()
cache.Set("setting_chunk_retries", "1", 0)
cache.Set("setting_use_temp_chunk_buffer", "false", 0)
// 小文件，简单上传，失败
{
client.Credential.ExpiresIn = 0
err := client.Upload(ctx, "123.jpg", 3, strings.NewReader("123"))
err := client.Upload(ctx, &fsctx.FileStream{
Size: 5,
File: io.NopCloser(strings.NewReader("12345")),
})
asserts.Error(err)
}
// 上下文取消
// 无法创建分片会话
{
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
@ -598,20 +600,20 @@ func TestClient_Upload(t *testing.T) {
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
StatusCode: 400,
Body: ioutil.NopCloser(strings.NewReader(`{"uploadUrl":"123321"}`)),
},
})
client.Request = clientMock
ctx, cancel := context.WithCancel(context.Background())
cancel()
err := client.Upload(ctx, "123.jpg", 15*1024*1024, strings.NewReader("123"))
err := client.Upload(context.Background(), &fsctx.FileStream{
Size: SmallFileSize + 1,
File: io.NopCloser(strings.NewReader("12345")),
})
clientMock.AssertExpectations(t)
asserts.Error(err)
asserts.Equal(ErrClientCanceled, err)
}
// 无法创建分片会话
// 分片上传失败
{
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
@ -621,6 +623,19 @@ func TestClient_Upload(t *testing.T) {
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"uploadUrl":"123321"}`)),
},
})
clientMock.On(
"Request",
"PUT",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
@ -629,9 +644,13 @@ func TestClient_Upload(t *testing.T) {
},
})
client.Request = clientMock
err := client.Upload(context.Background(), "123.jpg", 15*1024*1024, strings.NewReader("123"))
err := client.Upload(context.Background(), &fsctx.FileStream{
Size: SmallFileSize + 1,
File: io.NopCloser(strings.NewReader("12345")),
})
clientMock.AssertExpectations(t)
asserts.Error(err)
asserts.Contains(err.Error(), "failed to upload chunk")
}
}
@ -641,9 +660,9 @@ func TestClient_SimpleUpload(t *testing.T) {
client, _ := NewClient(&model.Policy{})
client.Credential.AccessToken = "AccessToken"
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
cache.Set("setting_onedrive_chunk_retries", "1", 0)
cache.Set("setting_chunk_retries", "1", 0)
// 请求失败，并重试
// 请求失败
{
client.Credential.ExpiresIn = 0
res, err := client.SimpleUpload(context.Background(), "123.jpg", strings.NewReader("123"), 3)
@ -651,7 +670,6 @@ func TestClient_SimpleUpload(t *testing.T) {
asserts.Nil(res)
}
cache.Set("setting_onedrive_chunk_retries", "0", 0)
// 返回未知响应
{
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
@ -988,7 +1006,7 @@ func TestClient_MonitorUpload(t *testing.T) {
asserts.NotPanics(func() {
go func() {
time.Sleep(time.Duration(1) * time.Second)
FinishCallback("key")
mq.GlobalMQ.Publish("key", mq.Message{})
}()
client.MonitorUpload("url", "key", "path", 10, 10)
})

18
pkg/filesystem/driver/onedrive/handler.go
Unescape View File

@ -226,16 +226,6 @@ func (handler Driver) replaceSourceHost(origin string) (string, error) {
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
fileInfo := file.Info()
// 如果小于4MB，则由服务端中转
if fileInfo.Size <= SmallFileSize {
return nil, nil
}
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/onedrive/finish/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI)
uploadURL, err := handler.Client.CreateUploadSession(ctx, fileInfo.SavePath, WithConflictBehavior("fail"))
if err != nil {
return nil, err
@ -244,13 +234,15 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
// 监控回调及上传
go handler.Client.MonitorUpload(uploadURL, uploadSession.Key, fileInfo.SavePath, fileInfo.Size, ttl)
uploadSession.UploadURL = uploadURL
return &serializer.UploadCredential{
Policy: uploadURL,
Token: apiURL.String(),
SessionID: uploadSession.Key,
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
UploadURLs: []string{uploadURL},
}, nil
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
return handler.Client.DeleteUploadSession(ctx, uploadSession.UploadURL)
}

121
pkg/filesystem/driver/onedrive/handler_test.go
Unescape View File

@ -4,6 +4,9 @@ import (
"context"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/jinzhu/gorm"
"io"
"io/ioutil"
"net/http"
@ -12,51 +15,23 @@ import (
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
)
func TestDriver_Token(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
},
}
// 无法获取文件路径
{
ctx := context.WithValue(context.Background(), fsctx.FileSizeCtx, uint64(10))
res, err := handler.Token(ctx, 10, "key", nil)
asserts.Error(err)
asserts.Equal(serializer.UploadCredential{}, res)
}
// 无法获取文件大小
{
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
res, err := handler.Token(ctx, 10, "key", nil)
asserts.Error(err)
asserts.Equal(serializer.UploadCredential{}, res)
}
// 小文件成功
{
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(10))
res, err := handler.Token(ctx, 10, "key", nil)
asserts.NoError(err)
asserts.Equal(serializer.UploadCredential{}, res)
}
h, _ := NewDriver(&model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
})
handler := h.(Driver)
// 分片上传 失败
{
@ -78,11 +53,9 @@ func TestDriver_Token(t *testing.T) {
},
})
handler.Client.Request = clientMock
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(20*1024*1024))
res, err := handler.Token(ctx, 10, "key", nil)
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
asserts.Error(err)
asserts.Equal(serializer.UploadCredential{}, res)
asserts.Nil(res)
}
// 分片上传 成功
@ -108,15 +81,13 @@ func TestDriver_Token(t *testing.T) {
},
})
handler.Client.Request = clientMock
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(20*1024*1024))
go func() {
time.Sleep(time.Duration(1) * time.Second)
FinishCallback("key")
mq.GlobalMQ.Publish("TestDriver_Token", mq.Message{})
}()
res, err := handler.Token(ctx, 10, "key", nil)
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{Key: "TestDriver_Token"}, &fsctx.FileStream{})
asserts.NoError(err)
asserts.Equal("123321", res.Policy)
asserts.Equal("123321", res.UploadURLs[0])
}
}
@ -295,12 +266,8 @@ func TestDriver_Thumb(t *testing.T) {
// 失败
{
ctx := context.WithValue(context.Background(), fsctx.ThumbSizeCtx, [2]uint{10, 20})
ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{})
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{PicInfo: "1,1", Model: gorm.Model{ID: 1}})
res, err := handler.Thumb(ctx, "123.jpg")
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
asserts.Empty(res.URL)
}
@ -308,7 +275,6 @@ func TestDriver_Thumb(t *testing.T) {
// 上下文错误
{
_, err := handler.Thumb(context.Background(), "123.jpg")
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
}
}
@ -329,7 +295,6 @@ func TestDriver_Delete(t *testing.T) {
// 失败
{
_, err := handler.Delete(context.Background(), []string{"1"})
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
}
@ -350,7 +315,7 @@ func TestDriver_Put(t *testing.T) {
// 失败
{
err := handler.Put(context.Background(), ioutil.NopCloser(strings.NewReader("")), "dst", 0)
err := handler.Put(context.Background(), &fsctx.FileStream{})
asserts.Error(err)
}
}
@ -418,3 +383,55 @@ func TestDriver_Get(t *testing.T) {
asserts.NoError(err)
asserts.Equal("123", string(content))
}
func TestDriver_replaceSourceHost(t *testing.T) {
tests := []struct {
name string
origin string
cdn string
want string
wantErr bool
}{
{"TestNoReplace", "http://1dr.ms/download.aspx?123456", "", "http://1dr.ms/download.aspx?123456", false},
{"TestReplaceCorrect", "http://1dr.ms/download.aspx?123456", "https://test.com:8080", "https://test.com:8080/download.aspx?123456", false},
{"TestCdnFormatError", "http://1dr.ms/download.aspx?123456", string([]byte{0x7f}), "", true},
{"TestSrcFormatError", string([]byte{0x7f}), "https://test.com:8080", "", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
policy := &model.Policy{}
policy.OptionsSerialized.OdProxy = tt.cdn
handler := Driver{
Policy: policy,
}
got, err := handler.replaceSourceHost(tt.origin)
if (err != nil) != tt.wantErr {
t.Errorf("replaceSourceHost() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("replaceSourceHost() got = %v, want %v", got, tt.want)
}
})
}
}
func TestDriver_CancelToken(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
},
}
handler.Client, _ = NewClient(&model.Policy{})
handler.Client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
// 失败
{
err := handler.CancelToken(context.Background(), &serializer.UploadSession{})
asserts.Error(err)
}
}

38
pkg/filesystem/driver/onedrive/handller_test.go
Unescape View File

@ -1,38 +0,0 @@
package onedrive
import (
model "github.com/cloudreve/Cloudreve/v3/models"
"testing"
)
func TestDriver_replaceSourceHost(t *testing.T) {
tests := []struct {
name string
origin string
cdn string
want string
wantErr bool
}{
{"TestNoReplace", "http://1dr.ms/download.aspx?123456", "", "http://1dr.ms/download.aspx?123456", false},
{"TestReplaceCorrect", "http://1dr.ms/download.aspx?123456", "https://test.com:8080", "https://test.com:8080/download.aspx?123456", false},
{"TestCdnFormatError", "http://1dr.ms/download.aspx?123456", string([]byte{0x7f}), "", true},
{"TestSrcFormatError", string([]byte{0x7f}), "https://test.com:8080", "", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
policy := &model.Policy{}
policy.OptionsSerialized.OdProxy = tt.cdn
handler := Driver{
Policy: policy,
}
got, err := handler.replaceSourceHost(tt.origin)
if (err != nil) != tt.wantErr {
t.Errorf("replaceSourceHost() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("replaceSourceHost() got = %v, want %v", got, tt.want)
}
})
}
}

17
pkg/filesystem/driver/onedrive/types.go
Unescape View File

@ -3,7 +3,6 @@ package onedrive
import (
"encoding/gob"
"net/url"
"sync"
)
// RespError 接口返回错误
@ -99,15 +98,6 @@ type ListResponse struct {
Context string `json:"@odata.context"`
}
// Chunk 文件分片
type Chunk struct {
Offset int
ChunkSize int
Total int
Retried int
Data []byte
}
// oauthEndpoint OAuth接口地址
type oauthEndpoint struct {
token url.URL
@ -143,10 +133,3 @@ type Site struct {
func init() {
gob.Register(Credential{})
}
// IsLast 返回是否为最后一个分片
func (chunk *Chunk) IsLast() bool {
return chunk.Total-chunk.Offset == chunk.ChunkSize
}
var callbackSignal sync.Map

188
pkg/filesystem/driver/oss/callback_test.go
Unescape View File

@ -1,188 +0,0 @@
package oss
import (
"io/ioutil"
"net/http"
"net/url"
"strings"
"testing"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/stretchr/testify/assert"
)
func TestGetPublicKey(t *testing.T) {
asserts := assert.New(t)
testCases := []struct {
Request http.Request
ResNil bool
Error bool
}{
// Header解码失败
{
Request: http.Request{
Header: http.Header{
"X-Oss-Pub-Key-Url": {"中文"},
},
},
ResNil: true,
Error: true,
},
// 公钥URL无效
{
Request: http.Request{
Header: http.Header{
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9wb3JuaHViLmNvbQ=="},
},
},
ResNil: true,
Error: true,
},
// 请求失败
{
Request: http.Request{
Header: http.Header{
"X-Oss-Pub-Key-Url": {"aHR0cDovL2dvc3NwdWJsaWMuYWxpY2RuLmNvbS8yMzQyMzQ="},
},
},
ResNil: true,
Error: true,
},
// 成功
{
Request: http.Request{
Header: http.Header{
"X-Oss-Pub-Key-Url": {"aHR0cDovL2dvc3NwdWJsaWMuYWxpY2RuLmNvbS9jYWxsYmFja19wdWJfa2V5X3YxLnBlbQ=="},
},
},
ResNil: false,
Error: false,
},
}
for i, testCase := range testCases {
asserts.NoError(cache.Deletes([]string{"oss_public_key"}, ""))
res, err := GetPublicKey(&testCase.Request)
if testCase.Error {
asserts.Error(err, "Test Case #%d", i)
} else {
asserts.NoError(err, "Test Case #%d", i)
}
if testCase.ResNil {
asserts.Empty(res, "Test Case #%d", i)
} else {
asserts.NotEmpty(res, "Test Case #%d", i)
}
}
// 测试缓存
asserts.NoError(cache.Set("oss_public_key", []byte("123"), 0))
res, err := GetPublicKey(nil)
asserts.NoError(err)
asserts.Equal([]byte("123"), res)
}
func TestVerifyCallbackSignature(t *testing.T) {
asserts := assert.New(t)
testPubKey := `-----BEGIN PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKs/JBGzwUB2aVht4crBx3oIPBLNsjGs
C0fTXv+nvlmklvkcolvpvXLTjaxUHR3W9LXxQ2EHXAJfCB+6H2YF1k8CAwEAAQ==
-----END PUBLIC KEY-----
`
// 成功
{
asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
r := http.Request{
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
Header: map[string][]string{
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
},
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
}
asserts.NoError(VerifyCallbackSignature(&r))
}
// 签名错误
{
asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
r := http.Request{
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
Header: map[string][]string{
"Authorization": {"e3LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
},
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
}
asserts.Error(VerifyCallbackSignature(&r))
}
// GetPubKey 失败
{
asserts.NoError(cache.Deletes([]string{"oss_public_key"}, ""))
r := http.Request{
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
Header: map[string][]string{
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
},
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
}
asserts.Error(VerifyCallbackSignature(&r))
}
// getRequestMD5 失败
{
asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
r := http.Request{
URL: &url.URL{Path: "%测试"},
Header: map[string][]string{
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
},
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
}
asserts.Error(VerifyCallbackSignature(&r))
}
// 无 Authorization 头
{
asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
r := http.Request{
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
Header: map[string][]string{
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
},
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
}
asserts.Error(VerifyCallbackSignature(&r))
}
// pub block 不存在
{
asserts.NoError(cache.Set("oss_public_key", []byte(""), 0))
r := http.Request{
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
Header: map[string][]string{
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
},
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
}
asserts.Error(VerifyCallbackSignature(&r))
}
// ParsePKIXPublicKey出错
{
asserts.NoError(cache.Set("oss_public_key", []byte("-----BEGIN PUBLIC KEY-----\n-----END PUBLIC KEY-----"), 0))
r := http.Request{
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
Header: map[string][]string{
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
},
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
}
asserts.Error(VerifyCallbackSignature(&r))
}
}

209
pkg/filesystem/driver/oss/handler.go
Unescape View File

@ -2,8 +2,6 @@ package oss
import (
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/json"
"errors"
@ -15,8 +13,10 @@ import (
"strings"
"time"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"github.com/HFO4/aliyun-oss-go-sdk/oss"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
@ -48,17 +48,25 @@ type Driver struct {
type key int
const (
chunkRetrySleep = time.Duration(5) * time.Second
// MultiPartUploadThreshold 服务端使用分片上传的阈值
MultiPartUploadThreshold uint64 = 5 * (1 << 30) // 5GB
// VersionID 文件版本标识
VersionID key = iota
)
// CORS 创建跨域策略
func (handler *Driver) CORS() error {
// 初始化客户端
if err := handler.InitOSSClient(false); err != nil {
return err
func NewDriver(policy *model.Policy) (*Driver, error) {
driver := &Driver{
Policy: policy,
HTTPClient: request.NewClient(),
}
return driver, driver.InitOSSClient(false)
}
// CORS 创建跨域策略
func (handler *Driver) CORS() error {
return handler.client.SetBucketCORS(handler.Policy.BucketName, []oss.CORSRule{
{
AllowedOrigin: []string{"*"},
@ -82,39 +90,31 @@ func (handler *Driver) InitOSSClient(forceUsePublicEndpoint bool) error {
return errors.New("存储策略为空")
}
if handler.client == nil {
// 决定是否使用内网 Endpoint
endpoint := handler.Policy.Server
if handler.Policy.OptionsSerialized.ServerSideEndpoint != "" && !forceUsePublicEndpoint {
endpoint = handler.Policy.OptionsSerialized.ServerSideEndpoint
}
// 初始化客户端
client, err := oss.New(endpoint, handler.Policy.AccessKey, handler.Policy.SecretKey)
if err != nil {
return err
}
handler.client = client
// 决定是否使用内网 Endpoint
endpoint := handler.Policy.Server
if handler.Policy.OptionsSerialized.ServerSideEndpoint != "" && !forceUsePublicEndpoint {
endpoint = handler.Policy.OptionsSerialized.ServerSideEndpoint
}
// 初始化存储桶
bucket, err := client.Bucket(handler.Policy.BucketName)
if err != nil {
return err
}
handler.bucket = bucket
// 初始化客户端
client, err := oss.New(endpoint, handler.Policy.AccessKey, handler.Policy.SecretKey)
if err != nil {
return err
}
handler.client = client
// 初始化存储桶
bucket, err := client.Bucket(handler.Policy.BucketName)
if err != nil {
return err
}
handler.bucket = bucket
return nil
}
// List 列出OSS上的文件
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// 初始化客户端
if err := handler.InitOSSClient(false); err != nil {
return nil, err
}
func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// 列取文件
base = strings.TrimPrefix(base, "/")
if base != "" {
@ -181,7 +181,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
}
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 通过VersionID禁止缓存
ctx = context.WithValue(ctx, VersionID, time.Now().UnixNano())
@ -224,17 +224,12 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
fileInfo := file.Info()
// 初始化客户端
if err := handler.InitOSSClient(false); err != nil {
return err
}
// 凭证有效期
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
credentialTTL := model.GetIntSetting("upload_session_timeout", 3600)
// 是否允许覆盖
overwrite := fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite
@ -243,23 +238,40 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
oss.ForbidOverWrite(!overwrite),
}
// 上传文件
err := handler.bucket.PutObject(fileInfo.SavePath, file, options...)
// 小文件直接上传
if fileInfo.Size < MultiPartUploadThreshold {
return handler.bucket.PutObject(fileInfo.SavePath, file, options...)
}
// 超过阈值时使用分片上传
imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...)
if err != nil {
return fmt.Errorf("failed to initiate multipart upload: %w", err)
}
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
Max: model.GetIntSetting("chunk_retries", 5),
Sleep: chunkRetrySleep,
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
_, err := handler.bucket.UploadPart(imur, content, current.Length(), current.Index()+1)
return err
}
return nil
for chunks.Next() {
if err := chunks.Process(uploadFunc); err != nil {
return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
}
}
_, err = handler.bucket.CompleteMultipartUpload(imur, oss.CompleteAll("yes"), oss.ForbidOverWrite(!overwrite))
return err
}
// Delete 删除一个或多个文件，
// 返回未删除的文件
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// 初始化客户端
if err := handler.InitOSSClient(false); err != nil {
return files, err
}
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// 删除文件
delRes, err := handler.bucket.DeleteObjects(files)
@ -277,7 +289,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
// 初始化客户端
if err := handler.InitOSSClient(true); err != nil {
return nil, err
@ -311,7 +323,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
}
// Source 获取外链URL
func (handler Driver) Source(
func (handler *Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
@ -356,7 +368,7 @@ func (handler Driver) Source(
return handler.signSourceURL(ctx, path, ttl, signOptions)
}
func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64, options []oss.Option) (string, error) {
func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64, options []oss.Option) (string, error) {
signedURL, err := handler.bucket.SignURL(path, oss.HTTPGet, ttl, options...)
if err != nil {
return "", err
@ -368,9 +380,6 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
return "", err
}
// 优先使用https
finalURL.Scheme = "https"
// 公有空间替换掉Key及不支持的头
if !handler.Policy.IsPrivate {
query := finalURL.Query()
@ -394,7 +403,8 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + uploadSession.Key)
@ -406,61 +416,66 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
CallbackBody: `{"name":${x:fname},"source_name":${object},"size":${size},"pic_info":"${imageInfo.width},${imageInfo.height}"}`,
CallbackBodyType: "application/json",
}
// 上传策略
savePath := file.Info().SavePath
postPolicy := UploadPolicy{
Expiration: time.Now().UTC().Add(time.Duration(ttl) * time.Second).Format(time.RFC3339),
Conditions: []interface{}{
map[string]string{"bucket": handler.Policy.BucketName},
[]string{"starts-with", "$key", path.Dir(savePath)},
},
callbackPolicyJSON, err := json.Marshal(callbackPolicy)
if err != nil {
return nil, fmt.Errorf("failed to encode callback policy: %w", err)
}
callbackPolicyEncoded := base64.StdEncoding.EncodeToString(callbackPolicyJSON)
if handler.Policy.MaxSize > 0 {
postPolicy.Conditions = append(postPolicy.Conditions,
[]interface{}{"content-length-range", 0, handler.Policy.MaxSize})
// 初始化分片上传
fileInfo := file.Info()
options := []oss.Option{
oss.Expires(time.Now().Add(time.Duration(ttl) * time.Second)),
oss.ForbidOverWrite(true),
}
return handler.getUploadCredential(ctx, postPolicy, callbackPolicy, ttl, savePath)
}
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback CallbackPolicy, TTL int64, savePath string) (*serializer.UploadCredential, error) {
// 处理回调策略
callbackPolicyEncoded := ""
if callback.CallbackURL != "" {
callbackPolicyJSON, err := json.Marshal(callback)
imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...)
if err != nil {
return nil, fmt.Errorf("failed to initialize multipart upload: %w", err)
}
uploadSession.UploadID = imur.UploadID
// 为每个分片签名上传 URL
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false)
urls := make([]string, chunks.Num())
for chunks.Next() {
err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error {
signedURL, err := handler.bucket.SignURL(fileInfo.SavePath, oss.HTTPPut, ttl,
oss.PartNumber(c.Index()+1),
oss.UploadID(imur.UploadID),
oss.ContentType("application/octet-stream"))
if err != nil {
return err
}
urls[c.Index()] = signedURL
return nil
})
if err != nil {
return nil, err
}
callbackPolicyEncoded = base64.StdEncoding.EncodeToString(callbackPolicyJSON)
policy.Conditions = append(policy.Conditions, map[string]string{"callback": callbackPolicyEncoded})
}
// 编码上传策略
policyJSON, err := json.Marshal(policy)
// 签名完成分片上传的URL
completeURL, err := handler.bucket.SignURL(fileInfo.SavePath, oss.HTTPPost, ttl,
oss.UploadID(imur.UploadID),
oss.Expires(time.Now().Add(time.Duration(ttl)*time.Second)),
oss.CompleteAll("yes"),
oss.ForbidOverWrite(true),
oss.CallbackParam(callbackPolicyEncoded))
if err != nil {
return nil, err
}
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
// 签名上传策略
hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey))
_, err = io.WriteString(hmacSign, policyEncoded)
if err != nil {
return nil, err
}
signature := base64.StdEncoding.EncodeToString(hmacSign.Sum(nil))
return &serializer.UploadCredential{
Policy: fmt.Sprintf("%s:%s", callbackPolicyEncoded, policyEncoded),
Path: savePath,
AccessKey: handler.Policy.AccessKey,
Token: signature,
SessionID: uploadSession.Key,
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
UploadID: imur.UploadID,
UploadURLs: urls,
CompleteURL: completeURL,
}, nil
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return handler.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{UploadID: uploadSession.UploadID, Key: uploadSession.SavePath}, nil)
}

354
pkg/filesystem/driver/oss/handler_test.go
Unescape View File

@ -1,354 +0,0 @@
package oss
import (
"context"
"io"
"io/ioutil"
"net/http"
"net/url"
"strings"
"testing"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
)
func TestDriver_InitOSSClient(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
},
}
// 成功
{
asserts.NoError(handler.InitOSSClient(false))
}
// 使用内网Endpoint
{
handler.Policy.OptionsSerialized.ServerSideEndpoint = "endpoint2"
asserts.NoError(handler.InitOSSClient(false))
}
// 未指定存储策略
{
handler := Driver{}
asserts.Error(handler.InitOSSClient(false))
}
}
func TestDriver_CORS(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
},
}
// 失败
{
asserts.NotPanics(func() {
handler.CORS()
})
}
}
func TestDriver_Token(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
},
}
// 成功
{
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
res, err := handler.Token(ctx, 10, "key", nil)
asserts.NoError(err)
asserts.NotEmpty(res.Policy)
asserts.NotEmpty(res.Token)
asserts.Equal(handler.Policy.AccessKey, res.AccessKey)
asserts.Equal("/123", res.Path)
}
// 上下文错误
{
ctx := context.Background()
_, err := handler.Token(ctx, 10, "key", nil)
asserts.Error(err)
}
}
func TestDriver_Source(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
IsPrivate: true,
},
}
// 正常 非下载 无限速
{
res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0)
asserts.NoError(err)
resURL, err := url.Parse(res)
asserts.NoError(err)
query := resURL.Query()
asserts.NotEmpty(query.Get("Signature"))
asserts.NotEmpty(query.Get("Expires"))
asserts.Equal("ak", query.Get("OSSAccessKeyId"))
}
// 限速 + 下载
{
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"})
res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 102401)
asserts.NoError(err)
resURL, err := url.Parse(res)
asserts.NoError(err)
query := resURL.Query()
asserts.NotEmpty(query.Get("Signature"))
asserts.NotEmpty(query.Get("Expires"))
asserts.Equal("ak", query.Get("OSSAccessKeyId"))
asserts.EqualValues("819208", query.Get("x-oss-traffic-limit"))
asserts.NotEmpty(query.Get("response-content-disposition"))
}
// 限速超出范围 + 下载
{
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"})
res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 10)
asserts.NoError(err)
resURL, err := url.Parse(res)
asserts.NoError(err)
query := resURL.Query()
asserts.NotEmpty(query.Get("Signature"))
asserts.NotEmpty(query.Get("Expires"))
asserts.Equal("ak", query.Get("OSSAccessKeyId"))
asserts.EqualValues("819200", query.Get("x-oss-traffic-limit"))
asserts.NotEmpty(query.Get("response-content-disposition"))
}
// 限速超出范围 + 下载
{
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"})
res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 838860801)
asserts.NoError(err)
resURL, err := url.Parse(res)
asserts.NoError(err)
query := resURL.Query()
asserts.NotEmpty(query.Get("Signature"))
asserts.NotEmpty(query.Get("Expires"))
asserts.Equal("ak", query.Get("OSSAccessKeyId"))
asserts.EqualValues("838860800", query.Get("x-oss-traffic-limit"))
asserts.NotEmpty(query.Get("response-content-disposition"))
}
// 公共空间
{
handler.Policy.IsPrivate = false
res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0)
asserts.NoError(err)
resURL, err := url.Parse(res)
asserts.NoError(err)
query := resURL.Query()
asserts.Empty(query.Get("Signature"))
}
// 正常 指定了CDN域名
{
handler.Policy.BaseURL = "https://cqu.edu.cn"
res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0)
asserts.NoError(err)
resURL, err := url.Parse(res)
asserts.NoError(err)
query := resURL.Query()
asserts.Empty(query.Get("Signature"))
asserts.Contains(resURL.String(), handler.Policy.BaseURL)
}
// 强制使用公网 Endpoint
{
handler.Policy.BaseURL = ""
handler.Policy.OptionsSerialized.ServerSideEndpoint = "endpoint.com"
res, err := handler.Source(context.WithValue(context.Background(), fsctx.ForceUsePublicEndpointCtx, false), "/123", url.URL{}, 10, false, 0)
asserts.NoError(err)
resURL, err := url.Parse(res)
asserts.NoError(err)
query := resURL.Query()
asserts.Empty(query.Get("Signature"))
asserts.Contains(resURL.String(), "endpoint.com")
}
}
func TestDriver_Thumb(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
},
}
// 上下文不存在
{
ctx := context.Background()
res, err := handler.Thumb(ctx, "/123.txt")
asserts.Error(err)
asserts.Nil(res)
}
// 成功
{
cache.Set("setting_preview_timeout", "60", 0)
ctx := context.WithValue(context.Background(), fsctx.ThumbSizeCtx, [2]uint{10, 20})
res, err := handler.Thumb(ctx, "/123.jpg")
asserts.NoError(err)
resURL, err := url.Parse(res.URL)
asserts.NoError(err)
urlQuery := resURL.Query()
asserts.Equal("image/resize,m_lfit,h_20,w_10", urlQuery.Get("x-oss-process"))
}
}
func TestDriver_Delete(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "oss-cn-shanghai.aliyuncs.com",
},
}
// 失败
{
res, err := handler.Delete(context.Background(), []string{"1", "2", "3"})
asserts.Error(err)
asserts.Equal([]string{"1", "2", "3"}, res)
}
}
func TestDriver_Put(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "oss-cn-shanghai.aliyuncs.com",
},
}
cache.Set("setting_upload_credential_timeout", "3600", 0)
ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
// 失败
{
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("123")), "/123.txt", 3)
asserts.Error(err)
}
}
type ClientMock struct {
testMock.Mock
}
func (m ClientMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
args := m.Called(method, target, body, opts)
return args.Get(0).(*request.Response)
}
func TestDriver_Get(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "oss-cn-shanghai.aliyuncs.com",
},
HTTPClient: request.NewClient(),
}
cache.Set("setting_preview_timeout", "3600", 0)
// 响应失败
{
res, err := handler.Get(context.Background(), "123.txt")
asserts.Error(err)
asserts.Nil(res)
}
// 响应成功
{
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Size: 3})
clientMock := ClientMock{}
clientMock.On(
"Request",
"GET",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`123`)),
},
})
handler.HTTPClient = clientMock
res, err := handler.Get(ctx, "123.txt")
clientMock.AssertExpectations(t)
asserts.NoError(err)
n, err := res.Seek(0, io.SeekEnd)
asserts.NoError(err)
asserts.EqualValues(3, n)
content, err := ioutil.ReadAll(res)
asserts.NoError(err)
asserts.Equal("123", string(content))
}
}
func TestDriver_List(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
IsPrivate: true,
},
}
// 连接失败
{
res, err := handler.List(context.Background(), "/", true)
asserts.Error(err)
asserts.Empty(res)
}
}

112
pkg/filesystem/driver/qiniu/handler.go
Unescape View File

@ -2,6 +2,7 @@ package qiniu
import (
"context"
"encoding/base64"
"errors"
"fmt"
"net/http"
@ -16,17 +17,31 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/qiniu/api.v7/v7/auth/qbox"
"github.com/qiniu/api.v7/v7/storage"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"github.com/qiniu/go-sdk/v7/storage"
)
// Driver 本地策略适配器
type Driver struct {
Policy *model.Policy
mac *qbox.Mac
cfg *storage.Config
bucket *storage.BucketManager
}
func NewDriver(policy *model.Policy) *Driver {
mac := qbox.NewMac(policy.AccessKey, policy.SecretKey)
cfg := &storage.Config{UseHTTPS: true}
return &Driver{
Policy: policy,
mac: mac,
cfg: cfg,
bucket: storage.NewBucketManager(mac, cfg),
}
}
// List 列出给定路径下的文件
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
base = strings.TrimPrefix(base, "/")
if base != "" {
base += "/"
@ -42,14 +57,8 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
delimiter = "/"
}
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
cfg := storage.Config{
UseHTTPS: true,
}
bucketManager := storage.NewBucketManager(mac, &cfg)
for {
entries, folders, nextMarker, hashNext, err := bucketManager.ListFiles(
entries, folders, nextMarker, hashNext, err := handler.bucket.ListFiles(
handler.Policy.BucketName,
base, delimiter, marker, 1000)
if err != nil {
@ -99,7 +108,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
}
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 给文件名加上随机参数以强制拉取
path = fmt.Sprintf("%s?v=%d", path, time.Now().UnixNano())
@ -143,17 +152,22 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
// 凭证有效期
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
credentialTTL := model.GetIntSetting("upload_session_timeout", 3600)
// 生成上传策略
fileInfo := file.Info()
scope := handler.Policy.BucketName
if fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite {
scope = fmt.Sprintf("%s:%s", handler.Policy.BucketName, fileInfo.SavePath)
}
putPolicy := storage.PutPolicy{
// 指定为覆盖策略
Scope: fmt.Sprintf("%s:%s", handler.Policy.BucketName, fileInfo.SavePath),
Scope: scope,
SaveKey: fileInfo.SavePath,
ForceSaveKey: true,
FsizeLimit: int64(fileInfo.Size),
@ -164,7 +178,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
}
// 生成上传凭证
token, err := handler.getUploadCredential(ctx, putPolicy, int64(credentialTTL))
token, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, int64(credentialTTL), false)
if err != nil {
return err
}
@ -178,7 +192,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
}
// 开始上传
err = formUploader.Put(ctx, &ret, token.Token, fileInfo.SavePath, file, int64(fileInfo.Size), &putExtra)
err = formUploader.Put(ctx, &ret, token.Credential, fileInfo.SavePath, file, int64(fileInfo.Size), &putExtra)
if err != nil {
return err
}
@ -188,19 +202,14 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
// Delete 删除一个或多个文件，
// 返回未删除的文件
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// TODO 大于一千个文件需要分批发送
deleteOps := make([]string, 0, len(files))
for _, key := range files {
deleteOps = append(deleteOps, storage.URIDelete(handler.Policy.BucketName, key))
}
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
cfg := storage.Config{
UseHTTPS: true,
}
bucketManager := storage.NewBucketManager(mac, &cfg)
rets, err := bucketManager.Batch(deleteOps)
rets, err := handler.bucket.Batch(deleteOps)
// 处理删除结果
if err != nil {
@ -217,7 +226,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
var (
thumbSize = [2]uint{400, 300}
ok = false
@ -238,7 +247,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
}
// Source 获取外链URL
func (handler Driver) Source(
func (handler *Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
@ -261,12 +270,11 @@ func (handler Driver) Source(
return handler.signSourceURL(ctx, path, ttl), nil
}
func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64) string {
func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64) string {
var sourceURL string
if handler.Policy.IsPrivate {
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
deadline := time.Now().Add(time.Second * time.Duration(ttl)).Unix()
sourceURL = storage.MakePrivateURL(mac, handler.Policy.BaseURL, path, deadline)
sourceURL = storage.MakePrivateURL(handler.mac, handler.Policy.BaseURL, path, deadline)
} else {
sourceURL = storage.MakePublicURL(handler.Policy.BaseURL, path)
}
@ -274,19 +282,20 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64)
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 创建上传策略
fileInfo := file.Info()
putPolicy := storage.PutPolicy{
Scope: handler.Policy.BucketName,
CallbackURL: apiURL.String(),
CallbackBody: `{"name":"$(fname)","source_name":"$(key)","size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`,
CallbackBody: `{"size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`,
CallbackBodyType: "application/json",
SaveKey: file.Info().SavePath,
SaveKey: fileInfo.SavePath,
ForceSaveKey: true,
FsizeLimit: int64(handler.Policy.MaxSize),
}
@ -295,21 +304,46 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType
}
return handler.getUploadCredential(ctx, putPolicy, ttl)
credential, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, ttl, true)
if err != nil {
return nil, fmt.Errorf("failed to init parts: %w", err)
}
credential.SessionID = uploadSession.Key
credential.ChunkSize = handler.Policy.OptionsSerialized.ChunkSize
uploadSession.UploadURL = credential.UploadURLs[0]
uploadSession.Credential = credential.Credential
return credential, nil
}
// getUploadCredential 签名上传策略
func (handler Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, TTL int64) (*serializer.UploadCredential, error) {
// getUploadCredential 签名上传策略并创建上传会话
func (handler *Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, file *fsctx.UploadTaskInfo, TTL int64, resume bool) (*serializer.UploadCredential, error) {
// 上传凭证
policy.Expires = uint64(TTL)
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
upToken := policy.UploadToken(mac)
upToken := policy.UploadToken(handler.mac)
// 初始化分片上传
resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
upHost, err := resumeUploader.UpHost(handler.Policy.AccessKey, handler.Policy.BucketName)
if err != nil {
return nil, err
}
ret := &storage.InitPartsRet{}
if resume {
err = resumeUploader.InitParts(ctx, upToken, upHost, handler.Policy.BucketName, file.SavePath, true, ret)
}
return &serializer.UploadCredential{
Token: upToken,
}, nil
UploadURLs: []string{upHost + "/buckets/" + handler.Policy.BucketName + "/objects/" + base64.URLEncoding.EncodeToString([]byte(file.SavePath)) + "/uploads/" + ret.UploadID},
Credential: upToken,
}, err
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
return resumeUploader.Client.CallWith(ctx, nil, "DELETE", uploadSession.UploadURL, http.Header{"Authorization": {"UpToken " + uploadSession.Credential}}, nil, 0)
}

4
pkg/filesystem/driver/remote/client.go
Unescape View File

@ -90,9 +90,9 @@ func (c *remoteClient) Upload(ctx context.Context, file fsctx.FileHeader) error
// Initial chunk groups
chunks := chunk.NewChunkGroup(file, c.policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
Max: model.GetIntSetting("onedrive_chunk_retries", 1),
Max: model.GetIntSetting("chunk_retries", 5),
Sleep: chunkRetrySleep,
})
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
return c.uploadChunk(ctx, session.Key, current.Index(), content, overwrite, current.Length())

262
pkg/filesystem/driver/remote/client_test.go
Unescape View File

@ -0,0 +1,262 @@
package remote
import (
"context"
"errors"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
"io/ioutil"
"net/http"
"strings"
"testing"
)
func TestNewClient(t *testing.T) {
a := assert.New(t)
policy := &model.Policy{}
// 无法解析服务端url
{
policy.Server = string([]byte{0x7f})
c, err := NewClient(policy)
a.Error(err)
a.Nil(c)
}
// 成功
{
policy.Server = ""
c, err := NewClient(policy)
a.NoError(err)
a.NotNil(c)
}
}
func TestRemoteClient_Upload(t *testing.T) {
a := assert.New(t)
c, _ := NewClient(&model.Policy{})
// 无法创建上传会话
{
clientMock := requestmock.RequestMock{}
c.(*remoteClient).httpClient = &clientMock
clientMock.On(
"Request",
"PUT",
"upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: errors.New("error"),
})
err := c.Upload(context.Background(), &fsctx.FileStream{})
a.Error(err)
a.Contains(err.Error(), "error")
clientMock.AssertExpectations(t)
}
// 分片上传失败，成功删除上传会话
{
cache.Set("setting_chunk_retries", "1", 0)
clientMock := requestmock.RequestMock{}
c.(*remoteClient).httpClient = &clientMock
clientMock.On(
"Request",
"PUT",
"upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
clientMock.On(
"Request",
"POST",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: errors.New("error"),
})
clientMock.On(
"Request",
"DELETE",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
err := c.Upload(context.Background(), &fsctx.FileStream{})
a.Error(err)
a.Contains(err.Error(), "error")
clientMock.AssertExpectations(t)
}
// 分片上传失败，无法删除上传会话
{
cache.Set("setting_chunk_retries", "1", 0)
clientMock := requestmock.RequestMock{}
c.(*remoteClient).httpClient = &clientMock
clientMock.On(
"Request",
"PUT",
"upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
clientMock.On(
"Request",
"POST",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: errors.New("error"),
})
clientMock.On(
"Request",
"DELETE",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: errors.New("error2"),
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
err := c.Upload(context.Background(), &fsctx.FileStream{})
a.Error(err)
a.Contains(err.Error(), "error")
clientMock.AssertExpectations(t)
}
// 成功
{
cache.Set("setting_chunk_retries", "1", 0)
clientMock := requestmock.RequestMock{}
c.(*remoteClient).httpClient = &clientMock
clientMock.On(
"Request",
"PUT",
"upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
clientMock.On(
"Request",
"POST",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
err := c.Upload(context.Background(), &fsctx.FileStream{})
a.NoError(err)
clientMock.AssertExpectations(t)
}
}
func TestRemoteClient_CreateUploadSessionFailed(t *testing.T) {
a := assert.New(t)
c, _ := NewClient(&model.Policy{})
clientMock := requestmock.RequestMock{}
c.(*remoteClient).httpClient = &clientMock
clientMock.On(
"Request",
"PUT",
"upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":500,"msg":"error"}`)),
},
})
err := c.Upload(context.Background(), &fsctx.FileStream{})
a.Error(err)
a.Contains(err.Error(), "error")
clientMock.AssertExpectations(t)
}
func TestRemoteClient_UploadChunkFailed(t *testing.T) {
a := assert.New(t)
c, _ := NewClient(&model.Policy{})
clientMock := requestmock.RequestMock{}
c.(*remoteClient).httpClient = &clientMock
clientMock.On(
"Request",
"POST",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":500,"msg":"error"}`)),
},
})
err := c.(*remoteClient).uploadChunk(context.Background(), "", 0, strings.NewReader(""), false, 0)
a.Error(err)
a.Contains(err.Error(), "error")
clientMock.AssertExpectations(t)
}
func TestRemoteClient_GetUploadURL(t *testing.T) {
a := assert.New(t)
c, _ := NewClient(&model.Policy{})
// url 解析失败
{
c.(*remoteClient).policy.Server = string([]byte{0x7f})
res, sign, err := c.GetUploadURL(0, "")
a.Error(err)
a.Empty(res)
a.Empty(sign)
}
// 成功
{
c.(*remoteClient).policy.Server = ""
res, sign, err := c.GetUploadURL(0, "")
a.NoError(err)
a.NotEmpty(res)
a.NotEmpty(sign)
}
}

179
pkg/filesystem/driver/remote/handler_test.go
Unescape View File

@ -2,6 +2,9 @@ package remote
import (
"context"
"errors"
"github.com/cloudreve/Cloudreve/v3/pkg/mocks/remoteclientmock"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"io"
"io/ioutil"
"net/http"
@ -14,45 +17,26 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
)
func TestHandler_Token(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
MaxSize: 10,
AutoRename: true,
DirNameRule: "dir",
FileNameRule: "file",
OptionsSerialized: model.PolicyOption{
FileType: []string{"txt"},
},
Server: "http://test.com",
},
AuthInstance: auth.HMACAuth{},
func TestNewDriver(t *testing.T) {
a := assert.New(t)
// remoteClient 初始化失败
{
d, err := NewDriver(&model.Policy{Server: string([]byte{0x7f})})
a.Error(err)
a.Nil(d)
}
ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
auth.General = auth.HMACAuth{SecretKey: []byte("test")}
// 成功
{
cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
credential, err := handler.Token(ctx, 10, "123", nil)
asserts.NoError(err)
policy, err := serializer.DecodeUploadPolicy(credential.Policy)
asserts.NoError(err)
asserts.Equal("http://test.cloudreve.org/api/v3/callback/remote/123", policy.CallbackURL)
asserts.Equal(uint64(10), policy.MaxSize)
asserts.Equal(true, policy.AutoRename)
asserts.Equal("dir", policy.SavePath)
asserts.Equal("file", policy.FileName)
asserts.Equal("file", policy.FileName)
asserts.Equal([]string{"txt"}, policy.AllowedExtension)
d, err := NewDriver(&model.Policy{})
a.NoError(err)
a.NotNil(d)
}
}
func TestHandler_Source(t *testing.T) {
@ -369,6 +353,20 @@ func TestHandler_Get(t *testing.T) {
}
func TestHandler_Put(t *testing.T) {
a := assert.New(t)
handler, _ := NewDriver(&model.Policy{
Type: "remote",
SecretKey: "test",
Server: "http://test.com",
})
clientMock := &remoteclientmock.RemoteClientMock{}
handler.uploadClient = clientMock
clientMock.On("Upload", testMock.Anything, testMock.Anything).Return(errors.New("error"))
a.Error(handler.Put(context.Background(), &fsctx.FileStream{}))
clientMock.AssertExpectations(t)
}
func TestHandler_Thumb(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
@ -379,92 +377,65 @@ func TestHandler_Put(t *testing.T) {
AuthInstance: auth.HMACAuth{},
}
ctx := context.Background()
asserts.NoError(cache.Set("setting_upload_credential_timeout", "3600", 0))
asserts.NoError(cache.Set("setting_preview_timeout", "60", 0))
resp, err := handler.Thumb(ctx, "/1.txt")
asserts.NoError(err)
asserts.True(resp.Redirect)
}
// 成功
func TestHandler_Token(t *testing.T) {
a := assert.New(t)
handler, _ := NewDriver(&model.Policy{})
// 无法创建上传会话
{
ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
handler.Client = clientMock
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
clientMock := &remoteclientmock.RemoteClientMock{}
handler.uploadClient = clientMock
clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10)).Return(errors.New("error"))
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
a.Error(err)
a.Contains(err.Error(), "error")
a.Nil(res)
clientMock.AssertExpectations(t)
asserts.NoError(err)
}
// 请求失败
// 无法创建上传地址
{
ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 404,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
handler.Client = clientMock
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
clientMock := &remoteclientmock.RemoteClientMock{}
handler.uploadClient = clientMock
clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10)).Return(nil)
clientMock.On("GetUploadURL", int64(10), "").Return("", "", errors.New("error"))
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
a.Error(err)
a.Contains(err.Error(), "error")
a.Nil(res)
clientMock.AssertExpectations(t)
asserts.Error(err)
}
// 返回错误
// 成功
{
ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":1}`)),
},
})
handler.Client = clientMock
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
clientMock := &remoteclientmock.RemoteClientMock{}
handler.uploadClient = clientMock
clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10)).Return(nil)
clientMock.On("GetUploadURL", int64(10), "").Return("1", "2", nil)
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
a.NoError(err)
a.NotNil(res)
a.Equal("1", res.UploadURLs[0])
a.Equal("2", res.Credential)
clientMock.AssertExpectations(t)
asserts.Error(err)
}
}
func TestHandler_Thumb(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
Type: "remote",
SecretKey: "test",
Server: "http://test.com",
},
AuthInstance: auth.HMACAuth{},
}
ctx := context.Background()
asserts.NoError(cache.Set("setting_preview_timeout", "60", 0))
resp, err := handler.Thumb(ctx, "/1.txt")
asserts.NoError(err)
asserts.True(resp.Redirect)
func TestDriver_CancelToken(t *testing.T) {
a := assert.New(t)
handler, _ := NewDriver(&model.Policy{})
clientMock := &remoteclientmock.RemoteClientMock{}
handler.uploadClient = clientMock
clientMock.On("DeleteUploadSession", testMock.Anything, "key").Return(errors.New("error"))
err := handler.CancelToken(context.Background(), &serializer.UploadSession{Key: "key"})
a.Error(err)
a.Contains(err.Error(), "error")
clientMock.AssertExpectations(t)
}

193
pkg/filesystem/driver/s3/handler.go
Unescape View File

@ -2,13 +2,12 @@ package s3
import (
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"io"
"net/http"
"net/url"
"path"
@ -47,6 +46,14 @@ type MetaData struct {
Etag string
}
func NewDriver(policy *model.Policy) (*Driver, error) {
driver := &Driver{
Policy: policy,
}
return driver, driver.InitS3Client()
}
// InitS3Client 初始化S3会话
func (handler *Driver) InitS3Client() error {
if handler.Policy == nil {
@ -72,13 +79,7 @@ func (handler *Driver) InitS3Client() error {
}
// List 列出给定路径下的文件
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// 初始化客户端
if err := handler.InitS3Client(); err != nil {
return nil, err
}
func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// 初始化列目录参数
base = strings.TrimPrefix(base, "/")
if base != "" {
@ -155,8 +156,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
}
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 获取文件源地址
downloadURL, err := handler.Source(
ctx,
@ -197,7 +197,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
// 初始化客户端
@ -205,13 +205,15 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
return err
}
uploader := s3manager.NewUploader(handler.sess)
uploader := s3manager.NewUploader(handler.sess, func(u *s3manager.Uploader) {
u.PartSize = int64(handler.Policy.OptionsSerialized.ChunkSize)
})
dst := file.Info().SavePath
_, err := uploader.Upload(&s3manager.UploadInput{
Bucket: &handler.Policy.BucketName,
Key: &dst,
Body: file,
Body: io.LimitReader(file, int64(file.Info().Size)),
})
if err != nil {
@ -223,13 +225,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
// Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// 初始化客户端
if err := handler.InitS3Client(); err != nil {
return files, err
}
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
failed := make([]string, 0, len(files))
deleted := make([]string, 0, len(files))
@ -263,12 +259,12 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
return nil, errors.New("未实现")
}
// Source 获取外链URL
func (handler Driver) Source(
func (handler *Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
@ -325,42 +321,75 @@ func (handler Driver) Source(
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/s3/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 上传策略
savePath := file.Info().SavePath
putPolicy := UploadPolicy{
Expiration: time.Now().UTC().Add(time.Duration(ttl) * time.Second).Format(time.RFC3339),
Conditions: []interface{}{
map[string]string{"bucket": handler.Policy.BucketName},
[]string{"starts-with", "$key", savePath},
[]string{"starts-with", "$success_action_redirect", apiURL.String()},
[]string{"starts-with", "$name", ""},
[]string{"starts-with", "$Content-Type", ""},
map[string]string{"x-amz-algorithm": "AWS4-HMAC-SHA256"},
},
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
// 检查文件是否存在
fileInfo := file.Info()
if _, err := handler.Meta(ctx, fileInfo.SavePath); err == nil {
return nil, fmt.Errorf("file already exist")
}
if handler.Policy.MaxSize > 0 {
putPolicy.Conditions = append(putPolicy.Conditions,
[]interface{}{"content-length-range", 0, handler.Policy.MaxSize})
// 创建分片上传
expires := time.Now().Add(time.Duration(ttl) * time.Second)
res, err := handler.svc.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
Bucket: &handler.Policy.BucketName,
Key: &fileInfo.SavePath,
Expires: &expires,
})
if err != nil {
return nil, fmt.Errorf("failed to create multipart upload: %w", err)
}
// 生成上传凭证
return handler.getUploadCredential(ctx, putPolicy, apiURL, savePath)
}
uploadSession.UploadID = *res.UploadId
// 为每个分片签名上传 URL
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false)
urls := make([]string, chunks.Num())
for chunks.Next() {
err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error {
signedReq, _ := handler.svc.UploadPartRequest(&s3.UploadPartInput{
Bucket: &handler.Policy.BucketName,
Key: &fileInfo.SavePath,
PartNumber: aws.Int64(int64(c.Index() + 1)),
UploadId: res.UploadId,
})
signedURL, err := signedReq.Presign(time.Duration(ttl) * time.Second)
if err != nil {
return err
}
urls[c.Index()] = signedURL
return nil
})
if err != nil {
return nil, err
}
}
// Meta 获取文件信息
func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
// 初始化客户端
if err := handler.InitS3Client(); err != nil {
// 签名完成分片上传的请求URL
signedReq, _ := handler.svc.CompleteMultipartUploadRequest(&s3.CompleteMultipartUploadInput{
Bucket: &handler.Policy.BucketName,
Key: &fileInfo.SavePath,
UploadId: res.UploadId,
})
signedURL, err := signedReq.Presign(time.Duration(ttl) * time.Second)
if err != nil {
return nil, err
}
// 生成上传凭证
return &serializer.UploadCredential{
SessionID: uploadSession.Key,
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
UploadID: *res.UploadId,
UploadURLs: urls,
CompleteURL: signedURL,
}, nil
}
// Meta 获取文件信息
func (handler *Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
res, err := handler.svc.GetObject(
&s3.GetObjectInput{
Bucket: &handler.Policy.BucketName,
@ -378,52 +407,8 @@ func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error)
}
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback *url.URL, savePath string) (*serializer.UploadCredential, error) {
longDate := time.Now().UTC().Format("20060102T150405Z")
shortDate := time.Now().UTC().Format("20060102")
credential := handler.Policy.AccessKey + "/" + shortDate + "/" + handler.Policy.OptionsSerialized.Region + "/s3/aws4_request"
policy.Conditions = append(policy.Conditions, map[string]string{"x-amz-credential": credential})
policy.Conditions = append(policy.Conditions, map[string]string{"x-amz-date": longDate})
// 编码上传策略
policyJSON, err := json.Marshal(policy)
if err != nil {
return nil, err
}
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
//签名
signature := getHMAC([]byte("AWS4"+handler.Policy.SecretKey), []byte(shortDate))
signature = getHMAC(signature, []byte(handler.Policy.OptionsSerialized.Region))
signature = getHMAC(signature, []byte("s3"))
signature = getHMAC(signature, []byte("aws4_request"))
signature = getHMAC(signature, []byte(policyEncoded))
return &serializer.UploadCredential{
Policy: policyEncoded,
Callback: callback.String(),
Token: hex.EncodeToString(signature),
AccessKey: credential,
Path: savePath,
KeyTime: longDate,
}, nil
}
func getHMAC(key []byte, data []byte) []byte {
hash := hmac.New(sha256.New, key)
hash.Write(data)
return hash.Sum(nil)
}
// CORS 创建跨域策略
func (handler Driver) CORS() error {
// 初始化客户端
if err := handler.InitS3Client(); err != nil {
return err
}
func (handler *Driver) CORS() error {
rule := s3.CORSRule{
AllowedMethods: aws.StringSlice([]string{
"GET",
@ -434,6 +419,7 @@ func (handler Driver) CORS() error {
}),
AllowedOrigins: aws.StringSlice([]string{"*"}),
AllowedHeaders: aws.StringSlice([]string{"*"}),
ExposeHeaders: aws.StringSlice([]string{"ETag"}),
MaxAgeSeconds: aws.Int64(3600),
}
@ -448,6 +434,11 @@ func (handler Driver) CORS() error {
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
_, err := handler.svc.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
UploadId: &uploadSession.UploadID,
Bucket: &handler.Policy.BucketName,
Key: &uploadSession.SavePath,
})
return err
}

25
pkg/filesystem/driver/upyun/handler.go
Unescape View File

@ -311,8 +311,6 @@ func (handler Driver) signURL(ctx context.Context, path *url.URL, TTL int64) (st
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
// 检查文件大小
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/upyun/" + uploadSession.Key)
@ -332,17 +330,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
}
// 生成上传凭证
return handler.getUploadCredential(ctx, putPolicy)
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
}
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy) (*serializer.UploadCredential, error) {
// 生成上传策略
policyJSON, err := json.Marshal(policy)
policyJSON, err := json.Marshal(putPolicy)
if err != nil {
return nil, err
}
@ -353,11 +341,18 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli
signStr := handler.Sign(ctx, elements)
return &serializer.UploadCredential{
Policy: policyEncoded,
Token: signStr,
SessionID: uploadSession.Key,
Policy: policyEncoded,
Credential: signStr,
UploadURLs: []string{"https://v0.api.upyun.com/" + handler.Policy.BucketName},
}, nil
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
}
// Sign 计算又拍云的签名头
func (handler Driver) Sign(ctx context.Context, elements []string) string {
password := fmt.Sprintf("%x", md5.Sum([]byte(handler.Policy.SecretKey)))

19
pkg/filesystem/file.go
Unescape View File

@ -48,9 +48,6 @@ func (fs *FileSystem) AddFile(ctx context.Context, parent *model.Folder, file fs
// 添加文件记录前的钩子
err := fs.Trigger(ctx, "BeforeAddFile", file)
if err != nil {
if err := fs.Trigger(ctx, "BeforeAddFileFailed", file); err != nil {
util.Log().Debug("BeforeAddFileFailed 钩子执行失败，%s", err)
}
return nil, err
}
@ -180,6 +177,7 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m
for policyID, toBeDeletedFiles := range files {
// 列举出需要物理删除的文件的物理路径
sourceNamesAll := make([]string, 0, len(toBeDeletedFiles))
uploadSessions := make([]*serializer.UploadSession, 0, len(toBeDeletedFiles))
for i := 0; i < len(toBeDeletedFiles); i++ {
sourceNamesAll = append(sourceNamesAll, toBeDeletedFiles[i].SourceName)
@ -187,11 +185,7 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m
if toBeDeletedFiles[i].UploadSessionID != nil {
if session, ok := cache.Get(UploadSessionCachePrefix + *toBeDeletedFiles[i].UploadSessionID); ok {
uploadSession := session.(serializer.UploadSession)
if err := fs.Handler.CancelToken(ctx, &uploadSession); err != nil {
util.Log().Warning("无法取消 [%s] 的上传会话: %s", err)
}
cache.Deletes([]string{*toBeDeletedFiles[i].UploadSessionID}, UploadSessionCachePrefix)
uploadSessions = append(uploadSessions, &uploadSession)
}
}
@ -205,6 +199,15 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m
continue
}
// 取消上传会话
for _, upSession := range uploadSessions {
if err := fs.Handler.CancelToken(ctx, upSession); err != nil {
util.Log().Warning("无法取消 [%s] 的上传会话: %s", upSession.Name, err)
}
cache.Deletes([]string{upSession.Key}, UploadSessionCachePrefix)
}
// 执行删除
failedFile, _ := fs.Handler.Delete(ctx, sourceNamesAll)
failed[policyID] = failedFile

61
pkg/filesystem/file_test.go
Unescape View File

@ -2,6 +2,7 @@ package filesystem
import (
"context"
"errors"
"os"
"testing"
@ -19,8 +20,9 @@ import (
func TestFileSystem_AddFile(t *testing.T) {
asserts := assert.New(t)
file := fsctx.FileStream{
Size: 5,
Name: "1.png",
Size: 5,
Name: "1.png",
SavePath: "/Uploads/1_sad.png",
}
folder := model.Folder{
Model: gorm.Model{
@ -39,24 +41,55 @@ func TestFileSystem_AddFile(t *testing.T) {
},
},
},
Policy: &model.Policy{Type: "cos"},
}
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/Uploads/1_sad.png")
_, err := fs.AddFile(ctx, &folder)
_, err := fs.AddFile(context.Background(), &folder, &file)
asserts.Error(err)
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
f, err := fs.AddFile(ctx, &folder)
f, err := fs.AddFile(context.Background(), &folder, &file)
asserts.NoError(err)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Equal("/Uploads/1_sad.png", f.SourceName)
asserts.NotEmpty(f.PicInfo)
// 前置钩子执行失败
{
hookExecuted := false
fs.Use("BeforeAddFile", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
hookExecuted = true
return errors.New("error")
})
f, err := fs.AddFile(context.Background(), &folder, &file)
asserts.Error(err)
asserts.Nil(f)
asserts.True(hookExecuted)
}
// 后置钩子执行失败
{
hookExecuted := false
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
fs.Hooks = map[string][]Hook{}
fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
hookExecuted = true
return errors.New("error")
})
f, err := fs.AddFile(context.Background(), &folder, &file)
asserts.Error(err)
asserts.Nil(f)
asserts.True(hookExecuted)
asserts.NoError(mock.ExpectationsWereMet())
}
}
func TestFileSystem_GetContent(t *testing.T) {
@ -263,6 +296,22 @@ func TestFileSystem_deleteGroupedFile(t *testing.T) {
3: {},
}, failed)
}
// 包含上传会话文件
{
sessionID := "session"
cache.Set(UploadSessionCachePrefix+sessionID, serializer.UploadSession{Key: sessionID}, 0)
files[1].Policy.Type = "local"
files[3].Policy.Type = "local"
files[0].UploadSessionID = &sessionID
failed := fs.deleteGroupedFile(ctx, fs.GroupFileByPolicy(ctx, files))
asserts.Equal(map[uint][]string{
1: {},
2: {},
3: {},
}, failed)
_, ok := cache.Get(UploadSessionCachePrefix + sessionID)
asserts.False(ok)
}
}
func TestFileSystem_GetSource(t *testing.T) {

28
pkg/filesystem/filesystem.go
Unescape View File

@ -121,12 +121,11 @@ func NewAnonymousFileSystem() (*FileSystem, error) {
// DispatchHandler 根据存储策略分配文件适配器
func (fs *FileSystem) DispatchHandler() error {
currentPolicy := fs.Policy
policyType := currentPolicy.Type
if currentPolicy == nil {
return ErrUnknownPolicyType
if fs.Policy == nil {
return errors.New("未设置存储策略")
}
policyType := fs.Policy.Type
currentPolicy := fs.Policy
switch policyType {
case "mock", "anonymous":
@ -144,16 +143,12 @@ func (fs *FileSystem) DispatchHandler() error {
fs.Handler = handler
case "qiniu":
fs.Handler = qiniu.Driver{
Policy: currentPolicy,
}
fs.Handler = qiniu.NewDriver(currentPolicy)
return nil
case "oss":
fs.Handler = oss.Driver{
Policy: currentPolicy,
HTTPClient: request.NewClient(),
}
return nil
handler, err := oss.NewDriver(currentPolicy)
fs.Handler = handler
return err
case "upyun":
fs.Handler = upyun.Driver{
Policy: currentPolicy,
@ -178,10 +173,9 @@ func (fs *FileSystem) DispatchHandler() error {
}
return nil
case "s3":
fs.Handler = s3.Driver{
Policy: currentPolicy,
}
return nil
handler, err := s3.NewDriver(currentPolicy)
fs.Handler = handler
return err
default:
return ErrUnknownPolicyType
}

45
pkg/filesystem/filesystem_test.go
Unescape View File

@ -2,16 +2,16 @@ package filesystem
import (
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/masterinslave"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/slaveinmaster"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"net/http/httptest"
"github.com/DATA-DOG/go-sqlmock"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/remote"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
@ -36,7 +36,7 @@ func TestNewFileSystem(t *testing.T) {
fs, err = NewFileSystem(&user)
asserts.NoError(err)
asserts.NotNil(fs.Handler)
asserts.IsType(remote.Driver{}, fs.Handler)
asserts.IsType(&remote.Driver{}, fs.Handler)
user.Policy.Type = "unknown"
fs, err = NewFileSystem(&user)
@ -64,9 +64,10 @@ func TestNewFileSystemFromContext(t *testing.T) {
func TestDispatchHandler(t *testing.T) {
asserts := assert.New(t)
fs := &FileSystem{
User: &model.User{Policy: model.Policy{
User: &model.User{},
Policy: &model.Policy{
Type: "local",
}},
},
}
// 未指定，使用用户默认
@ -95,7 +96,7 @@ func TestDispatchHandler(t *testing.T) {
err = fs.DispatchHandler()
asserts.NoError(err)
fs.Policy = &model.Policy{Type: "oss"}
fs.Policy = &model.Policy{Type: "oss", Server: "https://s.com", BucketName: "1234"}
err = fs.DispatchHandler()
asserts.NoError(err)
@ -140,23 +141,6 @@ func TestNewFileSystemFromCallback(t *testing.T) {
asserts.Error(err)
}
// 找不到上传策略
{
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Set("user", &model.User{
Policy: model.Policy{
Type: "local",
},
})
c.Set("callbackSession", &serializer.UploadSession{PolicyID: 138})
cache.Deletes([]string{"138"}, "policy_")
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}))
fs, err := NewFileSystemFromCallback(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Nil(fs)
asserts.Error(err)
}
// 成功
{
c, _ := gin.CreateTestContext(httptest.NewRecorder())
@ -165,11 +149,8 @@ func TestNewFileSystemFromCallback(t *testing.T) {
Type: "local",
},
})
c.Set("callbackSession", &serializer.UploadSession{PolicyID: 138})
cache.Deletes([]string{"138"}, "policy_")
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id", "type", "options"}).AddRow(138, "local", "{}"))
c.Set(UploadSessionCtx, &serializer.UploadSession{Policy: model.Policy{Type: "local"}})
fs, err := NewFileSystemFromCallback(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NotNil(fs)
asserts.NoError(err)
}
@ -234,6 +215,16 @@ func TestNewAnonymousFileSystem(t *testing.T) {
asserts.Error(err)
asserts.Nil(fs)
}
// 从机
{
conf.SystemConfig.Mode = "slave"
fs, err := NewAnonymousFileSystem()
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
asserts.NotNil(fs)
asserts.NotNil(fs.Handler)
}
}
func TestFileSystem_Recycle(t *testing.T) {

7
pkg/filesystem/fsctx/stream.go
Unescape View File

@ -1,6 +1,7 @@
package fsctx
import (
"errors"
"io"
"time"
)
@ -75,7 +76,11 @@ func (file *FileStream) Close() error {
}
func (file *FileStream) Seek(offset int64, whence int) (int64, error) {
return file.Seeker.Seek(offset, whence)
if file.Seekable() {
return file.Seeker.Seek(offset, whence)
}
return 0, errors.New("no seeker")
}
func (file *FileStream) Seekable() bool {

74
pkg/filesystem/fsctx/stream_test.go
Unescape View File

@ -1,30 +1,15 @@
package fsctx
import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/stretchr/testify/assert"
"io"
"io/ioutil"
"os"
"strings"
"testing"
)
func TestFileStream_GetFileName(t *testing.T) {
asserts := assert.New(t)
file := FileStream{Name: "123"}
asserts.Equal("123", file.GetFileName())
}
func TestFileStream_GetMIMEType(t *testing.T) {
asserts := assert.New(t)
file := FileStream{MIMEType: "123"}
asserts.Equal("123", file.GetMIMEType())
}
func TestFileStream_GetSize(t *testing.T) {
asserts := assert.New(t)
file := FileStream{Size: 123}
asserts.Equal(uint64(123), file.GetSize())
}
func TestFileStream_Read(t *testing.T) {
asserts := assert.New(t)
file := FileStream{
@ -40,9 +25,54 @@ func TestFileStream_Read(t *testing.T) {
func TestFileStream_Close(t *testing.T) {
asserts := assert.New(t)
file := FileStream{
File: ioutil.NopCloser(strings.NewReader("123")),
{
file := FileStream{
File: ioutil.NopCloser(strings.NewReader("123")),
}
err := file.Close()
asserts.NoError(err)
}
{
file := FileStream{}
err := file.Close()
asserts.NoError(err)
}
}
func TestFileStream_Seek(t *testing.T) {
asserts := assert.New(t)
f, _ := os.CreateTemp("", "*")
defer func() {
f.Close()
os.Remove(f.Name())
}()
{
file := FileStream{
File: f,
Seeker: f,
}
res, err := file.Seek(0, io.SeekStart)
asserts.NoError(err)
asserts.EqualValues(0, res)
}
err := file.Close()
asserts.NoError(err)
{
file := FileStream{}
res, err := file.Seek(0, io.SeekStart)
asserts.Error(err)
asserts.EqualValues(0, res)
}
}
func TestFileStream_Info(t *testing.T) {
a := assert.New(t)
file := FileStream{}
a.NotNil(file.Info())
file.SetSize(10)
a.EqualValues(10, file.Info().Size)
file.SetModel(&model.File{})
a.NotNil(file.Info().Model)
}

1
pkg/filesystem/fsctx/taskinfo/taskinfo.go
Unescape View File

@ -1 +0,0 @@
package taskinfo

4
pkg/filesystem/hooks.go
Unescape View File

@ -5,7 +5,6 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
@ -149,7 +148,6 @@ func HookCancelContext(ctx context.Context, fs *FileSystem, file fsctx.FileHeade
}
// HookUpdateSourceName 更新文件SourceName
// TODO：测试
func HookUpdateSourceName(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File)
if !ok {
@ -245,7 +243,7 @@ func HookGenerateThumb(ctx context.Context, fs *FileSystem, fileHeader fsctx.Fil
fs.recycleLock.Lock()
go func() {
defer fs.recycleLock.Unlock()
_, _ = fs.Handler.Delete(ctx, []string{fileMode.SourceName + conf.ThumbConfig.FileSuffix})
_, _ = fs.Handler.Delete(ctx, []string{fileMode.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")})
fs.GenerateThumbnail(ctx, fileMode)
}()
}

537
pkg/filesystem/hooks_test.go
Unescape View File

@ -3,22 +3,20 @@ package filesystem
import (
"context"
"errors"
"io"
"io/ioutil"
"net/http"
"os"
"strings"
"sync"
"testing"
"github.com/DATA-DOG/go-sqlmock"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"io/ioutil"
"net/http"
"strings"
"testing"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
@ -26,78 +24,72 @@ import (
func TestGenericBeforeUpload(t *testing.T) {
asserts := assert.New(t)
file := fsctx.FileStream{
file := &fsctx.FileStream{
Size: 5,
Name: "1.txt",
}
ctx := context.Background()
cache.Set("pack_size_0", uint64(0), 0)
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
fs := FileSystem{
User: &model.User{
Storage: 0,
Group: model.Group{
MaxStorage: 11,
},
Policy: model.Policy{
MaxSize: 4,
OptionsSerialized: model.PolicyOption{
FileType: []string{"txt"},
},
},
Policy: &model.Policy{
MaxSize: 4,
OptionsSerialized: model.PolicyOption{
FileType: []string{"txt"},
},
},
}
asserts.Error(HookValidateFile(ctx, &fs))
asserts.Error(HookValidateFile(ctx, &fs, file))
file.Size = 1
file.Name = "1"
ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
asserts.Error(HookValidateFile(ctx, &fs))
asserts.Error(HookValidateFile(ctx, &fs, file))
file.Name = "1.txt"
ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
asserts.NoError(HookValidateFile(ctx, &fs))
asserts.NoError(HookValidateFile(ctx, &fs, file))
file.Name = "1.t/xt"
ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
asserts.Error(HookValidateFile(ctx, &fs))
asserts.Error(HookValidateFile(ctx, &fs, file))
}
func TestGenericAfterUploadCanceled(t *testing.T) {
asserts := assert.New(t)
f, err := os.Create("TestGenericAfterUploadCanceled")
asserts.NoError(err)
f.Close()
file := fsctx.FileStream{
Size: 5,
Name: "TestGenericAfterUploadCanceled",
file := &fsctx.FileStream{
Size: 5,
Name: "TestGenericAfterUploadCanceled",
SavePath: "TestGenericAfterUploadCanceled",
}
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "TestGenericAfterUploadCanceled")
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
ctx := context.Background()
fs := FileSystem{
User: &model.User{Storage: 5},
Handler: local.Driver{},
User: &model.User{},
}
// 成功
err = HookDeleteTempFile(ctx, &fs)
asserts.NoError(err)
err = HookGiveBackCapacity(ctx, &fs)
asserts.NoError(err)
asserts.Equal(uint64(0), fs.User.Storage)
f, err = os.Create("TestGenericAfterUploadCanceled")
asserts.NoError(err)
f.Close()
{
mockHandler := &FileHeaderMock{}
fs.Handler = mockHandler
mockHandler.On("Delete", testMock.Anything, testMock.Anything).Return([]string{}, nil)
err := HookDeleteTempFile(ctx, &fs, file)
asserts.NoError(err)
mockHandler.AssertExpectations(t)
}
// 容量不能再降低
err = HookGiveBackCapacity(ctx, &fs)
asserts.Error(err)
// 失败
{
mockHandler := &FileHeaderMock{}
fs.Handler = mockHandler
mockHandler.On("Delete", testMock.Anything, testMock.Anything).Return([]string{}, errors.New(""))
err := HookDeleteTempFile(ctx, &fs, file)
asserts.NoError(err)
mockHandler.AssertExpectations(t)
}
//文件不存在
fs.User.Storage = 5
err = HookDeleteTempFile(ctx, &fs)
asserts.NoError(err)
}
func TestGenericAfterUpload(t *testing.T) {
@ -108,13 +100,14 @@ func TestGenericAfterUpload(t *testing.T) {
ID: 1,
},
},
Policy: &model.Policy{},
}
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{
ctx := context.Background()
file := &fsctx.FileStream{
VirtualPath: "/我的文件",
Name: "test.txt",
})
ctx = context.WithValue(ctx, fsctx.SavePathCtx, "")
}
// 正常
mock.ExpectQuery("SELECT(.+)").
@ -127,9 +120,10 @@ func TestGenericAfterUpload(t *testing.T) {
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found"))
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := GenericAfterUpload(ctx, &fs)
err := GenericAfterUpload(ctx, &fs, file)
asserts.NoError(err)
asserts.NoError(mock.ExpectationsWereMet())
@ -137,7 +131,7 @@ func TestGenericAfterUpload(t *testing.T) {
mock.ExpectQuery("SELECT(.+)folders(.+)").WillReturnRows(
mock.NewRows([]string{"name"}),
)
err = GenericAfterUpload(ctx, &fs)
err = GenericAfterUpload(ctx, &fs, file)
asserts.Equal(ErrRootProtected, err)
asserts.NoError(mock.ExpectationsWereMet())
@ -152,10 +146,25 @@ func TestGenericAfterUpload(t *testing.T) {
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnRows(
mock.NewRows([]string{"name"}).AddRow("test.txt"),
)
err = GenericAfterUpload(ctx, &fs)
err = GenericAfterUpload(ctx, &fs, file)
asserts.Equal(ErrFileExisted, err)
asserts.NoError(mock.ExpectationsWereMet())
// 文件已存在, 且为上传占位符
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
// 1
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 1, "我的文件").
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1))
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnRows(
mock.NewRows([]string{"name", "upload_session_id"}).AddRow("test.txt", "1"),
)
err = GenericAfterUpload(ctx, &fs, file)
asserts.Equal(ErrFileUploadSessionExisted, err)
asserts.NoError(mock.ExpectationsWereMet())
// 插入失败
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
@ -170,7 +179,7 @@ func TestGenericAfterUpload(t *testing.T) {
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
err = GenericAfterUpload(ctx, &fs)
err = GenericAfterUpload(ctx, &fs, file)
asserts.Equal(ErrInsertFileRecord, err)
asserts.NoError(mock.ExpectationsWereMet())
@ -180,7 +189,7 @@ func TestFileSystem_Use(t *testing.T) {
asserts := assert.New(t)
fs := FileSystem{}
hook := func(ctx context.Context, fs *FileSystem) error {
hook := func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
return nil
}
@ -215,77 +224,79 @@ func TestFileSystem_Trigger(t *testing.T) {
}
ctx := context.Background()
hook := func(ctx context.Context, fs *FileSystem) error {
hook := func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
fs.User.Storage++
return nil
}
// 一个
fs.Use("BeforeUpload", hook)
err := fs.Trigger(ctx, "BeforeUpload")
err := fs.Trigger(ctx, "BeforeUpload", nil)
asserts.NoError(err)
asserts.Equal(uint64(1), fs.User.Storage)
// 多个
fs.Use("BeforeUpload", hook)
fs.Use("BeforeUpload", hook)
err = fs.Trigger(ctx, "BeforeUpload")
err = fs.Trigger(ctx, "BeforeUpload", nil)
asserts.NoError(err)
asserts.Equal(uint64(4), fs.User.Storage)
// 多个,有失败
fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
return errors.New("error")
})
fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
asserts.Fail("following hooks executed")
return nil
})
err = fs.Trigger(ctx, "BeforeUpload", nil)
asserts.Error(err)
}
func TestHookIsFileExist(t *testing.T) {
func TestHookValidateCapacity(t *testing.T) {
asserts := assert.New(t)
cache.Set("pack_size_1", uint64(0), 0)
fs := &FileSystem{User: &model.User{
Model: gorm.Model{
ID: 1,
Model: gorm.Model{ID: 1},
Storage: 0,
Group: model.Group{
MaxStorage: 11,
},
}}
ctx := context.WithValue(context.Background(), fsctx.PathCtx, "/test.txt")
ctx := context.Background()
file := &fsctx.FileStream{Size: 11}
{
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)").WithArgs(uint(1), "test.txt").WillReturnRows(
sqlmock.NewRows([]string{"Name"}).AddRow("s"),
)
err := HookIsFileExist(ctx, fs)
asserts.NoError(mock.ExpectationsWereMet())
err := HookValidateCapacity(ctx, fs, file)
asserts.NoError(err)
}
{
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)").WithArgs(uint(1), "test.txt").WillReturnRows(
sqlmock.NewRows([]string{"Name"}),
)
err := HookIsFileExist(ctx, fs)
asserts.NoError(mock.ExpectationsWereMet())
file.Size = 12
err := HookValidateCapacity(ctx, fs, file)
asserts.Error(err)
}
}
func TestHookValidateCapacity(t *testing.T) {
asserts := assert.New(t)
cache.Set("pack_size_1", uint64(0), 0)
func TestHookValidateCapacityDiff(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{User: &model.User{
Model: gorm.Model{ID: 1},
Storage: 0,
Group: model.Group{
MaxStorage: 11,
},
}}
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{Size: 10})
file := model.File{Size: 10}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
// 无需操作
{
err := HookValidateCapacity(ctx, fs)
asserts.NoError(err)
a.NoError(HookValidateCapacityDiff(ctx, fs, &fsctx.FileStream{Size: 10}))
}
// 需要验证
{
err := HookValidateCapacity(ctx, fs)
asserts.Error(err)
a.Error(HookValidateCapacityDiff(ctx, fs, &fsctx.FileStream{Size: 12}))
}
}
func TestHookResetPolicy(t *testing.T) {
@ -301,7 +312,7 @@ func TestHookResetPolicy(t *testing.T) {
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "type"}).AddRow(2, "local"))
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
err := HookResetPolicy(ctx, fs)
err := HookResetPolicy(ctx, fs, nil)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
@ -310,76 +321,22 @@ func TestHookResetPolicy(t *testing.T) {
{
cache.Deletes([]string{"2"}, "policy_")
ctx := context.Background()
err := HookResetPolicy(ctx, fs)
err := HookResetPolicy(ctx, fs, nil)
asserts.Error(err)
}
}
func TestHookChangeCapacity(t *testing.T) {
asserts := assert.New(t)
cache.Set("pack_size_1", uint64(0), 0)
// 容量增加 失败
{
fs := &FileSystem{User: &model.User{
Model: gorm.Model{ID: 1},
}}
newFile := fsctx.FileStream{Size: 10}
oldFile := model.File{Size: 9}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
err := HookChangeCapacity(ctx, fs)
asserts.Equal(ErrInsufficientCapacity, err)
}
// 容量增加 成功
{
fs := &FileSystem{User: &model.User{
Model: gorm.Model{ID: 1},
Group: model.Group{MaxStorage: 1},
}}
newFile := fsctx.FileStream{Size: 10}
oldFile := model.File{Size: 9}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WithArgs(1, sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
err := HookChangeCapacity(ctx, fs)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
asserts.Equal(uint64(1), fs.User.Storage)
}
// 容量减少
{
fs := &FileSystem{User: &model.User{
Model: gorm.Model{ID: 1},
Storage: 1,
}}
newFile := fsctx.FileStream{Size: 9}
oldFile := model.File{Size: 10}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
err := HookChangeCapacity(ctx, fs)
asserts.NoError(err)
asserts.Equal(uint64(0), fs.User.Storage)
}
}
func TestHookCleanFileContent(t *testing.T) {
asserts := assert.New(t)
fs := &FileSystem{User: &model.User{
Model: gorm.Model{ID: 1},
}}
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "123/123")
file := &fsctx.FileStream{SavePath: "123/123"}
handlerMock := FileHeaderMock{}
handlerMock.On("Put", testMock.Anything, testMock.Anything, "123/123").Return(errors.New("error"))
handlerMock.On("Put", testMock.Anything, testMock.Anything).Return(errors.New("error"))
fs.Handler = handlerMock
err := HookCleanFileContent(ctx, fs)
err := HookCleanFileContent(context.Background(), fs, file)
asserts.Error(err)
handlerMock.AssertExpectations(t)
}
@ -395,14 +352,17 @@ func TestHookClearFileSize(t *testing.T) {
ctx := context.WithValue(
context.Background(),
fsctx.FileModelCtx,
model.File{Model: gorm.Model{ID: 1}},
model.File{Model: gorm.Model{ID: 1}, Size: 10},
)
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").
WithArgs(0, sqlmock.AnyArg(), 1).
mock.ExpectExec("UPDATE(.+)files(.+)").
WithArgs(0, sqlmock.AnyArg(), 1, 10).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)users(.+)").
WithArgs(10, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := HookClearFileSize(ctx, fs)
err := HookClearFileSize(ctx, fs, nil)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
@ -410,7 +370,7 @@ func TestHookClearFileSize(t *testing.T) {
// 上下文对象不存在
{
ctx := context.Background()
err := HookClearFileSize(ctx, fs)
err := HookClearFileSize(ctx, fs, nil)
asserts.Error(err)
}
@ -432,7 +392,7 @@ func TestHookUpdateSourceName(t *testing.T) {
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WithArgs("new.txt", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := HookUpdateSourceName(ctx, fs)
err := HookUpdateSourceName(ctx, fs, nil)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
@ -440,7 +400,7 @@ func TestHookUpdateSourceName(t *testing.T) {
// 上下文错误
{
ctx := context.Background()
err := HookUpdateSourceName(ctx, fs)
err := HookUpdateSourceName(ctx, fs, nil)
asserts.Error(err)
}
}
@ -457,41 +417,32 @@ func TestGenericAfterUpdate(t *testing.T) {
Model: gorm.Model{ID: 1},
PicInfo: "1,1",
}
newFile := fsctx.FileStream{Size: 10}
newFile := &fsctx.FileStream{Size: 10}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
handlerMock := FileHeaderMock{}
handlerMock.On("Delete", testMock.Anything, []string{"._thumb"}).Return([]string{}, nil)
fs.Handler = handlerMock
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").
WithArgs(10, sqlmock.AnyArg(), 1).
mock.ExpectExec("UPDATE(.+)files(.+)").
WithArgs(10, sqlmock.AnyArg(), 1, 0).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)users(.+)").
WithArgs(10, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := GenericAfterUpdate(ctx, fs)
err := GenericAfterUpdate(ctx, fs, newFile)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
// 新文件上下文不存在
{
originFile := model.File{
Model: gorm.Model{ID: 1},
PicInfo: "1,1",
}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
err := GenericAfterUpdate(ctx, fs)
asserts.Error(err)
}
// 原始文件上下文不存在
{
newFile := fsctx.FileStream{Size: 10}
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, newFile)
err := GenericAfterUpdate(ctx, fs)
newFile := &fsctx.FileStream{Size: 10}
ctx := context.Background()
err := GenericAfterUpdate(ctx, fs, newFile)
asserts.Error(err)
}
@ -502,91 +453,41 @@ func TestGenericAfterUpdate(t *testing.T) {
Model: gorm.Model{ID: 1},
PicInfo: "1,1",
}
newFile := fsctx.FileStream{Size: 10}
newFile := &fsctx.FileStream{Size: 10}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").
WithArgs(10, sqlmock.AnyArg(), 1).
WithArgs(10, sqlmock.AnyArg(), 1, 0).
WillReturnError(errors.New("error"))
mock.ExpectRollback()
err := GenericAfterUpdate(ctx, fs)
err := GenericAfterUpdate(ctx, fs, newFile)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
}
}
func TestHookSlaveUploadValidate(t *testing.T) {
asserts := assert.New(t)
conf.SystemConfig.Mode = "slave"
fs, err := NewAnonymousFileSystem()
conf.SystemConfig.Mode = "master"
asserts.NoError(err)
// 正常
{
policy := serializer.UploadPolicy{
SavePath: "",
MaxSize: 10,
AllowedExtension: nil,
}
file := fsctx.FileStream{Name: "1.txt", Size: 10}
ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
asserts.NoError(HookSlaveUploadValidate(ctx, fs))
}
// 尺寸太大
{
policy := serializer.UploadPolicy{
SavePath: "",
MaxSize: 10,
AllowedExtension: nil,
}
file := fsctx.FileStream{Name: "1.txt", Size: 11}
ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
asserts.Equal(ErrFileSizeTooBig, HookSlaveUploadValidate(ctx, fs))
}
// 文件名非法
{
policy := serializer.UploadPolicy{
SavePath: "",
MaxSize: 10,
AllowedExtension: nil,
}
file := fsctx.FileStream{Name: "/1.txt", Size: 10}
ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
asserts.Equal(ErrIllegalObjectName, HookSlaveUploadValidate(ctx, fs))
}
// 扩展名非法
{
policy := serializer.UploadPolicy{
SavePath: "",
MaxSize: 10,
AllowedExtension: []string{"jpg"},
}
file := fsctx.FileStream{Name: "1.txt", Size: 10}
ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
asserts.Equal(ErrFileExtensionNotAllowed, HookSlaveUploadValidate(ctx, fs))
func TestHookGenerateThumb(t *testing.T) {
a := assert.New(t)
mockHandler := &FileHeaderMock{}
fs := &FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
},
Handler: mockHandler,
Policy: &model.Policy{Type: "local"},
}
}
type ClientMock struct {
testMock.Mock
}
func (m ClientMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
args := m.Called(method, target, body, opts)
return args.Get(0).(*request.Response)
mockHandler.On("Delete", testMock.Anything, []string{"1.txt._thumb"}).Return([]string{}, nil)
a.NoError(HookGenerateThumb(context.Background(), fs, &fsctx.FileStream{
Model: &model.File{
SourceName: "1.txt",
},
}))
fs.Recycle()
mockHandler.AssertExpectations(t)
}
func TestSlaveAfterUpload(t *testing.T) {
@ -598,7 +499,7 @@ func TestSlaveAfterUpload(t *testing.T) {
// 成功
{
clientMock := ClientMock{}
clientMock := requestmock.RequestMock{}
clientMock.On(
"Request",
"POST",
@ -613,19 +514,28 @@ func TestSlaveAfterUpload(t *testing.T) {
},
})
request.GeneralClient = clientMock
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{
file := &fsctx.FileStream{
Size: 10,
VirtualPath: "/my",
Name: "test.txt",
})
ctx = context.WithValue(ctx, fsctx.UploadPolicyCtx, serializer.UploadPolicy{
CallbackURL: "http://test/callbakc",
})
ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/not_exist")
err := SlaveAfterUpload(ctx, fs)
SavePath: "/not_exist",
}
err := SlaveAfterUpload(&serializer.UploadSession{Callback: "http://test/callbakc"})(context.Background(), fs, file)
clientMock.AssertExpectations(t)
asserts.NoError(err)
}
// 跳过回调
{
file := &fsctx.FileStream{
Size: 10,
VirtualPath: "/my",
Name: "test.txt",
SavePath: "/not_exist",
}
err := SlaveAfterUpload(&serializer.UploadSession{})(context.Background(), fs, file)
asserts.NoError(err)
}
}
func TestFileSystem_CleanHooks(t *testing.T) {
@ -663,7 +573,7 @@ func TestHookCancelContext(t *testing.T) {
// empty ctx
{
asserts.NoError(HookCancelContext(ctx, fs))
asserts.NoError(HookCancelContext(ctx, fs, nil))
select {
case <-ctx.Done():
t.Errorf("Channel should not be closed")
@ -675,62 +585,99 @@ func TestHookCancelContext(t *testing.T) {
// with cancel ctx
{
ctx = context.WithValue(ctx, fsctx.CancelFuncCtx, cancel)
asserts.NoError(HookCancelContext(ctx, fs))
asserts.NoError(HookCancelContext(ctx, fs, nil))
_, ok := <-ctx.Done()
asserts.False(ok)
}
}
func TestHookGiveBackCapacity(t *testing.T) {
asserts := assert.New(t)
fs := &FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
Storage: 10,
func TestHookClearFileHeaderSize(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{}
file := &fsctx.FileStream{Size: 10}
a.NoError(HookClearFileHeaderSize(context.Background(), fs, file))
a.EqualValues(0, file.Size)
}
func TestHookTruncateFileTo(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{}
file := &fsctx.FileStream{}
a.NoError(HookTruncateFileTo(0)(context.Background(), fs, file))
fs.Handler = local.Driver{}
a.Error(HookTruncateFileTo(0)(context.Background(), fs, file))
}
func TestHookChunkUploaded(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{}
file := &fsctx.FileStream{
AppendStart: 10,
Size: 10,
Model: &model.File{
Model: gorm.Model{ID: 1},
},
}
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{Size: 1})
// without once limit
{
asserts.NoError(HookGiveBackCapacity(ctx, fs))
asserts.EqualValues(9, fs.User.Storage)
asserts.NoError(HookGiveBackCapacity(ctx, fs))
asserts.EqualValues(8, fs.User.Storage)
}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(20, sqlmock.AnyArg(), 1, 0).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)users(.+)").
WithArgs(20, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
a.NoError(HookChunkUploaded(context.Background(), fs, file))
a.NoError(mock.ExpectationsWereMet())
}
// with once limit
{
ctx = context.WithValue(ctx, fsctx.ValidateCapacityOnceCtx, &sync.Once{})
asserts.NoError(HookGiveBackCapacity(ctx, fs))
asserts.EqualValues(7, fs.User.Storage)
asserts.NoError(HookGiveBackCapacity(ctx, fs))
asserts.EqualValues(7, fs.User.Storage)
func TestHookChunkUploadFailed(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{}
file := &fsctx.FileStream{
AppendStart: 10,
Size: 10,
Model: &model.File{
Model: gorm.Model{ID: 1},
},
}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(10, sqlmock.AnyArg(), 1, 0).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)users(.+)").
WithArgs(10, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
a.NoError(HookChunkUploadFailed(context.Background(), fs, file))
a.NoError(mock.ExpectationsWereMet())
}
func TestHookValidateCapacityWithoutIncrease(t *testing.T) {
func TestHookPopPlaceholderToFile(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
Storage: 10,
Group: model.Group{},
fs := &FileSystem{}
file := &fsctx.FileStream{
Model: &model.File{
Model: gorm.Model{ID: 1},
},
}
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{Size: 1})
// not enough
{
fs.User.Group.MaxStorage = 10
a.Error(HookValidateCapacity(ctx, fs))
a.EqualValues(10, fs.User.Storage)
}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
a.NoError(HookPopPlaceholderToFile("1,1")(context.Background(), fs, file))
a.NoError(mock.ExpectationsWereMet())
}
// enough
{
fs.User.Group.MaxStorage = 11
a.NoError(HookValidateCapacity(ctx, fs))
a.EqualValues(10, fs.User.Storage)
func TestHookDeleteUploadSession(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{}
file := &fsctx.FileStream{
Model: &model.File{
Model: gorm.Model{ID: 1},
},
}
cache.Set(UploadSessionCachePrefix+"TestHookDeleteUploadSession", "", 0)
a.NoError(HookDeleteUploadSession("TestHookDeleteUploadSession")(context.Background(), fs, file))
_, ok := cache.Get(UploadSessionCachePrefix + "TestHookDeleteUploadSession")
a.False(ok)
}

17
pkg/filesystem/image.go
Unescape View File

@ -3,7 +3,6 @@ package filesystem
import (
"context"
"fmt"
"strconv"
"sync"
"runtime"
@ -65,7 +64,7 @@ type Pool struct {
// Init 初始化任务池
func getThumbWorker() *Pool {
once.Do(func() {
maxWorker := conf.ThumbConfig.MaxTaskCount
maxWorker := model.GetIntSetting("thumb_max_task_count", -1)
if maxWorker <= 0 {
maxWorker = runtime.GOMAXPROCS(0)
}
@ -118,9 +117,9 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
// 生成缩略图
image.GetThumb(fs.GenerateThumbnailSize(w, h))
// 保存到文件
err = image.Save(util.RelativePath(file.SourceName + conf.ThumbConfig.FileSuffix))
err = image.Save(util.RelativePath(file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")))
image = nil
if conf.ThumbConfig.GCAfterGen {
if model.IsTrueVal(model.GetSettingByName("thumb_gc_after_gen")) {
util.Log().Debug("GenerateThumbnail runtime.GC")
runtime.GC()
}
@ -139,17 +138,11 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
// 失败时删除缩略图文件
if err != nil {
_, _ = fs.Handler.Delete(newCtx, []string{file.SourceName + conf.ThumbConfig.FileSuffix})
_, _ = fs.Handler.Delete(newCtx, []string{file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")})
}
}
// GenerateThumbnailSize 获取要生成的缩略图的尺寸
func (fs *FileSystem) GenerateThumbnailSize(w, h int) (uint, uint) {
if conf.SystemConfig.Mode == "master" {
options := model.GetSettingByNames("thumb_width", "thumb_height")
w, _ := strconv.ParseUint(options["thumb_width"], 10, 32)
h, _ := strconv.ParseUint(options["thumb_height"], 10, 32)
return uint(w), uint(h)
}
return conf.ThumbConfig.MaxWidth, conf.ThumbConfig.MaxHeight
return uint(model.GetIntSetting("thumb_width", 400)), uint(model.GetIntSetting("thumb_width", 300))
}

27
pkg/filesystem/image_test.go
Unescape View File

@ -2,13 +2,15 @@ package filesystem
import (
"context"
"testing"
"errors"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/stretchr/testify/assert"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
testMock "github.com/stretchr/testify/mock"
"testing"
"github.com/stretchr/testify/assert"
)
func TestFileSystem_GetThumb(t *testing.T) {
@ -47,3 +49,22 @@ func TestFileSystem_ThumbWorker(t *testing.T) {
getThumbWorker().releaseWorker()
})
}
func TestFileSystem_GenerateThumbnail(t *testing.T) {
fs := &FileSystem{User: &model.User{}}
// 无法生成缩略图
{
fs.SetTargetFile(&[]model.File{{}})
fs.GenerateThumbnail(context.Background(), &model.File{})
}
// 无法获取文件数据
{
testHandller := new(FileHeaderMock)
testHandller.On("Get", testMock.Anything, "").Return(request.NopRSCloser{}, errors.New("error"))
fs.Handler = testHandller
fs.GenerateThumbnail(context.Background(), &model.File{Name: "test.png"})
testHandller.AssertExpectations(t)
}
}

9
pkg/filesystem/manage.go
Unescape View File

@ -221,6 +221,15 @@ func (fs *FileSystem) ListDeleteDirs(ctx context.Context, ids []uint) error {
if err != nil {
return ErrDBListObjects.WithError(err)
}
// 忽略根目录
for i := 0; i < len(folders); i++ {
if folders[i].ParentID == nil {
folders = append(folders[:i], folders[i+1:]...)
break
}
}
fs.SetTargetDir(&folders)
// 检索目录下的子文件

98
pkg/filesystem/manage_test.go
Unescape View File

@ -3,6 +3,8 @@ package filesystem
import (
"context"
"errors"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
testMock "github.com/stretchr/testify/mock"
"os"
"testing"
@ -11,12 +13,10 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
)
func TestFileSystem_ListPhysical(t *testing.T) {
@ -296,12 +296,12 @@ func TestFileSystem_ListDeleteDirs(t *testing.T) {
{
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
sqlmock.NewRows([]string{"id"}).
AddRow(1).
AddRow(2).
AddRow(3),
sqlmock.NewRows([]string{"id", "parent_id"}).
AddRow(1, 0).
AddRow(2, 0).
AddRow(3, 0),
)
mock.ExpectQuery("SELECT(.+)").
mock.ExpectQuery("SELECT(.+)files(.+)").
WithArgs(1, 2, 3).
WillReturnRows(
sqlmock.NewRows([]string{"id", "name"}).
@ -316,21 +316,47 @@ func TestFileSystem_ListDeleteDirs(t *testing.T) {
asserts.NoError(mock.ExpectationsWereMet())
}
// 成功,忽略根目录
{
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
sqlmock.NewRows([]string{"id", "parent_id"}).
AddRow(1, 0).
AddRow(2, nil).
AddRow(3, 0),
)
mock.ExpectQuery("SELECT(.+)files(.+)").
WithArgs(1, 3).
WillReturnRows(
sqlmock.NewRows([]string{"id", "name"}).
AddRow(4, "1.txt").
AddRow(5, "2.txt").
AddRow(6, "3.txt"),
)
fs.CleanTargets()
err := fs.ListDeleteDirs(context.Background(), []uint{1})
asserts.NoError(err)
asserts.Len(fs.FileTarget, 3)
asserts.Len(fs.DirTarget, 2)
asserts.NoError(mock.ExpectationsWereMet())
}
// 检索文件发生错误
{
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
sqlmock.NewRows([]string{"id"}).
AddRow(1).
AddRow(2).
AddRow(3),
sqlmock.NewRows([]string{"id", "parent_id"}).
AddRow(1, 0).
AddRow(2, 0).
AddRow(3, 0),
)
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 2, 3).
WillReturnError(errors.New("error"))
fs.CleanTargets()
err := fs.ListDeleteDirs(context.Background(), []uint{1})
asserts.Error(err)
asserts.Len(fs.DirTarget, 6)
asserts.Len(fs.DirTarget, 3)
asserts.NoError(mock.ExpectationsWereMet())
}
// 检索目录发生错误
@ -349,7 +375,7 @@ func TestFileSystem_Delete(t *testing.T) {
cache.Set("pack_size_1", uint64(0), 0)
fs := &FileSystem{User: &model.User{
Model: gorm.Model{
ID: 1,
ID: 0,
},
Storage: 3,
Group: model.Group{MaxStorage: 3},
@ -361,10 +387,10 @@ func TestFileSystem_Delete(t *testing.T) {
fs.CleanTargets()
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
sqlmock.NewRows([]string{"id"}).
AddRow(1).
AddRow(2).
AddRow(3),
sqlmock.NewRows([]string{"id", "parent_id"}).
AddRow(1, 0).
AddRow(2, 0).
AddRow(3, 0),
)
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 2, 3).
@ -380,18 +406,16 @@ func TestFileSystem_Delete(t *testing.T) {
// 删除文件记录
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(0, 3))
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec("UPDATE(.+)users(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
// 删除对应分享
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)shares").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
// 归还容量
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
// 删除目录
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
@ -407,7 +431,6 @@ func TestFileSystem_Delete(t *testing.T) {
fs.DirTarget = []model.Folder{}
err := fs.Delete(ctx, []uint{1}, []uint{1}, true)
asserts.NoError(err)
asserts.Equal(uint64(0), fs.User.Storage)
}
//全部成功
{
@ -419,10 +442,10 @@ func TestFileSystem_Delete(t *testing.T) {
asserts.NoError(err)
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
sqlmock.NewRows([]string{"id"}).
AddRow(1).
AddRow(2).
AddRow(3),
sqlmock.NewRows([]string{"id", "parent_id"}).
AddRow(1, 0).
AddRow(2, 0).
AddRow(3, 0),
)
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 2, 3).
@ -438,18 +461,16 @@ func TestFileSystem_Delete(t *testing.T) {
// 删除文件记录
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(0, 3))
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec("UPDATE(.+)users(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
// 删除对应分享
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)shares").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
// 归还容量
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
// 删除目录
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
@ -465,7 +486,6 @@ func TestFileSystem_Delete(t *testing.T) {
fs.DirTarget = []model.Folder{}
err = fs.Delete(ctx, []uint{1}, []uint{1}, false)
asserts.NoError(err)
asserts.Equal(uint64(0), fs.User.Storage)
}
}
@ -573,7 +593,9 @@ func TestFileSystem_Rename(t *testing.T) {
Model: gorm.Model{
ID: 1,
},
}}
},
Policy: &model.Policy{},
}
ctx := context.Background()
// 重命名文件 成功
@ -683,7 +705,7 @@ func TestFileSystem_Rename(t *testing.T) {
// 新名字是文件，扩展名不合法
{
fs.User.Policy.OptionsSerialized.FileType = []string{"txt"}
fs.Policy.OptionsSerialized.FileType = []string{"txt"}
err := fs.Rename(ctx, []uint{}, []uint{10}, "1.jpg")
asserts.Error(err)
asserts.Equal(ErrIllegalObjectName, err)
@ -691,7 +713,7 @@ func TestFileSystem_Rename(t *testing.T) {
// 新名字是目录，不应该检测扩展名
{
fs.User.Policy.OptionsSerialized.FileType = []string{"txt"}
fs.Policy.OptionsSerialized.FileType = []string{"txt"}
mock.ExpectQuery("SELECT(.+)folders(.+)").
WithArgs(10, 1).
WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))

41
pkg/filesystem/upload.go
Unescape View File

@ -91,40 +91,17 @@ func (fs *FileSystem) Upload(ctx context.Context, file *fsctx.FileStream) (err e
// TODO 完善测试
func (fs *FileSystem) GenerateSavePath(ctx context.Context, file fsctx.FileHeader) string {
fileInfo := file.Info()
if fs.User.Model.ID != 0 {
return path.Join(
fs.Policy.GeneratePath(
fs.User.Model.ID,
fileInfo.VirtualPath,
),
fs.Policy.GenerateFileName(
fs.User.Model.ID,
fileInfo.FileName,
),
)
}
// 匿名文件系统尝试根据上下文中的上传策略生成路径
var anonymousPolicy model.Policy
if policy, ok := ctx.Value(fsctx.UploadPolicyCtx).(serializer.UploadPolicy); ok {
anonymousPolicy = model.Policy{
Type: "remote",
AutoRename: policy.AutoRename,
DirNameRule: policy.SavePath,
FileNameRule: policy.FileName,
}
}
return path.Join(
anonymousPolicy.GeneratePath(
0,
"",
fs.Policy.GeneratePath(
fs.User.Model.ID,
fileInfo.VirtualPath,
),
anonymousPolicy.GenerateFileName(
0,
fs.Policy.GenerateFileName(
fs.User.Model.ID,
fileInfo.FileName,
),
)
}
// CancelUpload 监测客户端取消上传
@ -174,9 +151,6 @@ func (fs *FileSystem) CreateUploadSession(ctx context.Context, file *fsctx.FileS
fs.Use("BeforeUpload", HookValidateFile)
fs.Use("BeforeUpload", HookValidateCapacity)
if !fs.Policy.IsUploadPlaceholderWithSize() {
fs.Use("AfterUpload", HookClearFileHeaderSize)
}
// 验证文件规格
if err := fs.Upload(ctx, file); err != nil {
@ -202,6 +176,9 @@ func (fs *FileSystem) CreateUploadSession(ctx context.Context, file *fsctx.FileS
}
// 创建占位符
if !fs.Policy.IsUploadPlaceholderWithSize() {
fs.Use("AfterUpload", HookClearFileHeaderSize)
}
fs.Use("AfterUpload", GenericAfterUpload)
if err := fs.Upload(ctx, file); err != nil {
return nil, err

159
pkg/filesystem/upload_test.go
Unescape View File

@ -3,14 +3,7 @@ package filesystem
import (
"context"
"errors"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"github.com/DATA-DOG/go-sqlmock"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
@ -20,12 +13,33 @@ import (
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
)
type FileHeaderMock struct {
testMock.Mock
}
func (m FileHeaderMock) Put(ctx context.Context, file fsctx.FileHeader) error {
args := m.Called(ctx, file)
return args.Error(0)
}
func (m FileHeaderMock) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
args := m.Called(ctx, ttl, uploadSession, file)
return args.Get(0).(*serializer.UploadCredential), args.Error(1)
}
func (m FileHeaderMock) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
args := m.Called(ctx, uploadSession)
return args.Error(0)
}
func (m FileHeaderMock) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
args := m.Called(ctx, path, recursive)
return args.Get(0).([]response.Object), args.Error(1)
@ -36,11 +50,6 @@ func (m FileHeaderMock) Get(ctx context.Context, path string) (response.RSCloser
return args.Get(0).(response.RSCloser), args.Error(1)
}
func (m FileHeaderMock) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
args := m.Called(ctx, file, dst)
return args.Error(0)
}
func (m FileHeaderMock) Delete(ctx context.Context, files []string) ([]string, error) {
args := m.Called(ctx, files)
return args.Get(0).([]string), args.Error(1)
@ -56,11 +65,6 @@ func (m FileHeaderMock) Source(ctx context.Context, path string, url url.URL, ex
return args.Get(0).(string), args.Error(1)
}
func (m FileHeaderMock) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
args := m.Called(ctx, ttl, uploadSession)
return args.Get(0).(serializer.UploadCredential), args.Error(1)
}
func TestFileSystem_Upload(t *testing.T) {
asserts := assert.New(t)
@ -73,10 +77,10 @@ func TestFileSystem_Upload(t *testing.T) {
Model: gorm.Model{
ID: 1,
},
Policy: model.Policy{
AutoRename: false,
DirNameRule: "{path}",
},
},
Policy: &model.Policy{
AutoRename: false,
DirNameRule: "{path}",
},
}
ctx, cancel := context.WithCancel(context.Background())
@ -84,7 +88,7 @@ func TestFileSystem_Upload(t *testing.T) {
c.Request, _ = http.NewRequest("POST", "/", nil)
ctx = context.WithValue(ctx, fsctx.GinCtx, c)
cancel()
file := fsctx.FileStream{
file := &fsctx.FileStream{
Size: 5,
VirtualPath: "/",
Name: "1.txt",
@ -94,17 +98,17 @@ func TestFileSystem_Upload(t *testing.T) {
// 正常，上下文已指定源文件
testHandler = new(FileHeaderMock)
testHandler.On("Put", testMock.Anything, testMock.Anything, "123/123.txt").Return(nil)
testHandler.On("Put", testMock.Anything, testMock.Anything).Return(nil)
fs = &FileSystem{
Handler: testHandler,
User: &model.User{
Model: gorm.Model{
ID: 1,
},
Policy: model.Policy{
AutoRename: false,
DirNameRule: "{path}",
},
},
Policy: &model.Policy{
AutoRename: false,
DirNameRule: "{path}",
},
}
ctx, cancel = context.WithCancel(context.Background())
@ -113,7 +117,7 @@ func TestFileSystem_Upload(t *testing.T) {
ctx = context.WithValue(ctx, fsctx.GinCtx, c)
ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{SourceName: "123/123.txt"})
cancel()
file = fsctx.FileStream{
file = &fsctx.FileStream{
Size: 5,
VirtualPath: "/",
Name: "1.txt",
@ -123,7 +127,7 @@ func TestFileSystem_Upload(t *testing.T) {
asserts.NoError(err)
// BeforeUpload 返回错误
fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem) error {
fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
return errors.New("error")
})
err = fs.Upload(ctx, file)
@ -133,7 +137,7 @@ func TestFileSystem_Upload(t *testing.T) {
// 上传文件失败
testHandler2 := new(FileHeaderMock)
testHandler2.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(errors.New("error"))
testHandler2.On("Put", testMock.Anything, testMock.Anything).Return(errors.New("error"))
fs.Handler = testHandler2
err = fs.Upload(ctx, file)
asserts.Error(err)
@ -141,12 +145,12 @@ func TestFileSystem_Upload(t *testing.T) {
// AfterUpload失败
testHandler3 := new(FileHeaderMock)
testHandler3.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(nil)
testHandler3.On("Put", testMock.Anything, testMock.Anything).Return(nil)
fs.Handler = testHandler3
fs.Use("AfterUpload", func(ctx context.Context, fs *FileSystem) error {
fs.Use("AfterUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
return errors.New("error")
})
fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem) error {
fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
return errors.New("error")
})
err = fs.Upload(ctx, file)
@ -155,43 +159,39 @@ func TestFileSystem_Upload(t *testing.T) {
}
func TestFileSystem_GenerateSavePath_Anonymous(t *testing.T) {
asserts := assert.New(t)
fs := FileSystem{User: &model.User{}}
ctx := context.WithValue(
context.Background(),
fsctx.UploadPolicyCtx,
serializer.UploadPolicy{
SavePath: "{randomkey16}",
AutoRename: false,
},
)
savePath := fs.GenerateSavePath(ctx, fsctx.FileStream{
Name: "test.test",
})
asserts.Len(savePath, 26)
asserts.Contains(savePath, "test.test")
}
func TestFileSystem_GetUploadToken(t *testing.T) {
asserts := assert.New(t)
fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}}
fs := FileSystem{
User: &model.User{Model: gorm.Model{ID: 1}},
Policy: &model.Policy{},
}
ctx := context.Background()
// 成功
{
cache.SetSettings(map[string]string{
"upload_credential_timeout": "10",
"upload_session_timeout": "10",
"upload_session_timeout": "10",
}, "setting_")
testHandler := new(FileHeaderMock)
testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{Token: "test"}, nil)
testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything, testMock.Anything).Return(&serializer.UploadCredential{Credential: "test"}, nil)
fs.Handler = testHandler
res, err := fs.CreateUploadSession(ctx, "/", 10, "123")
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found"))
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
res, err := fs.CreateUploadSession(ctx, &fsctx.FileStream{
Size: 0,
Name: "file",
VirtualPath: "/",
})
asserts.NoError(mock.ExpectationsWereMet())
testHandler.AssertExpectations(t)
asserts.NoError(err)
asserts.Equal("test", res.Token)
asserts.Equal("test", res.Credential)
}
// 无法获取上传凭证
@ -201,9 +201,22 @@ func TestFileSystem_GetUploadToken(t *testing.T) {
"upload_session_timeout": "10",
}, "setting_")
testHandler := new(FileHeaderMock)
testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{}, errors.New("error"))
testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything, testMock.Anything).Return(&serializer.UploadCredential{}, errors.New("error"))
fs.Handler = testHandler
_, err := fs.CreateUploadSession(ctx, "/", 10, "123")
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found"))
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
_, err := fs.CreateUploadSession(ctx, &fsctx.FileStream{
Size: 0,
Name: "file",
VirtualPath: "/",
})
asserts.NoError(mock.ExpectationsWereMet())
testHandler.AssertExpectations(t)
asserts.Error(err)
}
@ -211,27 +224,41 @@ func TestFileSystem_GetUploadToken(t *testing.T) {
func TestFileSystem_UploadFromStream(t *testing.T) {
asserts := assert.New(t)
fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}}
fs := FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
Policy: model.Policy{Type: "mock"},
},
Policy: &model.Policy{Type: "mock"},
}
ctx := context.Background()
err := fs.UploadFromStream(ctx, ioutil.NopCloser(strings.NewReader("123")), "/1.txt", 1)
err := fs.UploadFromStream(ctx, &fsctx.FileStream{
File: ioutil.NopCloser(strings.NewReader("123")),
}, true)
asserts.Error(err)
}
func TestFileSystem_UploadFromPath(t *testing.T) {
asserts := assert.New(t)
fs := FileSystem{User: &model.User{Policy: model.Policy{Type: "mock"}, Model: gorm.Model{ID: 1}}}
fs := FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
Policy: model.Policy{Type: "mock"},
},
Policy: &model.Policy{Type: "mock"},
}
ctx := context.Background()
// 文件不存在
{
err := fs.UploadFromPath(ctx, "test/not_exist", "/", true)
err := fs.UploadFromPath(ctx, "test/not_exist", "/", fsctx.Overwrite)
asserts.Error(err)
}
// 文存在,上传失败
{
err := fs.UploadFromPath(ctx, "tests/test.zip", "/", true)
err := fs.UploadFromPath(ctx, "tests/test.zip", "/", fsctx.Overwrite)
asserts.Error(err)
}
}

22
pkg/filesystem/validator_test.go
Unescape View File

@ -68,10 +68,9 @@ func TestFileSystem_ValidateFileSize(t *testing.T) {
asserts := assert.New(t)
ctx := context.Background()
fs := FileSystem{
User: &model.User{
Policy: model.Policy{
MaxSize: 10,
},
User: &model.User{},
Policy: &model.Policy{
MaxSize: 10,
},
}
@ -80,7 +79,7 @@ func TestFileSystem_ValidateFileSize(t *testing.T) {
asserts.False(fs.ValidateFileSize(ctx, 11))
// 无限制
fs.User.Policy.MaxSize = 0
fs.Policy.MaxSize = 0
asserts.True(fs.ValidateFileSize(ctx, 11))
}
@ -88,11 +87,10 @@ func TestFileSystem_ValidateExtension(t *testing.T) {
asserts := assert.New(t)
ctx := context.Background()
fs := FileSystem{
User: &model.User{
Policy: model.Policy{
OptionsSerialized: model.PolicyOption{
FileType: nil,
},
User: &model.User{},
Policy: &model.Policy{
OptionsSerialized: model.PolicyOption{
FileType: nil,
},
},
}
@ -100,11 +98,11 @@ func TestFileSystem_ValidateExtension(t *testing.T) {
asserts.True(fs.ValidateExtension(ctx, "1"))
asserts.True(fs.ValidateExtension(ctx, "1.txt"))
fs.User.Policy.OptionsSerialized.FileType = []string{}
fs.Policy.OptionsSerialized.FileType = []string{}
asserts.True(fs.ValidateExtension(ctx, "1"))
asserts.True(fs.ValidateExtension(ctx, "1.txt"))
fs.User.Policy.OptionsSerialized.FileType = []string{"txt", "jpg"}
fs.Policy.OptionsSerialized.FileType = []string{"txt", "jpg"}
asserts.False(fs.ValidateExtension(ctx, "1"))
asserts.False(fs.ValidateExtension(ctx, "1.jpg.png"))
asserts.True(fs.ValidateExtension(ctx, "1.txt"))

10
pkg/mocks/mocks.go
Unescape View File

@ -7,11 +7,9 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/balancer"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/task"
testMock "github.com/stretchr/testify/mock"
"io"
)
type NodePoolMock struct {
@ -151,11 +149,3 @@ func (t TaskPoolMock) Add(num int) {
func (t TaskPoolMock) Submit(job task.Job) {
t.Called(job)
}
type RequestMock struct {
testMock.Mock
}
func (r RequestMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
return r.Called(method, target, body, opts).Get(0).(*request.Response)
}

32
pkg/mocks/remoteclientmock/mock.go
Unescape View File

@ -0,0 +1,32 @@
package remoteclientmock
import (
"context"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/stretchr/testify/mock"
)
type RemoteClientMock struct {
mock.Mock
}
func (r *RemoteClientMock) CreateUploadSession(ctx context.Context, session *serializer.UploadSession, ttl int64) error {
return r.Called(ctx, session, ttl).Error(0)
}
func (r *RemoteClientMock) GetUploadURL(ttl int64, sessionID string) (string, string, error) {
args := r.Called(ttl, sessionID)
return args.String(0), args.String(1), args.Error(2)
}
func (r *RemoteClientMock) Upload(ctx context.Context, file fsctx.FileHeader) error {
args := r.Called(ctx, file)
return args.Error(0)
}
func (r *RemoteClientMock) DeleteUploadSession(ctx context.Context, sessionID string) error {
args := r.Called(ctx, sessionID)
return args.Error(0)
}

15
pkg/mocks/requestmock/request.go
Unescape View File

@ -0,0 +1,15 @@
package requestmock
import (
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/stretchr/testify/mock"
"io"
)
type RequestMock struct {
mock.Mock
}
func (r RequestMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
return r.Called(method, target, body, opts).Get(0).(*request.Response)
}

2
pkg/request/request_test.go
Unescape View File

@ -82,6 +82,8 @@ func TestHTTPClient_Request(t *testing.T) {
WithTimeout(time.Duration(1)*time.Microsecond),
WithCredential(auth.HMACAuth{SecretKey: []byte("123")}, 10),
WithContext(context.Background()),
WithoutHeader([]string{"s s", "s s"}),
WithMasterMeta(),
)
asserts.Error(resp.Err)
asserts.Nil(resp.Response)

15
pkg/serializer/explorer_test.go
Unescape View File

@ -0,0 +1,15 @@
package serializer
import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/stretchr/testify/assert"
"testing"
)
func TestBuildObjectList(t *testing.T) {
a := assert.New(t)
res := BuildObjectList(1, []Object{{}, {}}, &model.Policy{})
a.NotEmpty(res.Parent)
a.NotNil(res.Policy)
a.Len(res.Objects, 2)
}

59
pkg/serializer/upload.go
Unescape View File

@ -1,9 +1,7 @@
package serializer
import (
"encoding/base64"
"encoding/gob"
"encoding/json"
model "github.com/cloudreve/Cloudreve/v3/models"
"time"
)
@ -20,19 +18,18 @@ type UploadPolicy struct {
// UploadCredential 返回给客户端的上传凭证
type UploadCredential struct {
SessionID string `json:"sessionID"`
ChunkSize uint64 `json:"chunkSize"` // 分块大小，0 为部分快
Expires int64 `json:"expires"` // 上传凭证过期时间， Unix 时间戳
UploadURLs []string `json:"uploadURLs"`
Credential string `json:"credential"`
Token string `json:"token"`
Policy string `json:"policy"`
Path string `json:"path"` // 存储路径
AccessKey string `json:"ak"`
KeyTime string `json:"key_time,omitempty"` // COS用有效期
Callback string `json:"callback,omitempty"` // 回调地址
Key string `json:"key,omitempty"` // 文件标识符，通常为回调key
SessionID string `json:"sessionID"`
ChunkSize uint64 `json:"chunkSize"` // 分块大小，0 为部分快
Expires int64 `json:"expires"` // 上传凭证过期时间， Unix 时间戳
UploadURLs []string `json:"uploadURLs,omitempty"`
Credential string `json:"credential,omitempty"`
UploadID string `json:"uploadID,omitempty"`
Callback string `json:"callback,omitempty"` // 回调地址
Path string `json:"path,omitempty"` // 存储路径
AccessKey string `json:"ak,omitempty"`
KeyTime string `json:"keyTime,omitempty"` // COS用有效期
Policy string `json:"policy,omitempty"`
CompleteURL string `json:"completeURL,omitempty"`
}
// UploadSession 上传会话
@ -47,6 +44,9 @@ type UploadSession struct {
Policy model.Policy
Callback string // 回调 URL 地址
CallbackSecret string // 回调 URL
UploadURL string
UploadID string
Credential string
}
// UploadCallback 上传回调正文
@ -62,32 +62,3 @@ type GeneralUploadCallbackFailed struct {
func init() {
gob.Register(UploadSession{})
}
// DecodeUploadPolicy 反序列化Header中携带的上传策略
func DecodeUploadPolicy(raw string) (*UploadPolicy, error) {
var res UploadPolicy
rawJSON, err := base64.StdEncoding.DecodeString(raw)
if err != nil {
return nil, err
}
err = json.Unmarshal(rawJSON, &res)
if err != nil {
return nil, err
}
return &res, err
}
// EncodeUploadPolicy 序列化Header中携带的上传策略
func (policy *UploadPolicy) EncodeUploadPolicy() (string, error) {
jsonRes, err := json.Marshal(policy)
if err != nil {
return "", err
}
res := base64.StdEncoding.EncodeToString(jsonRes)
return res, nil
}

63
pkg/serializer/upload_test.go
Unescape View File

@ -1,63 +0,0 @@
package serializer
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestDecodeUploadPolicy(t *testing.T) {
asserts := assert.New(t)
testCases := []struct {
input string
expectError bool
expectNil bool
expectRes *UploadPolicy
}{
{
"错误的base64字符",
true,
true,
&UploadPolicy{},
},
{
"6ZSZ6K+v55qESlNPTuWtl+espg==",
true,
true,
&UploadPolicy{},
},
{
"e30=",
false,
false,
&UploadPolicy{},
},
{
"eyJjYWxsYmFja191cmwiOiJ0ZXN0In0=",
false,
false,
&UploadPolicy{CallbackURL: "test"},
},
}
for _, testCase := range testCases {
res, err := DecodeUploadPolicy(testCase.input)
if testCase.expectError {
asserts.Error(err)
}
if testCase.expectNil {
asserts.Nil(res)
}
if !testCase.expectNil {
asserts.Equal(testCase.expectRes, res)
}
}
}
func TestUploadPolicy_EncodeUploadPolicy(t *testing.T) {
asserts := assert.New(t)
testPolicy := UploadPolicy{}
res, err := testPolicy.EncodeUploadPolicy()
asserts.NoError(err)
asserts.NotEmpty(res)
}

12
pkg/serializer/user_test.go
Unescape View File

@ -29,23 +29,19 @@ func TestMain(m *testing.M) {
func TestBuildUser(t *testing.T) {
asserts := assert.New(t)
user := model.User{
Policy: model.Policy{MaxSize: 1024 * 1024},
}
user := model.User{}
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}))
res := BuildUser(user)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Equal("1.00mb", res.Policy.MaxSize)
asserts.NotNil(res)
}
func TestBuildUserResponse(t *testing.T) {
asserts := assert.New(t)
user := model.User{
Policy: model.Policy{MaxSize: 1024 * 1024},
}
user := model.User{}
res := BuildUserResponse(user)
asserts.Equal("1.00mb", res.Data.(User).Policy.MaxSize)
asserts.NotNil(res)
}
func TestBuildUserStorageResponse(t *testing.T) {

1
pkg/task/import_test.go
Unescape View File

@ -182,6 +182,7 @@ func TestImportTask_Do(t *testing.T) {
// 插入文件记录
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(2, 1))
mock.ExpectExec("UPDATE(.+)users(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(2, 1))
mock.ExpectCommit()
task.Do()

1
pkg/task/tranfer.go
Unescape View File

@ -117,7 +117,6 @@ func (job *TransferTask) Do() {
}
// 切换为从机节点处理上传
fs.SetPolicyFromPath(path.Dir(dst))
fs.SwitchToSlaveHandler(node)
err = fs.UploadFromStream(context.Background(), &fsctx.FileStream{
File: nil,

5
pkg/thumb/image.go
Unescape View File

@ -12,7 +12,6 @@ import (
"strings"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
//"github.com/nfnt/resize"
@ -78,11 +77,11 @@ func (image *Thumb) Save(path string) (err error) {
return err
}
defer out.Close()
switch conf.ThumbConfig.EncodeMethod {
switch model.GetSettingByNameWithDefault("thumb_encode_method", "jpg") {
case "png":
err = png.Encode(out, image.src)
default:
err = jpeg.Encode(out, image.src, &jpeg.Options{Quality: conf.ThumbConfig.EncodeQuality})
err = jpeg.Encode(out, image.src, &jpeg.Options{Quality: model.GetIntSetting("thumb_encode_quality", 85)})
}
return err

1
pkg/util/path.go
Unescape View File

@ -56,3 +56,4 @@ func RelativePath(name string) string {
e, _ := os.Executable()
return filepath.Join(filepath.Dir(e), name)
}

4
pkg/webdav/internal/xml/xml.go
Unescape View File

@ -1040,7 +1040,7 @@ Input:
d.buf.WriteByte(';')
n, err := strconv.ParseUint(s, base, 64)
if err == nil && n <= unicode.MaxRune {
text = string(n)
text = string(rune(n))
haveText = true
}
}
@ -1063,7 +1063,7 @@ Input:
if isName(name) {
s := string(name)
if r, ok := entity[s]; ok {
text = string(r)
text = string(rune(r))
haveText = true
} else if d.Entity != nil {
text, haveText = d.Entity[s]

1
routers/controllers/callback.go
Unescape View File

@ -112,7 +112,6 @@ func COSCallback(c *gin.Context) {
// S3Callback S3上传完成客户端回调
func S3Callback(c *gin.Context) {
c.Header("Access-Control-Allow-Origin", "*")
var callbackBody callback.S3Callback
if err := c.ShouldBindQuery(&callbackBody); err == nil {
res := callbackBody.PreProcess(c)

4
routers/controllers/file.go
Unescape View File

@ -5,7 +5,7 @@ import (
"fmt"
"net/http"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
@ -174,7 +174,7 @@ func Thumb(c *gin.Context) {
}
defer resp.Content.Close()
http.ServeContent(c.Writer, c.Request, "thumb."+conf.ThumbConfig.EncodeMethod, fs.FileTarget[0].UpdatedAt, resp.Content)
http.ServeContent(c.Writer, c.Request, "thumb."+model.GetSettingByNameWithDefault("thumb_encode_method", "jpg"), fs.FileTarget[0].UpdatedAt, resp.Content)
}

2
routers/controllers/site.go
Unescape View File

@ -49,7 +49,7 @@ func Ping(c *gin.Context) {
c.JSON(200, serializer.Response{
Code: 0,
Data: conf.BackendVersion + conf.IsPro,
Data: version,
})
}

2
routers/controllers/slave.go
Unescape View File

@ -2,8 +2,6 @@ package controllers
import (
"context"
"net/url"
"strconv"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"

180
routers/file_router_test.go
Unescape View File

@ -1,180 +0,0 @@
package routers
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/cloudreve/Cloudreve/v3/middleware"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/service/explorer"
"github.com/stretchr/testify/assert"
)
func TestListDirectoryRoute(t *testing.T) {
switchToMemDB()
asserts := assert.New(t)
router := InitMasterRouter()
w := httptest.NewRecorder()
// 成功
req, _ := http.NewRequest(
"GET",
"/api/v3/directory/",
nil,
)
middleware.SessionMock = map[string]interface{}{"user_id": 1}
router.ServeHTTP(w, req)
asserts.Equal(200, w.Code)
resJSON := &serializer.Response{}
err := json.Unmarshal(w.Body.Bytes(), resJSON)
asserts.NoError(err)
asserts.Equal(0, resJSON.Code)
w.Body.Reset()
}
func TestLocalFileUpload(t *testing.T) {
switchToMemDB()
asserts := assert.New(t)
router := InitMasterRouter()
w := httptest.NewRecorder()
middleware.SessionMock = map[string]interface{}{"user_id": 1}
testCases := []struct {
GetRequest func() *http.Request
ExpectCode int
RollBack func()
}{
// 文件大小指定错误
{
GetRequest: func() *http.Request {
req, _ := http.NewRequest(
"POST",
"/api/v3/file/upload",
nil,
)
req.Header.Add("Content-Length", "ddf")
return req
},
ExpectCode: 40001,
},
// 返回错误
{
GetRequest: func() *http.Request {
req, _ := http.NewRequest(
"POST",
"/api/v3/file/upload",
strings.NewReader("2333"),
)
req.Header.Add("Content-Length", "4")
req.Header.Add("X-Cr-FileName", "大地的%sfsf")
return req
},
ExpectCode: 40002,
},
// 成功
{
GetRequest: func() *http.Request {
req, _ := http.NewRequest(
"POST",
"/api/v3/file/upload",
strings.NewReader("2333"),
)
req.Header.Add("Content-Length", "4")
req.Header.Add("X-Cr-FileName", "TestFileUploadRoute.txt")
req.Header.Add("X-Cr-Path", "/")
return req
},
ExpectCode: 0,
},
}
for key, testCase := range testCases {
req := testCase.GetRequest()
router.ServeHTTP(w, req)
asserts.Equal(200, w.Code)
resJSON := &serializer.Response{}
err := json.Unmarshal(w.Body.Bytes(), resJSON)
asserts.NoError(err, "测试用例%d", key)
asserts.Equal(testCase.ExpectCode, resJSON.Code, "测试用例%d", key)
if testCase.RollBack != nil {
testCase.RollBack()
}
w.Body.Reset()
}
}
func TestObjectDelete(t *testing.T) {
asserts := assert.New(t)
router := InitMasterRouter()
w := httptest.NewRecorder()
middleware.SessionMock = map[string]interface{}{"user_id": 1}
testCases := []struct {
Mock []string
GetRequest func() *http.Request
ExpectCode int
RollBack []string
}{
// 路径不存在，返回无错误
{
GetRequest: func() *http.Request {
body := explorer.ItemService{
Items: []uint{1},
}
bodyStr, _ := json.Marshal(body)
req, _ := http.NewRequest(
"DELETE",
"/api/v3/object",
bytes.NewReader(bodyStr),
)
return req
},
ExpectCode: 0,
},
// 文件删除失败，返回203
{
Mock: []string{"INSERT INTO `files` (`id`, `created_at`, `updated_at`, `deleted_at`, `name`, `source_name`, `user_id`, `size`, `pic_info`, `folder_id`, `policy_id`) VALUES(5, '2019-11-30 07:08:33', '2019-11-30 07:08:33', NULL, 'pigeon.zip', '65azil3B_pigeon.zip', 1, 1667217, '', 1, 1);"},
GetRequest: func() *http.Request {
body := explorer.ItemService{
Items: []uint{5},
}
bodyStr, _ := json.Marshal(body)
req, _ := http.NewRequest(
"DELETE",
"/api/v3/object",
bytes.NewReader(bodyStr),
)
return req
},
RollBack: []string{"DELETE FROM `v3_files` WHERE `id`=5"},
ExpectCode: 203,
},
}
for key, testCase := range testCases {
for _, value := range testCase.Mock {
model.DB.Exec(value)
}
req := testCase.GetRequest()
router.ServeHTTP(w, req)
asserts.Equal(200, w.Code)
resJSON := &serializer.Response{}
err := json.Unmarshal(w.Body.Bytes(), resJSON)
asserts.NoError(err, "测试用例%d", key)
asserts.Equal(testCase.ExpectCode, resJSON.Code, "测试用例%d", key)
for _, value := range testCase.RollBack {
model.DB.Exec(value)
}
w.Body.Reset()
}
}

22
routers/router.go
Unescape View File

@ -244,19 +244,22 @@ func InitMasterRouter() *gin.Engine {
)
// 七牛策略上传回调
callback.POST(
"qiniu/:key",
"qiniu/:sessionID",
middleware.UseUploadSession("qiniu"),
middleware.QiniuCallbackAuth(),
controllers.QiniuCallback,
)
// 阿里云OSS策略上传回调
callback.POST(
"oss/:key",
"oss/:sessionID",
middleware.UseUploadSession("oss"),
middleware.OSSCallbackAuth(),
controllers.OSSCallback,
)
// 又拍云策略上传回调
callback.POST(
"upyun/:key",
"upyun/:sessionID",
middleware.UseUploadSession("upyun"),
middleware.UpyunCallbackAuth(),
controllers.UpyunCallback,
)
@ -264,11 +267,12 @@ func InitMasterRouter() *gin.Engine {
{
// 文件上传完成
onedrive.POST(
"finish/:key",
"finish/:sessionID",
middleware.UseUploadSession("onedrive"),
middleware.OneDriveCallbackAuth(),
controllers.OneDriveCallback,
)
// 文件上传完成
// OAuth 完成
onedrive.GET(
"auth",
controllers.OneDriveOAuth,
@ -276,14 +280,14 @@ func InitMasterRouter() *gin.Engine {
}
// 腾讯云COS策略上传回调
callback.GET(
"cos/:key",
middleware.COSCallbackAuth(),
"cos/:sessionID",
middleware.UseUploadSession("cos"),
controllers.COSCallback,
)
// AWS S3策略上传回调
callback.GET(
"s3/:key",
middleware.S3CallbackAuth(),
"s3/:sessionID",
middleware.UseUploadSession("s3"),
controllers.S3Callback,
)
}

3
routers/router_test.go
Unescape View File

@ -1,6 +1,7 @@
package routers
import (
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"net/http"
"net/http/httptest"
"testing"
@ -19,7 +20,7 @@ func TestPing(t *testing.T) {
router.ServeHTTP(w, req)
assert.Equal(t, 200, w.Code)
asserts.Contains(w.Body.String(), "Pong")
asserts.Contains(w.Body.String(), conf.BackendVersion)
}
func TestCaptcha(t *testing.T) {

21
service/admin/policy.go
Unescape View File

@ -149,9 +149,9 @@ func (service *PolicyService) AddCORS() serializer.Response {
switch policy.Type {
case "oss":
handler := oss.Driver{
Policy: &policy,
HTTPClient: request.NewClient(),
handler, err := oss.NewDriver(&policy)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err)
}
if err := handler.CORS(); err != nil {
return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err)
@ -169,13 +169,16 @@ func (service *PolicyService) AddCORS() serializer.Response {
},
}),
}
if err := handler.CORS(); err != nil {
return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err)
}
case "s3":
handler := s3.Driver{
Policy: &policy,
handler, err := s3.NewDriver(&policy)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err)
}
if err := handler.CORS(); err != nil {
return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err)
}
@ -207,8 +210,12 @@ func (service *SlavePingService) Test() serializer.Response {
return serializer.ParamErr("从机无法向主机发送回调请求，请检查主机端 参数设置 - 站点信息 - 站点URL设置，并确保从机可以连接到此地址，"+err.Error(), nil)
}
if res.Data.(string) != conf.BackendVersion {
return serializer.ParamErr("Cloudreve版本不一致，主机："+res.Data.(string)+"，从机："+conf.BackendVersion, nil)
version := conf.BackendVersion
if conf.IsPro == "true" {
version += "-pro"
}
if res.Data.(string) != version {
return serializer.ParamErr("Cloudreve版本不一致，主机："+res.Data.(string)+"，从机："+version, nil)
}
return serializer.Response{}

52
service/callback/upload.go
Unescape View File

@ -50,7 +50,6 @@ type UpyunCallbackService struct {
// OneDriveCallback OneDrive 客户端回调正文
type OneDriveCallback struct {
ID string `json:"id" binding:"required"`
Meta *onedrive.FileInfo
}
@ -62,9 +61,6 @@ type COSCallback struct {
// S3Callback S3 客户端回调正文
type S3Callback struct {
Bucket string `form:"bucket"`
Etag string `form:"etag"`
Key string `form:"key"`
}
// GetBody 返回回调正文
@ -165,22 +161,21 @@ func (service *OneDriveCallback) PreProcess(c *gin.Context) serializer.Response
defer fs.Recycle()
// 获取回调会话
callbackSessionRaw, _ := c.Get("callbackSession")
callbackSession := callbackSessionRaw.(*serializer.UploadSession)
uploadSession := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
// 获取文件信息
info, err := fs.Handler.(onedrive.Driver).Client.Meta(context.Background(), service.ID, "")
info, err := fs.Handler.(onedrive.Driver).Client.Meta(context.Background(), "", uploadSession.SavePath)
if err != nil {
return serializer.Err(serializer.CodeUploadFailed, "文件元信息查询失败", err)
}
// 验证与回调会话中是否一致
actualPath := strings.TrimPrefix(callbackSession.SavePath, "/")
isSizeCheckFailed := callbackSession.Size != info.Size
actualPath := strings.TrimPrefix(uploadSession.SavePath, "/")
isSizeCheckFailed := uploadSession.Size != info.Size
// SharePoint 会对 Office 文档增加 meta data 导致文件大小不一致，这里增加 10 KB 宽容
// SharePoint 会对 Office 文档增加 meta data 导致文件大小不一致，这里增加 100 KB 宽容
// See: https://github.com/OneDrive/onedrive-api-docs/issues/935
if strings.Contains(fs.Policy.OptionsSerialized.OdDriver, "sharepoint.com") && isSizeCheckFailed && (info.Size > callbackSession.Size) && (info.Size-callbackSession.Size <= 10240) {
if strings.Contains(fs.Policy.OptionsSerialized.OdDriver, "sharepoint.com") && isSizeCheckFailed && (info.Size > uploadSession.Size) && (info.Size-uploadSession.Size <= 102400) {
isSizeCheckFailed = false
}
@ -202,17 +197,16 @@ func (service *COSCallback) PreProcess(c *gin.Context) serializer.Response {
defer fs.Recycle()
// 获取回调会话
callbackSessionRaw, _ := c.Get("callbackSession")
callbackSession := callbackSessionRaw.(*serializer.UploadSession)
uploadSession := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
// 获取文件信息
info, err := fs.Handler.(cos.Driver).Meta(context.Background(), callbackSession.SavePath)
info, err := fs.Handler.(cos.Driver).Meta(context.Background(), uploadSession.SavePath)
if err != nil {
return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err)
}
// 验证实际文件信息与回调会话中是否一致
if callbackSession.Size != info.Size || callbackSession.Key != info.CallbackKey {
if uploadSession.Size != info.Size || uploadSession.Key != info.CallbackKey {
return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err)
}
@ -229,19 +223,39 @@ func (service *S3Callback) PreProcess(c *gin.Context) serializer.Response {
defer fs.Recycle()
// 获取回调会话
callbackSessionRaw, _ := c.Get("callbackSession")
callbackSession := callbackSessionRaw.(*serializer.UploadSession)
uploadSession := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
// 获取文件信息
info, err := fs.Handler.(s3.Driver).Meta(context.Background(), callbackSession.SavePath)
info, err := fs.Handler.(*s3.Driver).Meta(context.Background(), uploadSession.SavePath)
if err != nil {
return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err)
}
// 验证实际文件信息与回调会话中是否一致
if callbackSession.Size != info.Size || service.Etag != info.Etag {
if uploadSession.Size != info.Size {
return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err)
}
return ProcessCallback(service, c)
}
// PreProcess 对OneDrive客户端回调进行预处理验证
func (service *UploadCallbackService) PreProcess(c *gin.Context) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewFileSystemFromCallback(c)
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
defer fs.Recycle()
// 获取回调会话
uploadSession := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
// 验证文件大小
if uploadSession.Size != service.Size {
fs.Handler.Delete(context.Background(), []string{uploadSession.SavePath})
return serializer.Err(serializer.CodeUploadFailed, "文件大小不一致", nil)
}
return ProcessCallback(service, c)
}

Write Preview
Loading…
Cancel
Save