| | 激活{siteTitle}账户 | | 亲爱的{userName}： | | 感谢您注册{siteTitle},请点击下方按钮完成账户激活。 | | 激活账户 | | 感谢您选择{siteTitle}。 |
|
| |
`, Type: "mail_template"},
+ {Name: "forget_captcha", Value: `0`, Type: "login"},
+ {Name: "mail_reset_pwd_template", Value: ` | | 重设{siteTitle}密码 | | 亲爱的{userName}： | | 请点击下方按钮完成密码重设。如果非你本人操作，请忽略此邮件。 | | 重设密码 | | 感谢您选择{siteTitle}。 |
|
| |
`, Type: "mail_template"},
+ {Name: "db_version_" + conf.RequiredDBVersion, Value: `installed`, Type: "version"},
+ {Name: "hot_share_num", Value: `10`, Type: "share"},
+ {Name: "gravatar_server", Value: `https://www.gravatar.com/`, Type: "avatar"},
+ {Name: "defaultTheme", Value: `#3f51b5`, Type: "basic"},
+ {Name: "themes", Value: `{"#3f51b5":{"palette":{"primary":{"main":"#3f51b5"},"secondary":{"main":"#f50057"}}},"#2196f3":{"palette":{"primary":{"main":"#2196f3"},"secondary":{"main":"#FFC107"}}},"#673AB7":{"palette":{"primary":{"main":"#673AB7"},"secondary":{"main":"#2196F3"}}},"#E91E63":{"palette":{"primary":{"main":"#E91E63"},"secondary":{"main":"#42A5F5","contrastText":"#fff"}}},"#FF5722":{"palette":{"primary":{"main":"#FF5722"},"secondary":{"main":"#3F51B5"}}},"#FFC107":{"palette":{"primary":{"main":"#FFC107"},"secondary":{"main":"#26C6DA"}}},"#8BC34A":{"palette":{"primary":{"main":"#8BC34A","contrastText":"#fff"},"secondary":{"main":"#FF8A65","contrastText":"#fff"}}},"#009688":{"palette":{"primary":{"main":"#009688"},"secondary":{"main":"#4DD0E1","contrastText":"#fff"}}},"#607D8B":{"palette":{"primary":{"main":"#607D8B"},"secondary":{"main":"#F06292"}}},"#795548":{"palette":{"primary":{"main":"#795548"},"secondary":{"main":"#4CAF50","contrastText":"#fff"}}}}`, Type: "basic"},
+ {Name: "max_worker_num", Value: `10`, Type: "task"},
+ {Name: "max_parallel_transfer", Value: `4`, Type: "task"},
+ {Name: "secret_key", Value: util.RandStringRunes(256), Type: "auth"},
+ {Name: "temp_path", Value: "temp", Type: "path"},
+ {Name: "avatar_path", Value: "avatar", Type: "path"},
+ {Name: "avatar_size", Value: "2097152", Type: "avatar"},
+ {Name: "avatar_size_l", Value: "200", Type: "avatar"},
+ {Name: "avatar_size_m", Value: "130", Type: "avatar"},
+ {Name: "avatar_size_s", Value: "50", Type: "avatar"},
+ {Name: "home_view_method", Value: "icon", Type: "view"},
+ {Name: "share_view_method", Value: "list", Type: "view"},
+ {Name: "cron_garbage_collect", Value: "@hourly", Type: "cron"},
+ {Name: "cron_recycle_upload_session", Value: "@every 1h30m", Type: "cron"},
+ {Name: "authn_enabled", Value: "0", Type: "authn"},
+ {Name: "captcha_type", Value: "normal", Type: "captcha"},
+ {Name: "captcha_height", Value: "60", Type: "captcha"},
+ {Name: "captcha_width", Value: "240", Type: "captcha"},
+ {Name: "captcha_mode", Value: "3", Type: "captcha"},
+ {Name: "captcha_ComplexOfNoiseText", Value: "0", Type: "captcha"},
+ {Name: "captcha_ComplexOfNoiseDot", Value: "0", Type: "captcha"},
+ {Name: "captcha_IsShowHollowLine", Value: "0", Type: "captcha"},
+ {Name: "captcha_IsShowNoiseDot", Value: "1", Type: "captcha"},
+ {Name: "captcha_IsShowNoiseText", Value: "0", Type: "captcha"},
+ {Name: "captcha_IsShowSlimeLine", Value: "1", Type: "captcha"},
+ {Name: "captcha_IsShowSineLine", Value: "0", Type: "captcha"},
+ {Name: "captcha_CaptchaLen", Value: "6", Type: "captcha"},
+ {Name: "captcha_ReCaptchaKey", Value: "defaultKey", Type: "captcha"},
+ {Name: "captcha_ReCaptchaSecret", Value: "defaultSecret", Type: "captcha"},
+ {Name: "captcha_TCaptcha_CaptchaAppId", Value: "", Type: "captcha"},
+ {Name: "captcha_TCaptcha_AppSecretKey", Value: "", Type: "captcha"},
+ {Name: "captcha_TCaptcha_SecretId", Value: "", Type: "captcha"},
+ {Name: "captcha_TCaptcha_SecretKey", Value: "", Type: "captcha"},
+ {Name: "thumb_width", Value: "400", Type: "thumb"},
+ {Name: "thumb_height", Value: "300", Type: "thumb"},
+ {Name: "thumb_file_suffix", Value: "._thumb", Type: "thumb"},
+ {Name: "thumb_max_task_count", Value: "-1", Type: "thumb"},
+ {Name: "thumb_encode_method", Value: "jpg", Type: "thumb"},
+ {Name: "thumb_gc_after_gen", Value: "false", Type: "thumb"},
+ {Name: "thumb_encode_quality", Value: "85", Type: "thumb"},
+ {Name: "pwa_small_icon", Value: "/static/img/favicon.ico", Type: "pwa"},
+ {Name: "pwa_medium_icon", Value: "/static/img/logo192.png", Type: "pwa"},
+ {Name: "pwa_large_icon", Value: "/static/img/logo512.png", Type: "pwa"},
+ {Name: "pwa_display", Value: "standalone", Type: "pwa"},
+ {Name: "pwa_theme_color", Value: "#000000", Type: "pwa"},
+ {Name: "pwa_background_color", Value: "#ffffff", Type: "pwa"},
+ {Name: "office_preview_service", Value: "https://view.officeapps.live.com/op/view.aspx?src={$src}", Type: "preview"},
+}
diff --git a/models/file_test.go b/models/file_test.go
index 7e53c499..0f971848 100644
--- a/models/file_test.go
+++ b/models/file_test.go
@@ -15,22 +15,62 @@ func TestFile_Create(t *testing.T) {
Name: "123",
}
- mock.ExpectBegin()
- mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1))
- mock.ExpectCommit()
- fileID, err := file.Create()
- asserts.NoError(err)
- asserts.Equal(uint(5), fileID)
- asserts.Equal(uint(5), file.ID)
- asserts.NoError(mock.ExpectationsWereMet())
+ // 无法插入文件记录
+ {
+ mock.ExpectBegin()
+ mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
+ mock.ExpectRollback()
+ err := file.Create()
+ asserts.Error(err)
+ asserts.NoError(mock.ExpectationsWereMet())
+ }
- mock.ExpectBegin()
- mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
- mock.ExpectRollback()
- fileID, err = file.Create()
- asserts.Error(err)
- asserts.NoError(mock.ExpectationsWereMet())
+ // 无法更新用户容量
+ {
+ mock.ExpectBegin()
+ mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1))
+ mock.ExpectExec("UPDATE(.+)").WillReturnError(errors.New("error"))
+ mock.ExpectRollback()
+ err := file.Create()
+ asserts.Error(err)
+ asserts.NoError(mock.ExpectationsWereMet())
+ }
+
+ // 成功
+ {
+ mock.ExpectBegin()
+ mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1))
+ mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectCommit()
+ err := file.Create()
+ asserts.NoError(err)
+ asserts.Equal(uint(5), file.ID)
+ asserts.NoError(mock.ExpectationsWereMet())
+ }
+}
+
+func TestFile_AfterFind(t *testing.T) {
+ a := assert.New(t)
+ file := File{
+ Name: "123",
+ Metadata: "{\"name\":\"123\"}",
+ }
+
+ a.NoError(file.AfterFind())
+ a.Equal("123", file.MetadataSerialized["name"])
+}
+
+func TestFile_BeforeSave(t *testing.T) {
+ a := assert.New(t)
+ file := File{
+ Name: "123",
+ MetadataSerialized: map[string]string{
+ "name": "123",
+ },
+ }
+ a.NoError(file.BeforeSave())
+ a.Equal("{\"name\":\"123\"}", file.Metadata)
}
func TestFolder_GetChildFile(t *testing.T) {
@@ -175,6 +215,17 @@ func TestGetChildFilesOfFolders(t *testing.T) {
}
}
+func TestGetUploadPlaceholderFiles(t *testing.T) {
+ a := assert.New(t)
+
+ mock.ExpectQuery("SELECT(.+)upload_session_id(.+)").
+ WithArgs(1).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "1"))
+ files := GetUploadPlaceholderFiles(1)
+ a.NoError(mock.ExpectationsWereMet())
+ a.Len(files, 1)
+}
+
func TestFile_GetPolicy(t *testing.T) {
asserts := assert.New(t)
@@ -282,28 +333,50 @@ func TestRemoveFilesWithSoftLinks(t *testing.T) {
}
}
-func TestDeleteFileByIDs(t *testing.T) {
- asserts := assert.New(t)
+func TestDeleteFiles(t *testing.T) {
+ a := assert.New(t)
- // 出错
+ // uid 不一致
+ {
+ err := DeleteFiles([]*File{{}}, 1)
+ a.Contains("User id not consistent", err.Error())
+ }
+
+ // 删除失败
{
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
WillReturnError(errors.New("error"))
mock.ExpectRollback()
- err := DeleteFileByIDs([]uint{1, 2, 3})
- asserts.NoError(mock.ExpectationsWereMet())
- asserts.Error(err)
+ err := DeleteFiles([]*File{{}}, 0)
+ a.NoError(mock.ExpectationsWereMet())
+ a.Error(err)
}
- // 成功
+
+ // 无法变更用户容量
{
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
- WillReturnResult(sqlmock.NewResult(0, 3))
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnError(errors.New("error"))
+ mock.ExpectRollback()
+ err := DeleteFiles([]*File{{}}, 0)
+ a.NoError(mock.ExpectationsWereMet())
+ a.Error(err)
+ }
+
+ // 成功，其中一个文件已经不存在
+ {
+ mock.ExpectBegin()
+ mock.ExpectExec("DELETE(.+)").
+ WillReturnResult(sqlmock.NewResult(1, 0))
+ mock.ExpectExec("DELETE(.+)").
+ WillReturnResult(sqlmock.NewResult(2, 1))
+ mock.ExpectExec("UPDATE(.+)storage(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
- err := DeleteFileByIDs([]uint{1, 2, 3})
- asserts.NoError(mock.ExpectationsWereMet())
- asserts.NoError(err)
+ err := DeleteFiles([]*File{{Size: 1}, {Size: 2}}, 0)
+ a.NoError(mock.ExpectationsWereMet())
+ a.NoError(err)
}
}
@@ -324,6 +397,19 @@ func TestGetFilesByParentIDs(t *testing.T) {
asserts.Len(files, 3)
}
+func TestGetFilesByUploadSession(t *testing.T) {
+ a := assert.New(t)
+
+ mock.ExpectQuery("SELECT(.+)").
+ WithArgs(1, "sessionID").
+ WillReturnRows(
+ sqlmock.NewRows([]string{"id", "name"}).AddRow(4, "4.txt"))
+ files, err := GetFilesByUploadSession("sessionID", 1)
+ a.NoError(err)
+ a.NoError(mock.ExpectationsWereMet())
+ a.Equal("4.txt", files.Name)
+}
+
func TestFile_Updates(t *testing.T) {
asserts := assert.New(t)
file := File{Model: gorm.Model{ID: 1}}
@@ -340,24 +426,93 @@ func TestFile_Updates(t *testing.T) {
// UpdatePicInfo
{
mock.ExpectBegin()
- mock.ExpectExec("UPDATE(.+)").WithArgs(10, sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE(.+)").WithArgs("1,1", 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
- err := file.UpdateSize(10)
+ err := file.UpdatePicInfo("1,1")
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
- // UpdatePicInfo
+ // UpdateSourceName
{
mock.ExpectBegin()
- mock.ExpectExec("UPDATE(.+)").WithArgs("1,1", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE(.+)").WithArgs("newName", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
- err := file.UpdatePicInfo("1,1")
+ err := file.UpdateSourceName("newName")
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
}
+func TestFile_UpdateSize(t *testing.T) {
+ a := assert.New(t)
+
+ // 增加成功
+ {
+ file := File{Size: 10}
+ mock.ExpectBegin()
+ mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(11, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE(.+)storage(.+)+(.+)").WithArgs(uint64(1), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectCommit()
+
+ a.NoError(file.UpdateSize(11))
+ a.NoError(mock.ExpectationsWereMet())
+ }
+
+ // 减少成功
+ {
+ file := File{Size: 10}
+ mock.ExpectBegin()
+ mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE(.+)storage(.+)-(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectCommit()
+
+ a.NoError(file.UpdateSize(8))
+ a.NoError(mock.ExpectationsWereMet())
+ }
+
+ // 文件更新失败
+ {
+ file := File{Size: 10}
+ mock.ExpectBegin()
+ mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnError(errors.New("error"))
+ mock.ExpectRollback()
+
+ a.Error(file.UpdateSize(8))
+ a.NoError(mock.ExpectationsWereMet())
+ }
+
+ // 用户容量更新失败
+ {
+ file := File{Size: 10}
+ mock.ExpectBegin()
+ mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE(.+)storage(.+)-(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnError(errors.New("error"))
+ mock.ExpectRollback()
+
+ a.Error(file.UpdateSize(8))
+ a.NoError(mock.ExpectationsWereMet())
+ }
+}
+
+func TestFile_PopChunkToFile(t *testing.T) {
+ a := assert.New(t)
+ timeNow := time.Now()
+ file := File{}
+ mock.ExpectBegin()
+ mock.ExpectExec("UPDATE(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectCommit()
+ a.NoError(file.PopChunkToFile(&timeNow, "1,1"))
+}
+
+func TestFile_CanCopy(t *testing.T) {
+ a := assert.New(t)
+ file := File{}
+ a.True(file.CanCopy())
+ file.UploadSessionID = &file.Name
+ a.False(file.CanCopy())
+}
+
func TestFile_FileInfoInterface(t *testing.T) {
asserts := assert.New(t)
file := File{
diff --git a/models/folder_test.go b/models/folder_test.go
index 7c74f489..8174708d 100644
--- a/models/folder_test.go
+++ b/models/folder_test.go
@@ -212,12 +212,14 @@ func TestFolder_MoveOrCopyFileTo(t *testing.T) {
WithArgs(
1,
2,
+ 3,
1,
1,
).WillReturnRows(
- sqlmock.NewRows([]string{"id", "size"}).
- AddRow(1, 10).
- AddRow(2, 20),
+ sqlmock.NewRows([]string{"id", "size", "upload_session_id"}).
+ AddRow(1, 10, nil).
+ AddRow(2, 20, nil).
+ AddRow(2, 20, &folder.Name),
)
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
@@ -226,7 +228,7 @@ func TestFolder_MoveOrCopyFileTo(t *testing.T) {
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
storage, err := folder.MoveOrCopyFileTo(
- []uint{1, 2},
+ []uint{1, 2, 3},
&dstFolder,
true,
)
@@ -335,7 +337,7 @@ func TestFolder_CopyFolderTo(t *testing.T) {
// 测试复制目录结构
// test(2)(5)
// 1(3)(6) 2.txt
- // 3(4)(7) 4.txt
+ // 3(4)(7) 4.txt 5.txt(上传中)
// 正常情况 成功
{
@@ -360,9 +362,10 @@ func TestFolder_CopyFolderTo(t *testing.T) {
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 2, 3, 4).
WillReturnRows(
- sqlmock.NewRows([]string{"id", "name", "folder_id", "size"}).
- AddRow(1, "2.txt", 2, 10).
- AddRow(2, "3.txt", 3, 20),
+ sqlmock.NewRows([]string{"id", "name", "folder_id", "size", "upload_session_id"}).
+ AddRow(1, "2.txt", 2, 10, nil).
+ AddRow(2, "3.txt", 3, 20, nil).
+ AddRow(3, "5.txt", 3, 20, &dstFolder.Name),
)
// 复制子文件
diff --git a/models/migration.go b/models/migration.go
index 1053e654..d3ba8dfd 100644
--- a/models/migration.go
+++ b/models/migration.go
@@ -7,7 +7,6 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/fatih/color"
- "github.com/gofrs/uuid"
"github.com/hashicorp/go-version"
"github.com/jinzhu/gorm"
"sort"
@@ -86,114 +85,6 @@ func addDefaultPolicy() {
}
func addDefaultSettings() {
- siteID, _ := uuid.NewV4()
-
- defaultSettings := []Setting{
- {Name: "siteURL", Value: `http://localhost`, Type: "basic"},
- {Name: "siteName", Value: `Cloudreve`, Type: "basic"},
- {Name: "siteICPId", Value: ``, Type: "basic"},
- {Name: "register_enabled", Value: `1`, Type: "register"},
- {Name: "default_group", Value: `2`, Type: "register"},
- {Name: "siteKeywords", Value: `网盘，网盘`, Type: "basic"},
- {Name: "siteDes", Value: `Cloudreve`, Type: "basic"},
- {Name: "siteTitle", Value: `平步云端`, Type: "basic"},
- {Name: "siteScript", Value: ``, Type: "basic"},
- {Name: "siteID", Value: siteID.String(), Type: "basic"},
- {Name: "fromName", Value: `Cloudreve`, Type: "mail"},
- {Name: "mail_keepalive", Value: `30`, Type: "mail"},
- {Name: "fromAdress", Value: `no-reply@acg.blue`, Type: "mail"},
- {Name: "smtpHost", Value: `smtp.mxhichina.com`, Type: "mail"},
- {Name: "smtpPort", Value: `25`, Type: "mail"},
- {Name: "replyTo", Value: `abslant@126.com`, Type: "mail"},
- {Name: "smtpUser", Value: `no-reply@acg.blue`, Type: "mail"},
- {Name: "smtpPass", Value: ``, Type: "mail"},
- {Name: "smtpEncryption", Value: `0`, Type: "mail"},
- {Name: "maxEditSize", Value: `4194304`, Type: "file_edit"},
- {Name: "archive_timeout", Value: `60`, Type: "timeout"},
- {Name: "download_timeout", Value: `60`, Type: "timeout"},
- {Name: "preview_timeout", Value: `60`, Type: "timeout"},
- {Name: "doc_preview_timeout", Value: `60`, Type: "timeout"},
- {Name: "upload_credential_timeout", Value: `1800`, Type: "timeout"},
- {Name: "upload_session_timeout", Value: `86400`, Type: "timeout"},
- {Name: "slave_api_timeout", Value: `60`, Type: "timeout"},
- {Name: "slave_node_retry", Value: `3`, Type: "slave"},
- {Name: "slave_ping_interval", Value: `60`, Type: "slave"},
- {Name: "slave_recover_interval", Value: `120`, Type: "slave"},
- {Name: "slave_transfer_timeout", Value: `172800`, Type: "timeout"},
- {Name: "onedrive_monitor_timeout", Value: `600`, Type: "timeout"},
- {Name: "share_download_session_timeout", Value: `2073600`, Type: "timeout"},
- {Name: "onedrive_callback_check", Value: `20`, Type: "timeout"},
- {Name: "folder_props_timeout", Value: `300`, Type: "timeout"},
- {Name: "onedrive_chunk_retries", Value: `1`, Type: "retry"},
- {Name: "slave_chunk_retries", Value: `1`, Type: "retry"},
- {Name: "onedrive_source_timeout", Value: `1800`, Type: "timeout"},
- {Name: "reset_after_upload_failed", Value: `0`, Type: "upload"},
- {Name: "login_captcha", Value: `0`, Type: "login"},
- {Name: "reg_captcha", Value: `0`, Type: "login"},
- {Name: "email_active", Value: `0`, Type: "register"},
- {Name: "mail_activation_template", Value: ` | | 激活{siteTitle}账户 | | 亲爱的{userName}： | | 感谢您注册{siteTitle},请点击下方按钮完成账户激活。 | | 激活账户 | | 感谢您选择{siteTitle}。 |
|
| |
`, Type: "mail_template"},
- {Name: "forget_captcha", Value: `0`, Type: "login"},
- {Name: "mail_reset_pwd_template", Value: ` | | 重设{siteTitle}密码 | | 亲爱的{userName}： | | 请点击下方按钮完成密码重设。如果非你本人操作，请忽略此邮件。 | | 重设密码 | | 感谢您选择{siteTitle}。 |
|
| |
`, Type: "mail_template"},
- {Name: "db_version_" + conf.RequiredDBVersion, Value: `installed`, Type: "version"},
- {Name: "hot_share_num", Value: `10`, Type: "share"},
- {Name: "gravatar_server", Value: `https://www.gravatar.com/`, Type: "avatar"},
- {Name: "defaultTheme", Value: `#3f51b5`, Type: "basic"},
- {Name: "themes", Value: `{"#3f51b5":{"palette":{"primary":{"main":"#3f51b5"},"secondary":{"main":"#f50057"}}},"#2196f3":{"palette":{"primary":{"main":"#2196f3"},"secondary":{"main":"#FFC107"}}},"#673AB7":{"palette":{"primary":{"main":"#673AB7"},"secondary":{"main":"#2196F3"}}},"#E91E63":{"palette":{"primary":{"main":"#E91E63"},"secondary":{"main":"#42A5F5","contrastText":"#fff"}}},"#FF5722":{"palette":{"primary":{"main":"#FF5722"},"secondary":{"main":"#3F51B5"}}},"#FFC107":{"palette":{"primary":{"main":"#FFC107"},"secondary":{"main":"#26C6DA"}}},"#8BC34A":{"palette":{"primary":{"main":"#8BC34A","contrastText":"#fff"},"secondary":{"main":"#FF8A65","contrastText":"#fff"}}},"#009688":{"palette":{"primary":{"main":"#009688"},"secondary":{"main":"#4DD0E1","contrastText":"#fff"}}},"#607D8B":{"palette":{"primary":{"main":"#607D8B"},"secondary":{"main":"#F06292"}}},"#795548":{"palette":{"primary":{"main":"#795548"},"secondary":{"main":"#4CAF50","contrastText":"#fff"}}}}`, Type: "basic"},
- {Name: "max_worker_num", Value: `10`, Type: "task"},
- {Name: "max_parallel_transfer", Value: `4`, Type: "task"},
- {Name: "secret_key", Value: util.RandStringRunes(256), Type: "auth"},
- {Name: "temp_path", Value: "temp", Type: "path"},
- {Name: "avatar_path", Value: "avatar", Type: "path"},
- {Name: "avatar_size", Value: "2097152", Type: "avatar"},
- {Name: "avatar_size_l", Value: "200", Type: "avatar"},
- {Name: "avatar_size_m", Value: "130", Type: "avatar"},
- {Name: "avatar_size_s", Value: "50", Type: "avatar"},
- {Name: "home_view_method", Value: "icon", Type: "view"},
- {Name: "share_view_method", Value: "list", Type: "view"},
- {Name: "cron_garbage_collect", Value: "@hourly", Type: "cron"},
- {Name: "cron_recycle_upload_session", Value: "@every 1h30m", Type: "cron"},
- {Name: "authn_enabled", Value: "0", Type: "authn"},
- {Name: "captcha_type", Value: "normal", Type: "captcha"},
- {Name: "captcha_height", Value: "60", Type: "captcha"},
- {Name: "captcha_width", Value: "240", Type: "captcha"},
- {Name: "captcha_mode", Value: "3", Type: "captcha"},
- {Name: "captcha_ComplexOfNoiseText", Value: "0", Type: "captcha"},
- {Name: "captcha_ComplexOfNoiseDot", Value: "0", Type: "captcha"},
- {Name: "captcha_IsShowHollowLine", Value: "0", Type: "captcha"},
- {Name: "captcha_IsShowNoiseDot", Value: "1", Type: "captcha"},
- {Name: "captcha_IsShowNoiseText", Value: "0", Type: "captcha"},
- {Name: "captcha_IsShowSlimeLine", Value: "1", Type: "captcha"},
- {Name: "captcha_IsShowSineLine", Value: "0", Type: "captcha"},
- {Name: "captcha_CaptchaLen", Value: "6", Type: "captcha"},
- {Name: "captcha_ReCaptchaKey", Value: "defaultKey", Type: "captcha"},
- {Name: "captcha_ReCaptchaSecret", Value: "defaultSecret", Type: "captcha"},
- {Name: "captcha_TCaptcha_CaptchaAppId", Value: "", Type: "captcha"},
- {Name: "captcha_TCaptcha_AppSecretKey", Value: "", Type: "captcha"},
- {Name: "captcha_TCaptcha_SecretId", Value: "", Type: "captcha"},
- {Name: "captcha_TCaptcha_SecretKey", Value: "", Type: "captcha"},
- {Name: "thumb_width", Value: "400", Type: "thumb"},
- {Name: "thumb_height", Value: "300", Type: "thumb"},
- {Name: "pwa_small_icon", Value: "/static/img/favicon.ico", Type: "pwa"},
- {Name: "pwa_medium_icon", Value: "/static/img/logo192.png", Type: "pwa"},
- {Name: "pwa_large_icon", Value: "/static/img/logo512.png", Type: "pwa"},
- {Name: "pwa_display", Value: "standalone", Type: "pwa"},
- {Name: "pwa_theme_color", Value: "#000000", Type: "pwa"},
- {Name: "pwa_background_color", Value: "#ffffff", Type: "pwa"},
- {Name: "office_preview_service", Value: "https://view.officeapps.live.com/op/view.aspx?src={$src}", Type: "preview"},
- }
-
for _, value := range defaultSettings {
DB.Where(Setting{Name: value.Name}).Create(&value)
}
diff --git a/models/policy.go b/models/policy.go
index 66716925..a5f48268 100644
--- a/models/policy.go
+++ b/models/policy.go
@@ -3,8 +3,7 @@ package model
import (
"encoding/gob"
"encoding/json"
- "fmt"
- "net/url"
+ "github.com/gofrs/uuid"
"path"
"path/filepath"
"strconv"
@@ -60,6 +59,8 @@ type PolicyOption struct {
ServerSideEndpoint string `json:"server_side_endpoint,omitempty"`
// 分片上传的分片大小
ChunkSize uint64 `json:"chunk_size,omitempty"`
+ // 分片上传时是否需要预留空间
+ PlaceholderWithSize bool `json:"placeholder_with_size,omitempty"`
}
var thumbSuffix = map[string][]string{
@@ -150,7 +151,7 @@ func (policy *Policy) GeneratePath(uid uint, origin string) string {
func (policy *Policy) GenerateFileName(uid uint, origin string) string {
// 未开启自动重命名时，直接返回原始文件名
if !policy.AutoRename {
- return policy.getOriginNameRule(origin)
+ return origin
}
fileRule := policy.FileNameRule
@@ -169,35 +170,15 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string {
"{hour}": time.Now().Format("15"),
"{minute}": time.Now().Format("04"),
"{second}": time.Now().Format("05"),
+ "{originname}": origin,
+ "{ext}": filepath.Ext(origin),
+ "{uuid}": uuid.Must(uuid.NewV4()).String(),
}
- replaceTable["{originname}"] = policy.getOriginNameRule(origin)
-
fileRule = util.Replace(replaceTable, fileRule)
return fileRule
}
-func (policy Policy) getOriginNameRule(origin string) string {
- // 部分存储策略可以使用{origin}代表原始文件名
- if origin == "" {
- // 如果上游未传回原始文件名，则使用占位符，让云存储端替换
- switch policy.Type {
- case "qiniu":
- // 七牛会将$(fname)自动替换为原始文件名
- return "$(fname)"
- case "local", "remote":
- return origin
- case "oss", "cos":
- // OSS会将${filename}自动替换为原始文件名
- return "${filename}"
- case "upyun":
- // Upyun会将{filename}{.suffix}自动替换为原始文件名
- return "{filename}{.suffix}"
- }
- }
- return origin
-}
-
// IsDirectlyPreview 返回此策略下文件是否可以直接预览（不需要重定向）
func (policy *Policy) IsDirectlyPreview() bool {
return policy.Type == "local"
@@ -226,7 +207,15 @@ func (policy *Policy) IsThumbGenerateNeeded() bool {
// IsUploadPlaceholderWithSize 返回此策略创建上传会话时是否需要预留空间
func (policy *Policy) IsUploadPlaceholderWithSize() bool {
- return policy.Type == "remote"
+ if policy.Type == "remote" {
+ return true
+ }
+
+ if util.ContainsString([]string{"onedrive", "oss", "qiniu", "cos", "s3"}, policy.Type) {
+ return policy.OptionsSerialized.PlaceholderWithSize
+ }
+
+ return false
}
// CanStructureBeListed 返回存储策略是否能被前台列物理目录
@@ -234,39 +223,6 @@ func (policy *Policy) CanStructureBeListed() bool {
return policy.Type != "local" && policy.Type != "remote"
}
-// GetUploadURL 获取文件上传服务API地址
-func (policy *Policy) GetUploadURL() string {
- server, err := url.Parse(policy.Server)
- if err != nil {
- return policy.Server
- }
-
- controller, _ := url.Parse("")
- switch policy.Type {
- case "local", "onedrive":
- return "/api/v3/file/upload"
- case "remote":
- controller, _ = url.Parse("/api/v3/slave/upload")
- case "oss":
- return "https://" + policy.BucketName + "." + policy.Server
- case "cos":
- return policy.Server
- case "upyun":
- return "https://v0.api.upyun.com/" + policy.BucketName
- case "s3":
- if policy.Server == "" {
- return fmt.Sprintf("https://%s.s3.%s.amazonaws.com/", policy.BucketName,
- policy.OptionsSerialized.Region)
- }
-
- if !strings.Contains(policy.Server, policy.BucketName) {
- controller, _ = url.Parse("/" + policy.BucketName)
- }
- }
-
- return server.ResolveReference(controller).String()
-}
-
// SaveAndClearCache 更新并清理缓存
func (policy *Policy) SaveAndClearCache() error {
err := DB.Save(policy).Error
diff --git a/models/policy_test.go b/models/policy_test.go
index 433d1ecc..c888ada9 100644
--- a/models/policy_test.go
+++ b/models/policy_test.go
@@ -104,7 +104,7 @@ func TestPolicy_GenerateFileName(t *testing.T) {
asserts.Equal("123.txt", testPolicy.GenerateFileName(1, "123.txt"))
testPolicy.Type = "oss"
- asserts.Equal("${filename}", testPolicy.GenerateFileName(1, ""))
+ asserts.Equal("origin", testPolicy.GenerateFileName(1, "origin"))
}
// 重命名开启
@@ -145,19 +145,23 @@ func TestPolicy_GenerateFileName(t *testing.T) {
testPolicy.Type = "oss"
testPolicy.FileNameRule = "{uid}123{originname}"
- asserts.Equal("1123${filename}", testPolicy.GenerateFileName(1, ""))
+ asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321"))
testPolicy.Type = "upyun"
testPolicy.FileNameRule = "{uid}123{originname}"
- asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, ""))
+ asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321"))
testPolicy.Type = "qiniu"
testPolicy.FileNameRule = "{uid}123{originname}"
- asserts.Equal("1123$(fname)", testPolicy.GenerateFileName(1, ""))
+ asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321"))
testPolicy.Type = "local"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123", testPolicy.GenerateFileName(1, ""))
+
+ testPolicy.Type = "local"
+ testPolicy.FileNameRule = "{ext}123{uuid}"
+ asserts.Contains(testPolicy.GenerateFileName(1, "123.txt"), ".txt123")
}
}
@@ -170,78 +174,6 @@ func TestPolicy_IsDirectlyPreview(t *testing.T) {
asserts.False(policy.IsDirectlyPreview())
}
-func TestPolicy_GetUploadURL(t *testing.T) {
- asserts := assert.New(t)
-
- // 本地
- {
- cache.Set("setting_siteURL", "http://127.0.0.1", 0)
- policy := Policy{Type: "local", Server: "http://127.0.0.1"}
- asserts.Equal("/api/v3/file/upload", policy.GetUploadURL())
- }
-
- // 远程
- {
- policy := Policy{Type: "remote", Server: "http://127.0.0.1"}
- asserts.Equal("http://127.0.0.1/api/v3/slave/upload", policy.GetUploadURL())
- }
-
- // OSS
- {
- policy := Policy{Type: "oss", BucketName: "base", Server: "127.0.0.1"}
- asserts.Equal("https://base.127.0.0.1", policy.GetUploadURL())
- }
-
- // cos
- {
- policy := Policy{Type: "cos", BaseURL: "base", Server: "http://127.0.0.1"}
- asserts.Equal("http://127.0.0.1", policy.GetUploadURL())
- }
-
- // upyun
- {
- policy := Policy{Type: "upyun", BucketName: "base", Server: "http://127.0.0.1"}
- asserts.Equal("https://v0.api.upyun.com/base", policy.GetUploadURL())
- }
-
- // 未知
- {
- policy := Policy{Type: "unknown", Server: "http://127.0.0.1"}
- asserts.Equal("http://127.0.0.1", policy.GetUploadURL())
- }
-
- // S3 未填写自动生成
- {
- policy := Policy{
- Type: "s3",
- Server: "",
- BucketName: "bucket",
- OptionsSerialized: PolicyOption{Region: "us-east"},
- }
- asserts.Equal("https://bucket.s3.us-east.amazonaws.com/", policy.GetUploadURL())
- }
-
- // s3 自己指定
- {
- policy := Policy{
- Type: "s3",
- Server: "https://s3.us-east.amazonaws.com/",
- BucketName: "bucket",
- OptionsSerialized: PolicyOption{Region: "us-east"},
- }
- asserts.Equal("https://s3.us-east.amazonaws.com/bucket", policy.GetUploadURL())
- }
-
-}
-
-func TestPolicy_IsPathGenerateNeeded(t *testing.T) {
- asserts := assert.New(t)
- policy := Policy{Type: "qiniu"}
- asserts.True(policy.IsPathGenerateNeeded())
- policy.Type = "remote"
- asserts.False(policy.IsPathGenerateNeeded())
-}
-
func TestPolicy_ClearCache(t *testing.T) {
asserts := assert.New(t)
cache.Set("policy_202", 1, 0)
@@ -266,15 +198,18 @@ func TestPolicy_UpdateAccessKey(t *testing.T) {
func TestPolicy_Props(t *testing.T) {
asserts := assert.New(t)
policy := Policy{Type: "onedrive"}
+ policy.OptionsSerialized.PlaceholderWithSize = true
asserts.False(policy.IsThumbGenerateNeeded())
- asserts.True(policy.IsPathGenerateNeeded())
- asserts.True(policy.IsTransitUpload(4))
+ asserts.False(policy.IsTransitUpload(4))
asserts.False(policy.IsTransitUpload(5 * 1024 * 1024))
asserts.True(policy.CanStructureBeListed())
+ asserts.True(policy.IsUploadPlaceholderWithSize())
policy.Type = "local"
asserts.True(policy.IsThumbGenerateNeeded())
- asserts.True(policy.IsPathGenerateNeeded())
asserts.False(policy.CanStructureBeListed())
+ asserts.False(policy.IsUploadPlaceholderWithSize())
+ policy.Type = "remote"
+ asserts.True(policy.IsUploadPlaceholderWithSize())
}
func TestPolicy_IsThumbExist(t *testing.T) {
diff --git a/models/setting.go b/models/setting.go
index 1738c1d8..643ca848 100644
--- a/models/setting.go
+++ b/models/setting.go
@@ -43,6 +43,15 @@ func GetSettingByName(name string) string {
return ""
}
+// GetSettingByNameWithDefault 用 Name 获取设置值, 取不到时使用缺省值
+func GetSettingByNameWithDefault(name, fallback string) string {
+ res := GetSettingByName(name)
+ if res == "" {
+ return fallback
+ }
+ return res
+}
+
// GetSettingByNames 用多个 Name 获取设置值
func GetSettingByNames(names ...string) map[string]string {
var queryRes []Setting
diff --git a/models/setting_test.go b/models/setting_test.go
index d9bf9626..96fc5e03 100644
--- a/models/setting_test.go
+++ b/models/setting_test.go
@@ -59,6 +59,15 @@ func TestGetSettingByType(t *testing.T) {
asserts.Equal(map[string]string{}, settings)
}
+func TestGetSettingByNameWithDefault(t *testing.T) {
+ a := assert.New(t)
+
+ rows := sqlmock.NewRows([]string{"name", "value", "type"})
+ mock.ExpectQuery("^SELECT \\* FROM `(.+)` WHERE `(.+)`\\.`deleted_at` IS NULL AND(.+)$").WillReturnRows(rows)
+ settings := GetSettingByNameWithDefault("123", "123321")
+ a.Equal("123321", settings)
+}
+
func TestGetSettingByNames(t *testing.T) {
cache.Store = cache.NewMemoStore()
asserts := assert.New(t)
diff --git a/models/tag_test.go b/models/tag_test.go
index 6063234e..be8d3fb5 100644
--- a/models/tag_test.go
+++ b/models/tag_test.go
@@ -56,8 +56,8 @@ func TestGetTagsByUID(t *testing.T) {
func TestGetTagsByID(t *testing.T) {
asserts := assert.New(t)
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"name"}).AddRow("tag"))
- res, err := GetTagsByUID(1)
+ res, err := GetTagsByID(1, 1)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
- asserts.EqualValues("tag", res[0].Name)
+ asserts.EqualValues("tag", res.Name)
}
diff --git a/pkg/cache/driver.go b/pkg/cache/driver.go
index 35f69228..74c12192 100644
--- a/pkg/cache/driver.go
+++ b/pkg/cache/driver.go
@@ -2,6 +2,7 @@ package cache
import (
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
+ "github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin"
)
@@ -9,9 +10,7 @@ import (
var Store Driver = NewMemoStore()
// Init 初始化缓存
-func Init() {
- //Store = NewRedisStore(10, "tcp", "127.0.0.1:6379", "", "0")
- //return
+func Init(isSlave bool) {
if conf.RedisConfig.Server != "" && gin.Mode() != gin.TestMode {
Store = NewRedisStore(
10,
@@ -21,6 +20,13 @@ func Init() {
conf.RedisConfig.DB,
)
}
+
+ if isSlave {
+ err := Store.Sets(conf.OptionOverwrite, "setting_")
+ if err != nil {
+ util.Log().Warning("无法覆盖数据库设置: %s", err)
+ }
+ }
}
// Driver 键值缓存存储容器
diff --git a/pkg/cache/driver_test.go b/pkg/cache/driver_test.go
index d30a67f2..a0c5cfc0 100644
--- a/pkg/cache/driver_test.go
+++ b/pkg/cache/driver_test.go
@@ -56,6 +56,10 @@ func TestInit(t *testing.T) {
asserts := assert.New(t)
asserts.NotPanics(func() {
- Init()
+ Init(false)
+ })
+
+ asserts.NotPanics(func() {
+ Init(true)
})
}
diff --git a/pkg/cluster/slave.go b/pkg/cluster/slave.go
index 49e2a48d..79118b23 100644
--- a/pkg/cluster/slave.go
+++ b/pkg/cluster/slave.go
@@ -413,7 +413,6 @@ func getAria2RequestBody(body *serializer.SlaveAria2Call) (io.Reader, error) {
return strings.NewReader(string(reqBodyEncoded)), nil
}
-// TODO: move to slave pkg
// RemoteCallback 发送远程存储策略上传回调请求
func RemoteCallback(url string, body serializer.UploadCallback) error {
callbackBody, err := json.Marshal(struct {
diff --git a/pkg/cluster/slave_test.go b/pkg/cluster/slave_test.go
index 25809361..1b1510f6 100644
--- a/pkg/cluster/slave_test.go
+++ b/pkg/cluster/slave_test.go
@@ -1,8 +1,12 @@
package cluster
import (
+ "bytes"
+ "encoding/json"
+ "errors"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
+ "github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/stretchr/testify/assert"
@@ -442,124 +446,114 @@ func TestSlaveCaller_DeleteTempFile(t *testing.T) {
}
}
-//func TestRemoteCallback(t *testing.T) {
-// asserts := assert.New(t)
-//
-// // 回调成功
-// {
-// clientMock := request.ClientMock{}
-// mockResp, _ := json.Marshal(serializer.Response{Code: 0})
-// clientMock.On(
-// "Request",
-// "POST",
-// "http://test/test/url",
-// testMock.Anything,
-// testMock.Anything,
-// ).Return(&request.Response{
-// Err: nil,
-// Response: &http.Response{
-// StatusCode: 200,
-// Body: ioutil.NopCloser(bytes.NewReader(mockResp)),
-// },
-// })
-// request.GeneralClient = clientMock
-// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
-// SourceName: "source",
-// })
-// asserts.NoError(resp)
-// clientMock.AssertExpectations(t)
-// }
-//
-// // 服务端返回业务错误
-// {
-// clientMock := request.ClientMock{}
-// mockResp, _ := json.Marshal(serializer.Response{Code: 401})
-// clientMock.On(
-// "Request",
-// "POST",
-// "http://test/test/url",
-// testMock.Anything,
-// testMock.Anything,
-// ).Return(&request.Response{
-// Err: nil,
-// Response: &http.Response{
-// StatusCode: 200,
-// Body: ioutil.NopCloser(bytes.NewReader(mockResp)),
-// },
-// })
-// request.GeneralClient = clientMock
-// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
-// SourceName: "source",
-// })
-// asserts.EqualValues(401, resp.(serializer.AppError).Code)
-// clientMock.AssertExpectations(t)
-// }
-//
-// // 无法解析回调响应
-// {
-// clientMock := request.ClientMock{}
-// clientMock.On(
-// "Request",
-// "POST",
-// "http://test/test/url",
-// testMock.Anything,
-// testMock.Anything,
-// ).Return(&request.Response{
-// Err: nil,
-// Response: &http.Response{
-// StatusCode: 200,
-// Body: ioutil.NopCloser(strings.NewReader("mockResp")),
-// },
-// })
-// request.GeneralClient = clientMock
-// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
-// SourceName: "source",
-// })
-// asserts.Error(resp)
-// clientMock.AssertExpectations(t)
-// }
-//
-// // HTTP状态码非200
-// {
-// clientMock := request.ClientMock{}
-// clientMock.On(
-// "Request",
-// "POST",
-// "http://test/test/url",
-// testMock.Anything,
-// testMock.Anything,
-// ).Return(&request.Response{
-// Err: nil,
-// Response: &http.Response{
-// StatusCode: 404,
-// Body: ioutil.NopCloser(strings.NewReader("mockResp")),
-// },
-// })
-// request.GeneralClient = clientMock
-// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
-// SourceName: "source",
-// })
-// asserts.Error(resp)
-// clientMock.AssertExpectations(t)
-// }
-//
-// // 无法发起回调
-// {
-// clientMock := request.ClientMock{}
-// clientMock.On(
-// "Request",
-// "POST",
-// "http://test/test/url",
-// testMock.Anything,
-// testMock.Anything,
-// ).Return(&request.Response{
-// Err: errors.New("error"),
-// })
-// request.GeneralClient = clientMock
-// resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
-// SourceName: "source",
-// })
-// asserts.Error(resp)
-// clientMock.AssertExpectations(t)
-// }
-//}
+func TestRemoteCallback(t *testing.T) {
+ asserts := assert.New(t)
+
+ // 回调成功
+ {
+ clientMock := requestmock.RequestMock{}
+ mockResp, _ := json.Marshal(serializer.Response{Code: 0})
+ clientMock.On(
+ "Request",
+ "POST",
+ "http://test/test/url",
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: nil,
+ Response: &http.Response{
+ StatusCode: 200,
+ Body: ioutil.NopCloser(bytes.NewReader(mockResp)),
+ },
+ })
+ request.GeneralClient = clientMock
+ resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
+ asserts.NoError(resp)
+ clientMock.AssertExpectations(t)
+ }
+
+ // 服务端返回业务错误
+ {
+ clientMock := requestmock.RequestMock{}
+ mockResp, _ := json.Marshal(serializer.Response{Code: 401})
+ clientMock.On(
+ "Request",
+ "POST",
+ "http://test/test/url",
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: nil,
+ Response: &http.Response{
+ StatusCode: 200,
+ Body: ioutil.NopCloser(bytes.NewReader(mockResp)),
+ },
+ })
+ request.GeneralClient = clientMock
+ resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
+ asserts.EqualValues(401, resp.(serializer.AppError).Code)
+ clientMock.AssertExpectations(t)
+ }
+
+ // 无法解析回调响应
+ {
+ clientMock := requestmock.RequestMock{}
+ clientMock.On(
+ "Request",
+ "POST",
+ "http://test/test/url",
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: nil,
+ Response: &http.Response{
+ StatusCode: 200,
+ Body: ioutil.NopCloser(strings.NewReader("mockResp")),
+ },
+ })
+ request.GeneralClient = clientMock
+ resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
+ asserts.Error(resp)
+ clientMock.AssertExpectations(t)
+ }
+
+ // HTTP状态码非200
+ {
+ clientMock := requestmock.RequestMock{}
+ clientMock.On(
+ "Request",
+ "POST",
+ "http://test/test/url",
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: nil,
+ Response: &http.Response{
+ StatusCode: 404,
+ Body: ioutil.NopCloser(strings.NewReader("mockResp")),
+ },
+ })
+ request.GeneralClient = clientMock
+ resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
+ asserts.Error(resp)
+ clientMock.AssertExpectations(t)
+ }
+
+ // 无法发起回调
+ {
+ clientMock := requestmock.RequestMock{}
+ clientMock.On(
+ "Request",
+ "POST",
+ "http://test/test/url",
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: errors.New("error"),
+ })
+ request.GeneralClient = clientMock
+ resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
+ asserts.Error(resp)
+ clientMock.AssertExpectations(t)
+ }
+}
diff --git a/pkg/conf/conf.go b/pkg/conf/conf.go
index 7258646b..35ed8b6a 100644
--- a/pkg/conf/conf.go
+++ b/pkg/conf/conf.go
@@ -45,21 +45,6 @@ type slave struct {
SignatureTTL int `validate:"omitempty,gte=1"`
}
-// captcha 验证码配置
-type captcha struct {
- Height int `validate:"gte=0"`
- Width int `validate:"gte=0"`
- Mode int `validate:"gte=0,lte=3"`
- ComplexOfNoiseText int `validate:"gte=0,lte=2"`
- ComplexOfNoiseDot int `validate:"gte=0,lte=2"`
- IsShowHollowLine bool
- IsShowNoiseDot bool
- IsShowNoiseText bool
- IsShowSlimeLine bool
- IsShowSineLine bool
- CaptchaLen int `validate:"gt=0"`
-}
-
// redis 配置
type redis struct {
Network string
@@ -68,17 +53,6 @@ type redis struct {
DB string
}
-// 缩略图 配置
-type thumb struct {
- MaxWidth uint
- MaxHeight uint
- FileSuffix string `validate:"min=1"`
- MaxTaskCount int
- EncodeMethod string `validate:"eq=jpg|eq=png"`
- EncodeQuality int `validate:"gte=1,lte=100"`
- GCAfterGen bool
-}
-
// 跨域配置
type cors struct {
AllowOrigins []string
@@ -91,6 +65,7 @@ type cors struct {
var cfg *ini.File
const defaultConf = `[System]
+Debug = false
Mode = master
Listen = :5212
SessionSecret = {SessionSecret}
@@ -131,9 +106,7 @@ func Init(path string) {
"System": SystemConfig,
"SSL": SSLConfig,
"UnixSocket": UnixConfig,
- "Captcha": CaptchaConfig,
"Redis": RedisConfig,
- "Thumbnail": ThumbConfig,
"CORS": CORSConfig,
"Slave": SlaveConfig,
}
@@ -144,6 +117,11 @@ func Init(path string) {
}
}
+ // 映射数据库配置覆盖
+ for _, key := range cfg.Section("OptionOverwrite").Keys() {
+ OptionOverwrite[key.Name()] = key.Value()
+ }
+
// 重设log等级
if !SystemConfig.Debug {
util.Level = util.LevelInformational
diff --git a/pkg/conf/conf_test.go b/pkg/conf/conf_test.go
index aa95a7ef..6d186ed4 100644
--- a/pkg/conf/conf_test.go
+++ b/pkg/conf/conf_test.go
@@ -56,7 +56,11 @@ User = root
Password = root
Host = 127.0.0.1:3306
Name = v3
-TablePrefix = v3_`
+TablePrefix = v3_
+
+[OptionOverwrite]
+key=value
+`
err := ioutil.WriteFile("testConf.ini", []byte(testCase), 0644)
defer func() { err = os.Remove("testConf.ini") }()
if err != nil {
@@ -65,6 +69,7 @@ TablePrefix = v3_`
asserts.NotPanics(func() {
Init("testConf.ini")
})
+ asserts.Equal(OptionOverwrite["key"], "value")
}
func TestMapSection(t *testing.T) {
diff --git a/pkg/conf/defaults.go b/pkg/conf/defaults.go
index b39329c9..44c5371e 100644
--- a/pkg/conf/defaults.go
+++ b/pkg/conf/defaults.go
@@ -1,7 +1,5 @@
package conf
-import "github.com/mojocn/base64Captcha"
-
// RedisConfig Redis服务器配置
var RedisConfig = &redis{
Network: "tcp",
@@ -25,21 +23,6 @@ var SystemConfig = &system{
Listen: ":5212",
}
-// CaptchaConfig 验证码配置
-var CaptchaConfig = &captcha{
- Height: 60,
- Width: 240,
- Mode: 3,
- ComplexOfNoiseText: base64Captcha.CaptchaComplexLower,
- ComplexOfNoiseDot: base64Captcha.CaptchaComplexLower,
- IsShowHollowLine: false,
- IsShowNoiseDot: false,
- IsShowNoiseText: false,
- IsShowSlimeLine: false,
- IsShowSineLine: false,
- CaptchaLen: 6,
-}
-
// CORSConfig 跨域配置
var CORSConfig = &cors{
AllowOrigins: []string{"UNSET"},
@@ -49,17 +32,6 @@ var CORSConfig = &cors{
ExposeHeaders: nil,
}
-// ThumbConfig 缩略图配置
-var ThumbConfig = &thumb{
- MaxWidth: 400,
- MaxHeight: 300,
- FileSuffix: "._thumb",
- MaxTaskCount: -1,
- EncodeMethod: "jpg",
- GCAfterGen: false,
- EncodeQuality: 85,
-}
-
// SlaveConfig 从机配置
var SlaveConfig = &slave{
CallbackTimeout: 20,
@@ -75,3 +47,5 @@ var SSLConfig = &ssl{
var UnixConfig = &unix{
Listen: "",
}
+
+var OptionOverwrite = map[string]interface{}{}
diff --git a/pkg/filesystem/archive_test.go b/pkg/filesystem/archive_test.go
index 7a697ce7..be65b29a 100644
--- a/pkg/filesystem/archive_test.go
+++ b/pkg/filesystem/archive_test.go
@@ -3,6 +3,9 @@ package filesystem
import (
"context"
"errors"
+ "github.com/cloudreve/Cloudreve/v3/pkg/request"
+ "github.com/cloudreve/Cloudreve/v3/pkg/util"
+ testMock "github.com/stretchr/testify/mock"
"io"
"os"
"strings"
@@ -12,11 +15,8 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
- "github.com/cloudreve/Cloudreve/v3/pkg/request"
- "github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
- testMock "github.com/stretchr/testify/mock"
)
func TestFileSystem_Compress(t *testing.T) {
diff --git a/pkg/filesystem/chunk/backoff/backoff_test.go b/pkg/filesystem/chunk/backoff/backoff_test.go
new file mode 100644
index 00000000..6419c715
--- /dev/null
+++ b/pkg/filesystem/chunk/backoff/backoff_test.go
@@ -0,0 +1,22 @@
+package backoff
+
+import (
+ "github.com/stretchr/testify/assert"
+ "testing"
+ "time"
+)
+
+func TestConstantBackoff_Next(t *testing.T) {
+ a := assert.New(t)
+
+ b := &ConstantBackoff{Sleep: time.Duration(0), Max: 3}
+ a.True(b.Next())
+ a.True(b.Next())
+ a.True(b.Next())
+ a.False(b.Next())
+ b.Reset()
+ a.True(b.Next())
+ a.True(b.Next())
+ a.True(b.Next())
+ a.False(b.Next())
+}
diff --git a/pkg/filesystem/chunk/chunk.go b/pkg/filesystem/chunk/chunk.go
index e8a63f7d..24e50a1c 100644
--- a/pkg/filesystem/chunk/chunk.go
+++ b/pkg/filesystem/chunk/chunk.go
@@ -7,50 +7,100 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"io"
+ "os"
)
+const bufferTempPattern = "cdChunk.*.tmp"
+
// ChunkProcessFunc callback function for processing a chunk
type ChunkProcessFunc func(c *ChunkGroup, chunk io.Reader) error
// ChunkGroup manage groups of chunks
type ChunkGroup struct {
- file fsctx.FileHeader
- chunkSize uint64
- backoff backoff.Backoff
+ file fsctx.FileHeader
+ chunkSize uint64
+ backoff backoff.Backoff
+ enableRetryBuffer bool
fileInfo *fsctx.UploadTaskInfo
currentIndex int
chunkNum uint64
+ bufferTemp *os.File
}
-func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Backoff) *ChunkGroup {
+func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Backoff, useBuffer bool) *ChunkGroup {
c := &ChunkGroup{
- file: file,
- chunkSize: chunkSize,
- backoff: backoff,
- fileInfo: file.Info(),
- currentIndex: -1,
+ file: file,
+ chunkSize: chunkSize,
+ backoff: backoff,
+ fileInfo: file.Info(),
+ currentIndex: -1,
+ enableRetryBuffer: useBuffer,
}
if c.chunkSize == 0 {
c.chunkSize = c.fileInfo.Size
}
- c.chunkNum = c.fileInfo.Size / c.chunkSize
- if c.fileInfo.Size%c.chunkSize != 0 || c.fileInfo.Size == 0 {
- c.chunkNum++
+ if c.fileInfo.Size == 0 {
+ c.chunkNum = 1
+ } else {
+ c.chunkNum = c.fileInfo.Size / c.chunkSize
+ if c.fileInfo.Size%c.chunkSize != 0 {
+ c.chunkNum++
+ }
}
return c
}
+// TempAvailable returns if current chunk temp file is available to be read
+func (c *ChunkGroup) TempAvailable() bool {
+ if c.bufferTemp != nil {
+ state, _ := c.bufferTemp.Stat()
+ return state != nil && state.Size() == c.Length()
+ }
+
+ return false
+}
+
// Process a chunk with retry logic
func (c *ChunkGroup) Process(processor ChunkProcessFunc) error {
- err := processor(c, io.LimitReader(c.file, int64(c.chunkSize)))
+ reader := io.LimitReader(c.file, int64(c.chunkSize))
+
+ // If useBuffer is enabled, tee the reader to a temp file
+ if c.enableRetryBuffer && c.bufferTemp == nil && !c.file.Seekable() {
+ c.bufferTemp, _ = os.CreateTemp("", bufferTempPattern)
+ reader = io.TeeReader(reader, c.bufferTemp)
+ }
+
+ if c.bufferTemp != nil {
+ defer func() {
+ if c.bufferTemp != nil {
+ c.bufferTemp.Close()
+ os.Remove(c.bufferTemp.Name())
+ c.bufferTemp = nil
+ }
+ }()
+
+ // if temp buffer file is available, use it
+ if c.TempAvailable() {
+ if _, err := c.bufferTemp.Seek(0, io.SeekStart); err != nil {
+ return fmt.Errorf("failed to seek temp file back to chunk start: %w", err)
+ }
+
+ util.Log().Debug("Chunk %d will be read from temp file %q.", c.Index(), c.bufferTemp.Name())
+ reader = c.bufferTemp
+ }
+ }
+
+ err := processor(c, reader)
if err != nil {
- if err != context.Canceled && c.file.Seekable() && c.backoff.Next() {
- if _, seekErr := c.file.Seek(c.Start(), io.SeekStart); seekErr != nil {
- return fmt.Errorf("failed to seek back to chunk start: %w, last error: %w", seekErr, err)
+ if err != context.Canceled && (c.file.Seekable() || c.TempAvailable()) && c.backoff.Next() {
+ if c.file.Seekable() {
+ if _, seekErr := c.file.Seek(c.Start(), io.SeekStart); seekErr != nil {
+ return fmt.Errorf("failed to seek back to chunk start: %w, last error: %s", seekErr, err)
+ }
}
util.Log().Debug("Retrying chunk %d, last error: %s", c.currentIndex, err)
@@ -60,6 +110,7 @@ func (c *ChunkGroup) Process(processor ChunkProcessFunc) error {
return err
}
+ util.Log().Debug("Chunk %d processed", c.currentIndex)
return nil
}
@@ -68,6 +119,21 @@ func (c *ChunkGroup) Start() int64 {
return int64(uint64(c.Index()) * c.chunkSize)
}
+// Total returns the total length
+func (c *ChunkGroup) Total() int64 {
+ return int64(c.fileInfo.Size)
+}
+
+// Num returns the total chunk number
+func (c *ChunkGroup) Num() int {
+ return int(c.chunkNum)
+}
+
+// RangeHeader returns header value of Content-Range
+func (c *ChunkGroup) RangeHeader() string {
+ return fmt.Sprintf("bytes %d-%d/%d", c.Start(), c.Start()+c.Length()-1, c.Total())
+}
+
// Index returns current chunk index, starts from 0
func (c *ChunkGroup) Index() int {
return c.currentIndex
@@ -89,3 +155,8 @@ func (c *ChunkGroup) Length() int64 {
return int64(contentLength)
}
+
+// IsLast returns if current chunk is the last one
+func (c *ChunkGroup) IsLast() bool {
+ return c.Index() == int(c.chunkNum-1)
+}
diff --git a/pkg/filesystem/chunk/chunk_test.go b/pkg/filesystem/chunk/chunk_test.go
new file mode 100644
index 00000000..4bdcd06d
--- /dev/null
+++ b/pkg/filesystem/chunk/chunk_test.go
@@ -0,0 +1,250 @@
+package chunk
+
+import (
+ "errors"
+ "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
+ "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
+ "github.com/stretchr/testify/assert"
+ "io"
+ "os"
+ "strings"
+ "testing"
+)
+
+func TestNewChunkGroup(t *testing.T) {
+ a := assert.New(t)
+
+ testCases := []struct {
+ fileSize uint64
+ chunkSize uint64
+ expectedInnerChunkSize uint64
+ expectedChunkNum uint64
+ expectedInfo [][2]int //Start, Index,Length
+ }{
+ {10, 0, 10, 1, [][2]int{{0, 10}}},
+ {0, 0, 0, 1, [][2]int{{0, 0}}},
+ {0, 10, 10, 1, [][2]int{{0, 0}}},
+ {50, 10, 10, 5, [][2]int{
+ {0, 10},
+ {10, 10},
+ {20, 10},
+ {30, 10},
+ {40, 10},
+ }},
+ {50, 50, 50, 1, [][2]int{
+ {0, 50},
+ }},
+
+ {50, 15, 15, 4, [][2]int{
+ {0, 15},
+ {15, 15},
+ {30, 15},
+ {45, 5},
+ }},
+ }
+
+ for index, testCase := range testCases {
+ file := &fsctx.FileStream{Size: testCase.fileSize}
+ chunkGroup := NewChunkGroup(file, testCase.chunkSize, &backoff.ConstantBackoff{}, true)
+ a.EqualValues(testCase.expectedChunkNum, chunkGroup.Num(),
+ "TestCase:%d,ChunkNum()", index)
+ a.EqualValues(testCase.expectedInnerChunkSize, chunkGroup.chunkSize,
+ "TestCase:%d,InnerChunkSize()", index)
+ a.EqualValues(testCase.expectedChunkNum, chunkGroup.Num(),
+ "TestCase:%d,len(Chunks)", index)
+ a.EqualValues(testCase.fileSize, chunkGroup.Total())
+
+ for cIndex, info := range testCase.expectedInfo {
+ a.True(chunkGroup.Next())
+ a.EqualValues(info[1], chunkGroup.Length(),
+ "TestCase:%d,Chunks[%d].Length()", index, cIndex)
+ a.EqualValues(info[0], chunkGroup.Start(),
+ "TestCase:%d,Chunks[%d].Start()", index, cIndex)
+
+ a.Equal(cIndex == len(testCase.expectedInfo)-1, chunkGroup.IsLast(),
+ "TestCase:%d,Chunks[%d].IsLast()", index, cIndex)
+
+ a.NotEmpty(chunkGroup.RangeHeader())
+ }
+ a.False(chunkGroup.Next())
+ }
+}
+
+func TestChunkGroup_TempAvailablet(t *testing.T) {
+ a := assert.New(t)
+
+ file := &fsctx.FileStream{Size: 1}
+ c := NewChunkGroup(file, 0, &backoff.ConstantBackoff{}, true)
+ a.False(c.TempAvailable())
+
+ f, err := os.CreateTemp("", "TestChunkGroup_TempAvailablet.*")
+ defer func() {
+ f.Close()
+ os.Remove(f.Name())
+ }()
+ a.NoError(err)
+ c.bufferTemp = f
+
+ a.False(c.TempAvailable())
+ f.Write([]byte("1"))
+ a.True(c.TempAvailable())
+
+}
+
+func TestChunkGroup_Process(t *testing.T) {
+ a := assert.New(t)
+ file := &fsctx.FileStream{Size: 10}
+
+ // success
+ {
+ file.File = io.NopCloser(strings.NewReader("1234567890"))
+ c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{}, true)
+ count := 0
+ a.True(c.Next())
+ a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
+ count++
+ res, err := io.ReadAll(chunk)
+ a.NoError(err)
+ a.EqualValues("12345", string(res))
+ return nil
+ }))
+ a.True(c.Next())
+ a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
+ count++
+ res, err := io.ReadAll(chunk)
+ a.NoError(err)
+ a.EqualValues("67890", string(res))
+ return nil
+ }))
+ a.False(c.Next())
+ a.Equal(2, count)
+ }
+
+ // retry, read from buffer file
+ {
+ file.File = io.NopCloser(strings.NewReader("1234567890"))
+ c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, true)
+ count := 0
+ a.True(c.Next())
+ a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
+ count++
+ res, err := io.ReadAll(chunk)
+ a.NoError(err)
+ a.EqualValues("12345", string(res))
+ return nil
+ }))
+ a.True(c.Next())
+ a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
+ count++
+ res, err := io.ReadAll(chunk)
+ a.NoError(err)
+ a.EqualValues("67890", string(res))
+ if count == 2 {
+ return errors.New("error")
+ }
+ return nil
+ }))
+ a.False(c.Next())
+ a.Equal(3, count)
+ }
+
+ // retry, read from seeker
+ {
+ f, _ := os.CreateTemp("", "TestChunkGroup_Process.*")
+ f.Write([]byte("1234567890"))
+ f.Seek(0, 0)
+ defer func() {
+ f.Close()
+ os.Remove(f.Name())
+ }()
+ file.File = f
+ file.Seeker = f
+ c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false)
+ count := 0
+ a.True(c.Next())
+ a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
+ count++
+ res, err := io.ReadAll(chunk)
+ a.NoError(err)
+ a.EqualValues("12345", string(res))
+ return nil
+ }))
+ a.True(c.Next())
+ a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
+ count++
+ res, err := io.ReadAll(chunk)
+ a.NoError(err)
+ a.EqualValues("67890", string(res))
+ if count == 2 {
+ return errors.New("error")
+ }
+ return nil
+ }))
+ a.False(c.Next())
+ a.Equal(3, count)
+ }
+
+ // retry, seek error
+ {
+ f, _ := os.CreateTemp("", "TestChunkGroup_Process.*")
+ f.Write([]byte("1234567890"))
+ f.Seek(0, 0)
+ defer func() {
+ f.Close()
+ os.Remove(f.Name())
+ }()
+ file.File = f
+ file.Seeker = f
+ c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false)
+ count := 0
+ a.True(c.Next())
+ a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
+ count++
+ res, err := io.ReadAll(chunk)
+ a.NoError(err)
+ a.EqualValues("12345", string(res))
+ return nil
+ }))
+ a.True(c.Next())
+ f.Close()
+ a.Error(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
+ count++
+ if count == 2 {
+ return errors.New("error")
+ }
+ return nil
+ }))
+ a.False(c.Next())
+ a.Equal(2, count)
+ }
+
+ // retry, finally error
+ {
+ f, _ := os.CreateTemp("", "TestChunkGroup_Process.*")
+ f.Write([]byte("1234567890"))
+ f.Seek(0, 0)
+ defer func() {
+ f.Close()
+ os.Remove(f.Name())
+ }()
+ file.File = f
+ file.Seeker = f
+ c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false)
+ count := 0
+ a.True(c.Next())
+ a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
+ count++
+ res, err := io.ReadAll(chunk)
+ a.NoError(err)
+ a.EqualValues("12345", string(res))
+ return nil
+ }))
+ a.True(c.Next())
+ a.Error(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
+ count++
+ return errors.New("error")
+ }))
+ a.False(c.Next())
+ a.Equal(4, count)
+ }
+}
diff --git a/pkg/filesystem/driver/cos/handler.go b/pkg/filesystem/driver/cos/handler.go
index cc55ac4b..53896416 100644
--- a/pkg/filesystem/driver/cos/handler.go
+++ b/pkg/filesystem/driver/cos/handler.go
@@ -357,8 +357,9 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
res, err := handler.getUploadCredential(ctx, postPolicy, keyTime, savePath)
if err == nil {
+ res.SessionID = uploadSession.Key
res.Callback = apiURL
- res.Key = uploadSession.Key
+ res.UploadURLs = []string{handler.Policy.Server}
}
return res, err
@@ -415,10 +416,10 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli
signature := hmacFinalSign.Sum(nil)
return &serializer.UploadCredential{
- Policy: policyEncoded,
- Path: savePath,
- AccessKey: handler.Policy.AccessKey,
- Token: fmt.Sprintf("%x", signature),
- KeyTime: keyTime,
+ Policy: policyEncoded,
+ Path: savePath,
+ AccessKey: handler.Policy.AccessKey,
+ Credential: fmt.Sprintf("%x", signature),
+ KeyTime: keyTime,
}, nil
}
diff --git a/pkg/filesystem/driver/local/handler.go b/pkg/filesystem/driver/local/handler.go
index 85bb3eb8..e5e8994a 100644
--- a/pkg/filesystem/driver/local/handler.go
+++ b/pkg/filesystem/driver/local/handler.go
@@ -12,7 +12,6 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
- "github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
@@ -162,7 +161,7 @@ func (handler Driver) Truncate(ctx context.Context, src string, size uint64) err
util.Log().Warning("截断文件 [%s] 至 [%d]", src, size)
out, err := os.OpenFile(src, os.O_WRONLY, Perm)
if err != nil {
- util.Log().Warning("无法打开或创建文件，%s", err)
+ util.Log().Warning("无法打开文件，%s", err)
return err
}
@@ -188,7 +187,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// 尝试删除文件的缩略图（如果有）
- _ = os.Remove(util.RelativePath(value + conf.ThumbConfig.FileSuffix))
+ _ = os.Remove(util.RelativePath(value + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")))
}
return deleteFailed, retErr
@@ -196,7 +195,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
- file, err := handler.Get(ctx, path+conf.ThumbConfig.FileSuffix)
+ file, err := handler.Get(ctx, path+model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"))
if err != nil {
return nil, err
}
diff --git a/pkg/filesystem/driver/local/handler_test.go b/pkg/filesystem/driver/local/handler_test.go
index dac4d54d..9167e82f 100644
--- a/pkg/filesystem/driver/local/handler_test.go
+++ b/pkg/filesystem/driver/local/handler_test.go
@@ -4,13 +4,12 @@ import (
"context"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
- "github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
+ "github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
"io"
- "io/ioutil"
"net/url"
"os"
"strings"
@@ -20,52 +19,74 @@ import (
func TestHandler_Put(t *testing.T) {
asserts := assert.New(t)
handler := Driver{}
- ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
- os.Remove(util.RelativePath("test/test/txt"))
+
+ defer func() {
+ os.Remove(util.RelativePath("TestHandler_Put.txt"))
+ os.Remove(util.RelativePath("inner/TestHandler_Put.txt"))
+ }()
testCases := []struct {
- file io.ReadCloser
- dst string
- err bool
+ file fsctx.FileHeader
+ errContains string
}{
- {
- file: ioutil.NopCloser(strings.NewReader("test input file")),
- dst: "test/test/txt",
- err: false,
- },
- {
- file: ioutil.NopCloser(strings.NewReader("test input file")),
- dst: "test/test/txt",
- err: true,
- },
- {
- file: ioutil.NopCloser(strings.NewReader("test input file")),
- dst: "/notexist:/S.TXT",
- err: true,
- },
+ {&fsctx.FileStream{
+ SavePath: "TestHandler_Put.txt",
+ File: io.NopCloser(strings.NewReader("")),
+ }, ""},
+ {&fsctx.FileStream{
+ SavePath: "TestHandler_Put.txt",
+ File: io.NopCloser(strings.NewReader("")),
+ }, "物理同名文件已存在或不可用"},
+ {&fsctx.FileStream{
+ SavePath: "inner/TestHandler_Put.txt",
+ File: io.NopCloser(strings.NewReader("")),
+ }, ""},
+ {&fsctx.FileStream{
+ Mode: fsctx.Append | fsctx.Overwrite,
+ SavePath: "inner/TestHandler_Put.txt",
+ File: io.NopCloser(strings.NewReader("123")),
+ }, ""},
+ {&fsctx.FileStream{
+ AppendStart: 10,
+ Mode: fsctx.Append | fsctx.Overwrite,
+ SavePath: "inner/TestHandler_Put.txt",
+ File: io.NopCloser(strings.NewReader("123")),
+ }, "未上传完成的文件分片与预期大小不一致"},
+ {&fsctx.FileStream{
+ Mode: fsctx.Append | fsctx.Overwrite,
+ SavePath: "inner/TestHandler_Put.txt",
+ File: io.NopCloser(strings.NewReader("123")),
+ }, ""},
}
for _, testCase := range testCases {
- err := handler.Put(ctx, testCase.file, testCase.dst, 15)
- if testCase.err {
+ err := handler.Put(context.Background(), testCase.file)
+ if testCase.errContains != "" {
asserts.Error(err)
+ asserts.Contains(err.Error(), testCase.errContains)
} else {
asserts.NoError(err)
- asserts.True(util.Exists(util.RelativePath(testCase.dst)))
+ asserts.True(util.Exists(util.RelativePath(testCase.file.Info().SavePath)))
}
}
}
+func TestDriver_TruncateFailed(t *testing.T) {
+ a := assert.New(t)
+ h := Driver{}
+ a.Error(h.Truncate(context.Background(), "TestDriver_TruncateFailed", 0))
+}
+
func TestHandler_Delete(t *testing.T) {
asserts := assert.New(t)
handler := Driver{}
ctx := context.Background()
- filePath := util.RelativePath("test.file")
+ filePath := util.RelativePath("TestHandler_Delete.file")
file, err := os.Create(filePath)
asserts.NoError(err)
_ = file.Close()
- list, err := handler.Delete(ctx, []string{"test.file"})
+ list, err := handler.Delete(ctx, []string{"TestHandler_Delete.file"})
asserts.Equal([]string{}, list)
asserts.NoError(err)
@@ -73,7 +94,7 @@ func TestHandler_Delete(t *testing.T) {
_ = file.Close()
file, _ = os.OpenFile(filePath, os.O_RDWR, os.FileMode(0))
asserts.NoError(err)
- list, err = handler.Delete(ctx, []string{"test.file", "test.notexist"})
+ list, err = handler.Delete(ctx, []string{"TestHandler_Delete.file", "test.notexist"})
file.Close()
asserts.Equal([]string{}, list)
asserts.NoError(err)
@@ -84,7 +105,7 @@ func TestHandler_Delete(t *testing.T) {
file, err = os.Create(filePath)
asserts.NoError(err)
- list, err = handler.Delete(ctx, []string{"test.file"})
+ list, err = handler.Delete(ctx, []string{"TestHandler_Delete.file"})
_ = file.Close()
asserts.Equal([]string{}, list)
asserts.NoError(err)
@@ -116,7 +137,7 @@ func TestHandler_Thumb(t *testing.T) {
asserts := assert.New(t)
handler := Driver{}
ctx := context.Background()
- file, err := os.Create(util.RelativePath("TestHandler_Thumb" + conf.ThumbConfig.FileSuffix))
+ file, err := os.Create(util.RelativePath("TestHandler_Thumb._thumb"))
asserts.NoError(err)
file.Close()
@@ -160,6 +181,25 @@ func TestHandler_Source(t *testing.T) {
asserts.Contains(sourceURL, "https://cloudreve.org")
}
+ // 下载
+ {
+ file := model.File{
+ Model: gorm.Model{
+ ID: 1,
+ },
+ Name: "test.jpg",
+ }
+ ctx := context.WithValue(ctx, fsctx.FileModelCtx, file)
+ baseURL, err := url.Parse("https://cloudreve.org")
+ asserts.NoError(err)
+ sourceURL, err := handler.Source(ctx, "", *baseURL, 0, true, 0)
+ asserts.NoError(err)
+ asserts.NotEmpty(sourceURL)
+ asserts.Contains(sourceURL, "sign=")
+ asserts.Contains(sourceURL, "download")
+ asserts.Contains(sourceURL, "https://cloudreve.org")
+ }
+
// 无法获取上下文
{
baseURL, err := url.Parse("https://cloudreve.org")
@@ -241,10 +281,29 @@ func TestHandler_GetDownloadURL(t *testing.T) {
func TestHandler_Token(t *testing.T) {
asserts := assert.New(t)
- handler := Driver{}
+ handler := Driver{
+ Policy: &model.Policy{},
+ }
ctx := context.Background()
- _, err := handler.Token(ctx, 10, "123")
+ upSession := &serializer.UploadSession{SavePath: "TestHandler_Token"}
+ _, err := handler.Token(ctx, 10, upSession, &fsctx.FileStream{})
asserts.NoError(err)
+
+ file, _ := os.Create("TestHandler_Token")
+ defer func() {
+ file.Close()
+ os.Remove("TestHandler_Token")
+ }()
+
+ _, err = handler.Token(ctx, 10, upSession, &fsctx.FileStream{})
+ asserts.Error(err)
+ asserts.Contains(err.Error(), "already exist")
+}
+
+func TestDriver_CancelToken(t *testing.T) {
+ a := assert.New(t)
+ handler := Driver{}
+ a.NoError(handler.CancelToken(context.Background(), &serializer.UploadSession{}))
}
func TestDriver_List(t *testing.T) {
diff --git a/pkg/filesystem/driver/onedrive/api.go b/pkg/filesystem/driver/onedrive/api.go
index c9e7ff27..5dc2ec99 100644
--- a/pkg/filesystem/driver/onedrive/api.go
+++ b/pkg/filesystem/driver/onedrive/api.go
@@ -1,7 +1,6 @@
package onedrive
import (
- "bytes"
"context"
"encoding/json"
"errors"
@@ -18,7 +17,10 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
+ "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
+ "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
+ "github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)
@@ -29,7 +31,8 @@ const (
// ChunkSize 服务端中转分片上传分片大小
ChunkSize uint64 = 10 * 1024 * 1024
// ListRetry 列取请求重试次数
- ListRetry = 1
+ ListRetry = 1
+ chunkRetrySleep = time.Second * 5
)
// GetSourcePath 获取文件的绝对路径
@@ -219,28 +222,21 @@ func (client *Client) GetUploadSessionStatus(ctx context.Context, uploadURL stri
}
// UploadChunk 上传分片
-func (client *Client) UploadChunk(ctx context.Context, uploadURL string, chunk *Chunk) (*UploadSessionResponse, error) {
+func (client *Client) UploadChunk(ctx context.Context, uploadURL string, content io.Reader, current *chunk.ChunkGroup) (*UploadSessionResponse, error) {
res, err := client.request(
- ctx, "PUT", uploadURL, bytes.NewReader(chunk.Data[0:chunk.ChunkSize]),
- request.WithContentLength(int64(chunk.ChunkSize)),
+ ctx, "PUT", uploadURL, content,
+ request.WithContentLength(current.Length()),
request.WithHeader(http.Header{
- "Content-Range": {fmt.Sprintf("bytes %d-%d/%d", chunk.Offset, chunk.Offset+chunk.ChunkSize-1, chunk.Total)},
+ "Content-Range": {current.RangeHeader()},
}),
request.WithoutHeader([]string{"Authorization", "Content-Type"}),
request.WithTimeout(time.Duration(300)*time.Second),
)
if err != nil {
- // 如果重试次数小于限制，5秒后重试
- if chunk.Retried < model.GetIntSetting("onedrive_chunk_retries", 1) {
- chunk.Retried++
- util.Log().Debug("分片偏移%d上传失败[%s]，5秒钟后重试", chunk.Offset, err)
- time.Sleep(time.Duration(5) * time.Second)
- return client.UploadChunk(ctx, uploadURL, chunk)
- }
- return nil, err
+ return nil, fmt.Errorf("failed to upload OneDrive chunk #%d: %w", current.Index(), err)
}
- if chunk.IsLast() {
+ if current.IsLast() {
return nil, nil
}
@@ -281,46 +277,24 @@ func (client *Client) Upload(ctx context.Context, file fsctx.FileHeader) error {
return err
}
- offset := 0
- chunkNum := size / int(ChunkSize)
- if size%int(ChunkSize) != 0 {
- chunkNum++
- }
-
- chunkData := make([]byte, ChunkSize)
-
- for i := 0; i < chunkNum; i++ {
- select {
- case <-ctx.Done():
- util.Log().Debug("OneDrive 客户端取消")
- return ErrClientCanceled
- default:
- // 分块
- chunkSize := int(ChunkSize)
- if size-offset < chunkSize {
- chunkSize = size - offset
- }
-
- // 因为后面需要错误重试，这里要把分片内容读到内存中
- chunkContent := chunkData[:chunkSize]
- _, err := io.ReadFull(file, chunkContent)
+ // Initial chunk groups
+ chunks := chunk.NewChunkGroup(file, client.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
+ Max: model.GetIntSetting("chunk_retries", 5),
+ Sleep: chunkRetrySleep,
+ }, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
- chunk := Chunk{
- Offset: offset,
- ChunkSize: chunkSize,
- Total: size,
- Data: chunkContent,
- }
+ uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
+ _, err := client.UploadChunk(ctx, uploadURL, content, current)
+ return err
+ }
- // 上传
- _, err = client.UploadChunk(ctx, uploadURL, &chunk)
- if err != nil {
- return err
- }
- offset += chunkSize
+ // upload chunks
+ for chunks.Next() {
+ if err := chunks.Process(uploadFunc); err != nil {
+ return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
}
-
}
+
return nil
}
@@ -349,16 +323,6 @@ func (client *Client) SimpleUpload(ctx context.Context, dst string, body io.Read
request.WithTimeout(time.Duration(150)*time.Second),
)
if err != nil {
- retried := 0
- if v, ok := ctx.Value(fsctx.RetryCtx).(int); ok {
- retried = v
- }
- if retried < model.GetIntSetting("onedrive_chunk_retries", 1) {
- retried++
- util.Log().Debug("文件[%s]上传失败[%s]，5秒钟后重试", dst, err)
- time.Sleep(time.Duration(5) * time.Second)
- return client.SimpleUpload(context.WithValue(ctx, fsctx.RetryCtx, retried), dst, body, size, opts...)
- }
return nil, err
}
@@ -487,9 +451,9 @@ func (client *Client) GetThumbURL(ctx context.Context, dst string, w, h uint) (s
// MonitorUpload 监控客户端分片上传进度
func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size uint64, ttl int64) {
// 回调完成通知chan
- callbackChan := make(chan bool)
- callbackSignal.Store(callbackKey, callbackChan)
- defer callbackSignal.Delete(callbackKey)
+ callbackChan := mq.GlobalMQ.Subscribe(callbackKey, 1)
+ defer mq.GlobalMQ.Unsubscribe(callbackKey, callbackChan)
+
timeout := model.GetIntSetting("onedrive_monitor_timeout", 600)
interval := model.GetIntSetting("onedrive_callback_check", 20)
@@ -514,16 +478,16 @@ func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size ui
if resErr, ok := err.(*RespError); ok {
if resErr.APIError.Code == "itemNotFound" {
util.Log().Debug("上传会话已完成，稍后检查回调")
- time.Sleep(time.Duration(interval) * time.Second)
- util.Log().Debug("开始检查回调")
- _, ok := cache.Get("callback_" + callbackKey)
- if ok {
+ select {
+ case <-time.After(time.Duration(interval) * time.Second):
util.Log().Warning("未发送回调，删除文件")
cache.Deletes([]string{callbackKey}, "callback_")
_, err = client.Delete(context.Background(), []string{path})
if err != nil {
util.Log().Warning("无法删除未回调的文件，%s", err)
}
+ case <-callbackChan:
+ util.Log().Debug("客户端完成回调")
}
return
}
@@ -560,15 +524,6 @@ func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size ui
}
}
-// FinishCallback 向Monitor发送回调结束信号
-func FinishCallback(key string) {
- if signal, ok := callbackSignal.Load(key); ok {
- if signalChan, ok := signal.(chan bool); ok {
- close(signalChan)
- }
- }
-}
-
func sysError(err error) *RespError {
return &RespError{APIError: APIError{
Code: "system",
diff --git a/pkg/filesystem/driver/onedrive/api_test.go b/pkg/filesystem/driver/onedrive/api_test.go
index 02fb2941..fb6393db 100644
--- a/pkg/filesystem/driver/onedrive/api_test.go
+++ b/pkg/filesystem/driver/onedrive/api_test.go
@@ -4,6 +4,11 @@ import (
"context"
"errors"
"fmt"
+ "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
+ "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
+ "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
+ "github.com/cloudreve/Cloudreve/v3/pkg/mq"
+ "io"
"io/ioutil"
"net/http"
"strings"
@@ -12,7 +17,6 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
- "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
@@ -307,6 +311,31 @@ func TestClient_Meta(t *testing.T) {
asserts.NotNil(res)
asserts.Equal("123321", res.Name)
}
+
+ // 返回正常, 使用资源id
+ {
+ client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
+ clientMock := ClientMock{}
+ clientMock.On(
+ "Request",
+ "GET",
+ testMock.Anything,
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: nil,
+ Response: &http.Response{
+ StatusCode: 200,
+ Body: ioutil.NopCloser(strings.NewReader(`{"name":"123321"}`)),
+ },
+ })
+ client.Request = clientMock
+ res, err := client.Meta(context.Background(), "123321", "123")
+ clientMock.AssertExpectations(t)
+ asserts.NoError(err)
+ asserts.NotNil(res)
+ asserts.Equal("123321", res.Name)
+ }
}
func TestClient_CreateUploadSession(t *testing.T) {
@@ -442,9 +471,11 @@ func TestClient_UploadChunk(t *testing.T) {
client, _ := NewClient(&model.Policy{})
client.Credential.AccessToken = "AccessToken"
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
+ cg := chunk.NewChunkGroup(&fsctx.FileStream{Size: 15}, 10, &backoff.ConstantBackoff{}, false)
// 非最后分片，正常
{
+ cg.Next()
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
clientMock.On(
@@ -453,6 +484,10 @@ func TestClient_UploadChunk(t *testing.T) {
"http://dev.com",
testMock.Anything,
testMock.Anything,
+ testMock.Anything,
+ testMock.Anything,
+ testMock.Anything,
+ testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
@@ -461,13 +496,7 @@ func TestClient_UploadChunk(t *testing.T) {
},
})
client.Request = clientMock
- res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{
- Offset: 0,
- ChunkSize: 10,
- Total: 100,
- Retried: 0,
- Data: []byte("12313121231312"),
- })
+ res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("1234567890"), cg)
clientMock.AssertExpectations(t)
asserts.NoError(err)
asserts.Equal("http://dev.com/2", res.UploadURL)
@@ -491,13 +520,7 @@ func TestClient_UploadChunk(t *testing.T) {
},
})
client.Request = clientMock
- res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{
- Offset: 0,
- ChunkSize: 10,
- Total: 100,
- Retried: 0,
- Data: []byte("12313112313122"),
- })
+ res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("1234567890"), cg)
clientMock.AssertExpectations(t)
asserts.Error(err)
asserts.Nil(res)
@@ -505,6 +528,7 @@ func TestClient_UploadChunk(t *testing.T) {
// 最后分片，正常
{
+ cg.Next()
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
clientMock.On(
@@ -521,53 +545,26 @@ func TestClient_UploadChunk(t *testing.T) {
},
})
client.Request = clientMock
- res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{
- Offset: 95,
- ChunkSize: 5,
- Total: 100,
- Retried: 0,
- Data: []byte("1231312"),
- })
+ res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("12345"), cg)
clientMock.AssertExpectations(t)
asserts.NoError(err)
asserts.Nil(res)
}
- // 最后分片，第一次失败，重试后成功
+ // 最后分片，失败
{
- cache.Set("setting_onedrive_chunk_retries", "1", 0)
+ cache.Set("setting_chunk_retries", "1", 0)
client.Credential.ExpiresIn = 0
go func() {
time.Sleep(time.Duration(2) * time.Second)
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
}()
clientMock := ClientMock{}
- clientMock.On(
- "Request",
- "PUT",
- "http://dev.com",
- testMock.Anything,
- testMock.Anything,
- ).Return(&request.Response{
- Err: nil,
- Response: &http.Response{
- StatusCode: 200,
- Body: ioutil.NopCloser(strings.NewReader(`???`)),
- },
- })
client.Request = clientMock
- chunk := &Chunk{
- Offset: 95,
- ChunkSize: 5,
- Total: 100,
- Retried: 0,
- Data: []byte("1231312"),
- }
- res, err := client.UploadChunk(context.Background(), "http://dev.com", chunk)
+ res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("12345"), cg)
clientMock.AssertExpectations(t)
- asserts.NoError(err)
+ asserts.Error(err)
asserts.Nil(res)
- asserts.EqualValues(1, chunk.Retried)
}
}
@@ -576,16 +573,21 @@ func TestClient_Upload(t *testing.T) {
client, _ := NewClient(&model.Policy{})
client.Credential.AccessToken = "AccessToken"
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
- ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
+ ctx := context.Background()
+ cache.Set("setting_chunk_retries", "1", 0)
+ cache.Set("setting_use_temp_chunk_buffer", "false", 0)
// 小文件，简单上传，失败
{
client.Credential.ExpiresIn = 0
- err := client.Upload(ctx, "123.jpg", 3, strings.NewReader("123"))
+ err := client.Upload(ctx, &fsctx.FileStream{
+ Size: 5,
+ File: io.NopCloser(strings.NewReader("12345")),
+ })
asserts.Error(err)
}
- // 上下文取消
+ // 无法创建分片会话
{
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
@@ -598,20 +600,20 @@ func TestClient_Upload(t *testing.T) {
).Return(&request.Response{
Err: nil,
Response: &http.Response{
- StatusCode: 200,
+ StatusCode: 400,
Body: ioutil.NopCloser(strings.NewReader(`{"uploadUrl":"123321"}`)),
},
})
client.Request = clientMock
- ctx, cancel := context.WithCancel(context.Background())
- cancel()
- err := client.Upload(ctx, "123.jpg", 15*1024*1024, strings.NewReader("123"))
+ err := client.Upload(context.Background(), &fsctx.FileStream{
+ Size: SmallFileSize + 1,
+ File: io.NopCloser(strings.NewReader("12345")),
+ })
clientMock.AssertExpectations(t)
asserts.Error(err)
- asserts.Equal(ErrClientCanceled, err)
}
- // 无法创建分片会话
+ // 分片上传失败
{
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
@@ -621,6 +623,19 @@ func TestClient_Upload(t *testing.T) {
testMock.Anything,
testMock.Anything,
testMock.Anything,
+ ).Return(&request.Response{
+ Err: nil,
+ Response: &http.Response{
+ StatusCode: 200,
+ Body: ioutil.NopCloser(strings.NewReader(`{"uploadUrl":"123321"}`)),
+ },
+ })
+ clientMock.On(
+ "Request",
+ "PUT",
+ testMock.Anything,
+ testMock.Anything,
+ testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
@@ -629,9 +644,13 @@ func TestClient_Upload(t *testing.T) {
},
})
client.Request = clientMock
- err := client.Upload(context.Background(), "123.jpg", 15*1024*1024, strings.NewReader("123"))
+ err := client.Upload(context.Background(), &fsctx.FileStream{
+ Size: SmallFileSize + 1,
+ File: io.NopCloser(strings.NewReader("12345")),
+ })
clientMock.AssertExpectations(t)
asserts.Error(err)
+ asserts.Contains(err.Error(), "failed to upload chunk")
}
}
@@ -641,9 +660,9 @@ func TestClient_SimpleUpload(t *testing.T) {
client, _ := NewClient(&model.Policy{})
client.Credential.AccessToken = "AccessToken"
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
- cache.Set("setting_onedrive_chunk_retries", "1", 0)
+ cache.Set("setting_chunk_retries", "1", 0)
- // 请求失败，并重试
+ // 请求失败
{
client.Credential.ExpiresIn = 0
res, err := client.SimpleUpload(context.Background(), "123.jpg", strings.NewReader("123"), 3)
@@ -651,7 +670,6 @@ func TestClient_SimpleUpload(t *testing.T) {
asserts.Nil(res)
}
- cache.Set("setting_onedrive_chunk_retries", "0", 0)
// 返回未知响应
{
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
@@ -988,7 +1006,7 @@ func TestClient_MonitorUpload(t *testing.T) {
asserts.NotPanics(func() {
go func() {
time.Sleep(time.Duration(1) * time.Second)
- FinishCallback("key")
+ mq.GlobalMQ.Publish("key", mq.Message{})
}()
client.MonitorUpload("url", "key", "path", 10, 10)
})
diff --git a/pkg/filesystem/driver/onedrive/handler.go b/pkg/filesystem/driver/onedrive/handler.go
index 2fa85b3a..8e8dacb5 100644
--- a/pkg/filesystem/driver/onedrive/handler.go
+++ b/pkg/filesystem/driver/onedrive/handler.go
@@ -226,16 +226,6 @@ func (handler Driver) replaceSourceHost(origin string) (string, error) {
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
fileInfo := file.Info()
- // 如果小于4MB，则由服务端中转
- if fileInfo.Size <= SmallFileSize {
- return nil, nil
- }
-
- // 生成回调地址
- siteURL := model.GetSiteURL()
- apiBaseURI, _ := url.Parse("/api/v3/callback/onedrive/finish/" + uploadSession.Key)
- apiURL := siteURL.ResolveReference(apiBaseURI)
-
uploadURL, err := handler.Client.CreateUploadSession(ctx, fileInfo.SavePath, WithConflictBehavior("fail"))
if err != nil {
return nil, err
@@ -244,13 +234,15 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
// 监控回调及上传
go handler.Client.MonitorUpload(uploadURL, uploadSession.Key, fileInfo.SavePath, fileInfo.Size, ttl)
+ uploadSession.UploadURL = uploadURL
return &serializer.UploadCredential{
- Policy: uploadURL,
- Token: apiURL.String(),
+ SessionID: uploadSession.Key,
+ ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
+ UploadURLs: []string{uploadURL},
}, nil
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
- return nil
+ return handler.Client.DeleteUploadSession(ctx, uploadSession.UploadURL)
}
diff --git a/pkg/filesystem/driver/onedrive/handler_test.go b/pkg/filesystem/driver/onedrive/handler_test.go
index c2b00ae1..7700e7af 100644
--- a/pkg/filesystem/driver/onedrive/handler_test.go
+++ b/pkg/filesystem/driver/onedrive/handler_test.go
@@ -4,6 +4,9 @@ import (
"context"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
+ "github.com/cloudreve/Cloudreve/v3/pkg/mq"
+ "github.com/cloudreve/Cloudreve/v3/pkg/serializer"
+ "github.com/jinzhu/gorm"
"io"
"io/ioutil"
"net/http"
@@ -12,51 +15,23 @@ import (
"testing"
"time"
- "github.com/DATA-DOG/go-sqlmock"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
- "github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
)
func TestDriver_Token(t *testing.T) {
asserts := assert.New(t)
- handler := Driver{
- Policy: &model.Policy{
- AccessKey: "ak",
- SecretKey: "sk",
- BucketName: "test",
- Server: "test.com",
- },
- }
-
- // 无法获取文件路径
- {
- ctx := context.WithValue(context.Background(), fsctx.FileSizeCtx, uint64(10))
- res, err := handler.Token(ctx, 10, "key", nil)
- asserts.Error(err)
- asserts.Equal(serializer.UploadCredential{}, res)
- }
-
- // 无法获取文件大小
- {
- ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
- res, err := handler.Token(ctx, 10, "key", nil)
- asserts.Error(err)
- asserts.Equal(serializer.UploadCredential{}, res)
- }
-
- // 小文件成功
- {
- ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
- ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(10))
- res, err := handler.Token(ctx, 10, "key", nil)
- asserts.NoError(err)
- asserts.Equal(serializer.UploadCredential{}, res)
- }
+ h, _ := NewDriver(&model.Policy{
+ AccessKey: "ak",
+ SecretKey: "sk",
+ BucketName: "test",
+ Server: "test.com",
+ })
+ handler := h.(Driver)
// 分片上传 失败
{
@@ -78,11 +53,9 @@ func TestDriver_Token(t *testing.T) {
},
})
handler.Client.Request = clientMock
- ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
- ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(20*1024*1024))
- res, err := handler.Token(ctx, 10, "key", nil)
+ res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
asserts.Error(err)
- asserts.Equal(serializer.UploadCredential{}, res)
+ asserts.Nil(res)
}
// 分片上传 成功
@@ -108,15 +81,13 @@ func TestDriver_Token(t *testing.T) {
},
})
handler.Client.Request = clientMock
- ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
- ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(20*1024*1024))
go func() {
time.Sleep(time.Duration(1) * time.Second)
- FinishCallback("key")
+ mq.GlobalMQ.Publish("TestDriver_Token", mq.Message{})
}()
- res, err := handler.Token(ctx, 10, "key", nil)
+ res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{Key: "TestDriver_Token"}, &fsctx.FileStream{})
asserts.NoError(err)
- asserts.Equal("123321", res.Policy)
+ asserts.Equal("123321", res.UploadURLs[0])
}
}
@@ -295,12 +266,8 @@ func TestDriver_Thumb(t *testing.T) {
// 失败
{
ctx := context.WithValue(context.Background(), fsctx.ThumbSizeCtx, [2]uint{10, 20})
- ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{})
- mock.ExpectBegin()
- mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
- mock.ExpectCommit()
+ ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{PicInfo: "1,1", Model: gorm.Model{ID: 1}})
res, err := handler.Thumb(ctx, "123.jpg")
- asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
asserts.Empty(res.URL)
}
@@ -308,7 +275,6 @@ func TestDriver_Thumb(t *testing.T) {
// 上下文错误
{
_, err := handler.Thumb(context.Background(), "123.jpg")
- asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
}
}
@@ -329,7 +295,6 @@ func TestDriver_Delete(t *testing.T) {
// 失败
{
_, err := handler.Delete(context.Background(), []string{"1"})
- asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
}
@@ -350,7 +315,7 @@ func TestDriver_Put(t *testing.T) {
// 失败
{
- err := handler.Put(context.Background(), ioutil.NopCloser(strings.NewReader("")), "dst", 0)
+ err := handler.Put(context.Background(), &fsctx.FileStream{})
asserts.Error(err)
}
}
@@ -418,3 +383,55 @@ func TestDriver_Get(t *testing.T) {
asserts.NoError(err)
asserts.Equal("123", string(content))
}
+
+func TestDriver_replaceSourceHost(t *testing.T) {
+ tests := []struct {
+ name string
+ origin string
+ cdn string
+ want string
+ wantErr bool
+ }{
+ {"TestNoReplace", "http://1dr.ms/download.aspx?123456", "", "http://1dr.ms/download.aspx?123456", false},
+ {"TestReplaceCorrect", "http://1dr.ms/download.aspx?123456", "https://test.com:8080", "https://test.com:8080/download.aspx?123456", false},
+ {"TestCdnFormatError", "http://1dr.ms/download.aspx?123456", string([]byte{0x7f}), "", true},
+ {"TestSrcFormatError", string([]byte{0x7f}), "https://test.com:8080", "", true},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ policy := &model.Policy{}
+ policy.OptionsSerialized.OdProxy = tt.cdn
+ handler := Driver{
+ Policy: policy,
+ }
+ got, err := handler.replaceSourceHost(tt.origin)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("replaceSourceHost() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if got != tt.want {
+ t.Errorf("replaceSourceHost() got = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestDriver_CancelToken(t *testing.T) {
+ asserts := assert.New(t)
+ handler := Driver{
+ Policy: &model.Policy{
+ AccessKey: "ak",
+ SecretKey: "sk",
+ BucketName: "test",
+ Server: "test.com",
+ },
+ }
+ handler.Client, _ = NewClient(&model.Policy{})
+ handler.Client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
+
+ // 失败
+ {
+ err := handler.CancelToken(context.Background(), &serializer.UploadSession{})
+ asserts.Error(err)
+ }
+}
diff --git a/pkg/filesystem/driver/onedrive/handller_test.go b/pkg/filesystem/driver/onedrive/handller_test.go
deleted file mode 100644
index 147a5477..00000000
--- a/pkg/filesystem/driver/onedrive/handller_test.go
+++ /dev/null
@@ -1,38 +0,0 @@
-package onedrive
-
-import (
- model "github.com/cloudreve/Cloudreve/v3/models"
- "testing"
-)
-
-func TestDriver_replaceSourceHost(t *testing.T) {
- tests := []struct {
- name string
- origin string
- cdn string
- want string
- wantErr bool
- }{
- {"TestNoReplace", "http://1dr.ms/download.aspx?123456", "", "http://1dr.ms/download.aspx?123456", false},
- {"TestReplaceCorrect", "http://1dr.ms/download.aspx?123456", "https://test.com:8080", "https://test.com:8080/download.aspx?123456", false},
- {"TestCdnFormatError", "http://1dr.ms/download.aspx?123456", string([]byte{0x7f}), "", true},
- {"TestSrcFormatError", string([]byte{0x7f}), "https://test.com:8080", "", true},
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- policy := &model.Policy{}
- policy.OptionsSerialized.OdProxy = tt.cdn
- handler := Driver{
- Policy: policy,
- }
- got, err := handler.replaceSourceHost(tt.origin)
- if (err != nil) != tt.wantErr {
- t.Errorf("replaceSourceHost() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- if got != tt.want {
- t.Errorf("replaceSourceHost() got = %v, want %v", got, tt.want)
- }
- })
- }
-}
diff --git a/pkg/filesystem/driver/onedrive/types.go b/pkg/filesystem/driver/onedrive/types.go
index 9dc14fa4..2a4307f2 100644
--- a/pkg/filesystem/driver/onedrive/types.go
+++ b/pkg/filesystem/driver/onedrive/types.go
@@ -3,7 +3,6 @@ package onedrive
import (
"encoding/gob"
"net/url"
- "sync"
)
// RespError 接口返回错误
@@ -99,15 +98,6 @@ type ListResponse struct {
Context string `json:"@odata.context"`
}
-// Chunk 文件分片
-type Chunk struct {
- Offset int
- ChunkSize int
- Total int
- Retried int
- Data []byte
-}
-
// oauthEndpoint OAuth接口地址
type oauthEndpoint struct {
token url.URL
@@ -143,10 +133,3 @@ type Site struct {
func init() {
gob.Register(Credential{})
}
-
-// IsLast 返回是否为最后一个分片
-func (chunk *Chunk) IsLast() bool {
- return chunk.Total-chunk.Offset == chunk.ChunkSize
-}
-
-var callbackSignal sync.Map
diff --git a/pkg/filesystem/driver/oss/callback_test.go b/pkg/filesystem/driver/oss/callback_test.go
deleted file mode 100644
index 1d293414..00000000
--- a/pkg/filesystem/driver/oss/callback_test.go
+++ /dev/null
@@ -1,188 +0,0 @@
-package oss
-
-import (
- "io/ioutil"
- "net/http"
- "net/url"
- "strings"
- "testing"
-
- "github.com/cloudreve/Cloudreve/v3/pkg/cache"
- "github.com/stretchr/testify/assert"
-)
-
-func TestGetPublicKey(t *testing.T) {
- asserts := assert.New(t)
- testCases := []struct {
- Request http.Request
- ResNil bool
- Error bool
- }{
- // Header解码失败
- {
- Request: http.Request{
- Header: http.Header{
- "X-Oss-Pub-Key-Url": {"中文"},
- },
- },
- ResNil: true,
- Error: true,
- },
- // 公钥URL无效
- {
- Request: http.Request{
- Header: http.Header{
- "X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9wb3JuaHViLmNvbQ=="},
- },
- },
- ResNil: true,
- Error: true,
- },
- // 请求失败
- {
- Request: http.Request{
- Header: http.Header{
- "X-Oss-Pub-Key-Url": {"aHR0cDovL2dvc3NwdWJsaWMuYWxpY2RuLmNvbS8yMzQyMzQ="},
- },
- },
- ResNil: true,
- Error: true,
- },
- // 成功
- {
- Request: http.Request{
- Header: http.Header{
- "X-Oss-Pub-Key-Url": {"aHR0cDovL2dvc3NwdWJsaWMuYWxpY2RuLmNvbS9jYWxsYmFja19wdWJfa2V5X3YxLnBlbQ=="},
- },
- },
- ResNil: false,
- Error: false,
- },
- }
-
- for i, testCase := range testCases {
- asserts.NoError(cache.Deletes([]string{"oss_public_key"}, ""))
- res, err := GetPublicKey(&testCase.Request)
- if testCase.Error {
- asserts.Error(err, "Test Case #%d", i)
- } else {
- asserts.NoError(err, "Test Case #%d", i)
- }
- if testCase.ResNil {
- asserts.Empty(res, "Test Case #%d", i)
- } else {
- asserts.NotEmpty(res, "Test Case #%d", i)
- }
- }
-
- // 测试缓存
- asserts.NoError(cache.Set("oss_public_key", []byte("123"), 0))
- res, err := GetPublicKey(nil)
- asserts.NoError(err)
- asserts.Equal([]byte("123"), res)
-}
-
-func TestVerifyCallbackSignature(t *testing.T) {
- asserts := assert.New(t)
- testPubKey := `-----BEGIN PUBLIC KEY-----
-MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKs/JBGzwUB2aVht4crBx3oIPBLNsjGs
-C0fTXv+nvlmklvkcolvpvXLTjaxUHR3W9LXxQ2EHXAJfCB+6H2YF1k8CAwEAAQ==
------END PUBLIC KEY-----
-`
-
- // 成功
- {
- asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
- r := http.Request{
- URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
- Header: map[string][]string{
- "Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
- "X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
- },
- Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
- }
- asserts.NoError(VerifyCallbackSignature(&r))
- }
-
- // 签名错误
- {
- asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
- r := http.Request{
- URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
- Header: map[string][]string{
- "Authorization": {"e3LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
- "X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
- },
- Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
- }
- asserts.Error(VerifyCallbackSignature(&r))
- }
-
- // GetPubKey 失败
- {
- asserts.NoError(cache.Deletes([]string{"oss_public_key"}, ""))
- r := http.Request{
- URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
- Header: map[string][]string{
- "Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
- },
- Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
- }
- asserts.Error(VerifyCallbackSignature(&r))
- }
-
- // getRequestMD5 失败
- {
- asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
- r := http.Request{
- URL: &url.URL{Path: "%测试"},
- Header: map[string][]string{
- "Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
- "X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
- },
- Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
- }
- asserts.Error(VerifyCallbackSignature(&r))
- }
-
- // 无 Authorization 头
- {
- asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
- r := http.Request{
- URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
- Header: map[string][]string{
- "X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
- },
- Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
- }
- asserts.Error(VerifyCallbackSignature(&r))
- }
-
- // pub block 不存在
- {
- asserts.NoError(cache.Set("oss_public_key", []byte(""), 0))
- r := http.Request{
- URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
- Header: map[string][]string{
- "Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
- "X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
- },
- Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
- }
- asserts.Error(VerifyCallbackSignature(&r))
- }
-
- // ParsePKIXPublicKey出错
- {
- asserts.NoError(cache.Set("oss_public_key", []byte("-----BEGIN PUBLIC KEY-----\n-----END PUBLIC KEY-----"), 0))
- r := http.Request{
- URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
- Header: map[string][]string{
- "Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
- "X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
- },
- Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
- }
- asserts.Error(VerifyCallbackSignature(&r))
- }
-}
diff --git a/pkg/filesystem/driver/oss/handler.go b/pkg/filesystem/driver/oss/handler.go
index bbe3fc18..b0b55d29 100644
--- a/pkg/filesystem/driver/oss/handler.go
+++ b/pkg/filesystem/driver/oss/handler.go
@@ -2,8 +2,6 @@ package oss
import (
"context"
- "crypto/hmac"
- "crypto/sha1"
"encoding/base64"
"encoding/json"
"errors"
@@ -15,8 +13,10 @@ import (
"strings"
"time"
- "github.com/aliyun/aliyun-oss-go-sdk/oss"
+ "github.com/HFO4/aliyun-oss-go-sdk/oss"
model "github.com/cloudreve/Cloudreve/v3/models"
+ "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
+ "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
@@ -48,17 +48,25 @@ type Driver struct {
type key int
const (
+ chunkRetrySleep = time.Duration(5) * time.Second
+
+ // MultiPartUploadThreshold 服务端使用分片上传的阈值
+ MultiPartUploadThreshold uint64 = 5 * (1 << 30) // 5GB
// VersionID 文件版本标识
VersionID key = iota
)
-// CORS 创建跨域策略
-func (handler *Driver) CORS() error {
- // 初始化客户端
- if err := handler.InitOSSClient(false); err != nil {
- return err
+func NewDriver(policy *model.Policy) (*Driver, error) {
+ driver := &Driver{
+ Policy: policy,
+ HTTPClient: request.NewClient(),
}
+ return driver, driver.InitOSSClient(false)
+}
+
+// CORS 创建跨域策略
+func (handler *Driver) CORS() error {
return handler.client.SetBucketCORS(handler.Policy.BucketName, []oss.CORSRule{
{
AllowedOrigin: []string{"*"},
@@ -82,39 +90,31 @@ func (handler *Driver) InitOSSClient(forceUsePublicEndpoint bool) error {
return errors.New("存储策略为空")
}
- if handler.client == nil {
- // 决定是否使用内网 Endpoint
- endpoint := handler.Policy.Server
- if handler.Policy.OptionsSerialized.ServerSideEndpoint != "" && !forceUsePublicEndpoint {
- endpoint = handler.Policy.OptionsSerialized.ServerSideEndpoint
- }
-
- // 初始化客户端
- client, err := oss.New(endpoint, handler.Policy.AccessKey, handler.Policy.SecretKey)
- if err != nil {
- return err
- }
- handler.client = client
+ // 决定是否使用内网 Endpoint
+ endpoint := handler.Policy.Server
+ if handler.Policy.OptionsSerialized.ServerSideEndpoint != "" && !forceUsePublicEndpoint {
+ endpoint = handler.Policy.OptionsSerialized.ServerSideEndpoint
+ }
- // 初始化存储桶
- bucket, err := client.Bucket(handler.Policy.BucketName)
- if err != nil {
- return err
- }
- handler.bucket = bucket
+ // 初始化客户端
+ client, err := oss.New(endpoint, handler.Policy.AccessKey, handler.Policy.SecretKey)
+ if err != nil {
+ return err
+ }
+ handler.client = client
+ // 初始化存储桶
+ bucket, err := client.Bucket(handler.Policy.BucketName)
+ if err != nil {
+ return err
}
+ handler.bucket = bucket
return nil
}
// List 列出OSS上的文件
-func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
- // 初始化客户端
- if err := handler.InitOSSClient(false); err != nil {
- return nil, err
- }
-
+func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// 列取文件
base = strings.TrimPrefix(base, "/")
if base != "" {
@@ -181,7 +181,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
}
// Get 获取文件
-func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
+func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 通过VersionID禁止缓存
ctx = context.WithValue(ctx, VersionID, time.Now().UnixNano())
@@ -224,17 +224,12 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// Put 将文件流保存到指定目录
-func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
+func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
fileInfo := file.Info()
- // 初始化客户端
- if err := handler.InitOSSClient(false); err != nil {
- return err
- }
-
// 凭证有效期
- credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
+ credentialTTL := model.GetIntSetting("upload_session_timeout", 3600)
// 是否允许覆盖
overwrite := fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite
@@ -243,23 +238,40 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
oss.ForbidOverWrite(!overwrite),
}
- // 上传文件
- err := handler.bucket.PutObject(fileInfo.SavePath, file, options...)
+ // 小文件直接上传
+ if fileInfo.Size < MultiPartUploadThreshold {
+ return handler.bucket.PutObject(fileInfo.SavePath, file, options...)
+ }
+
+ // 超过阈值时使用分片上传
+ imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...)
if err != nil {
+ return fmt.Errorf("failed to initiate multipart upload: %w", err)
+ }
+
+ chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
+ Max: model.GetIntSetting("chunk_retries", 5),
+ Sleep: chunkRetrySleep,
+ }, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
+
+ uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
+ _, err := handler.bucket.UploadPart(imur, content, current.Length(), current.Index()+1)
return err
}
- return nil
+ for chunks.Next() {
+ if err := chunks.Process(uploadFunc); err != nil {
+ return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
+ }
+ }
+
+ _, err = handler.bucket.CompleteMultipartUpload(imur, oss.CompleteAll("yes"), oss.ForbidOverWrite(!overwrite))
+ return err
}
// Delete 删除一个或多个文件，
// 返回未删除的文件
-func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
- // 初始化客户端
- if err := handler.InitOSSClient(false); err != nil {
- return files, err
- }
-
+func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// 删除文件
delRes, err := handler.bucket.DeleteObjects(files)
@@ -277,7 +289,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// Thumb 获取文件缩略图
-func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
+func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
// 初始化客户端
if err := handler.InitOSSClient(true); err != nil {
return nil, err
@@ -311,7 +323,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
}
// Source 获取外链URL
-func (handler Driver) Source(
+func (handler *Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
@@ -356,7 +368,7 @@ func (handler Driver) Source(
return handler.signSourceURL(ctx, path, ttl, signOptions)
}
-func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64, options []oss.Option) (string, error) {
+func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64, options []oss.Option) (string, error) {
signedURL, err := handler.bucket.SignURL(path, oss.HTTPGet, ttl, options...)
if err != nil {
return "", err
@@ -368,9 +380,6 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
return "", err
}
- // 优先使用https
- finalURL.Scheme = "https"
-
// 公有空间替换掉Key及不支持的头
if !handler.Policy.IsPrivate {
query := finalURL.Query()
@@ -394,7 +403,8 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
}
// Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
+func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
+
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + uploadSession.Key)
@@ -406,61 +416,66 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
CallbackBody: `{"name":${x:fname},"source_name":${object},"size":${size},"pic_info":"${imageInfo.width},${imageInfo.height}"}`,
CallbackBodyType: "application/json",
}
-
- // 上传策略
- savePath := file.Info().SavePath
- postPolicy := UploadPolicy{
- Expiration: time.Now().UTC().Add(time.Duration(ttl) * time.Second).Format(time.RFC3339),
- Conditions: []interface{}{
- map[string]string{"bucket": handler.Policy.BucketName},
- []string{"starts-with", "$key", path.Dir(savePath)},
- },
+ callbackPolicyJSON, err := json.Marshal(callbackPolicy)
+ if err != nil {
+ return nil, fmt.Errorf("failed to encode callback policy: %w", err)
}
+ callbackPolicyEncoded := base64.StdEncoding.EncodeToString(callbackPolicyJSON)
- if handler.Policy.MaxSize > 0 {
- postPolicy.Conditions = append(postPolicy.Conditions,
- []interface{}{"content-length-range", 0, handler.Policy.MaxSize})
+ // 初始化分片上传
+ fileInfo := file.Info()
+ options := []oss.Option{
+ oss.Expires(time.Now().Add(time.Duration(ttl) * time.Second)),
+ oss.ForbidOverWrite(true),
}
-
- return handler.getUploadCredential(ctx, postPolicy, callbackPolicy, ttl, savePath)
-}
-
-func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback CallbackPolicy, TTL int64, savePath string) (*serializer.UploadCredential, error) {
- // 处理回调策略
- callbackPolicyEncoded := ""
- if callback.CallbackURL != "" {
- callbackPolicyJSON, err := json.Marshal(callback)
+ imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...)
+ if err != nil {
+ return nil, fmt.Errorf("failed to initialize multipart upload: %w", err)
+ }
+ uploadSession.UploadID = imur.UploadID
+
+ // 为每个分片签名上传 URL
+ chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false)
+ urls := make([]string, chunks.Num())
+ for chunks.Next() {
+ err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error {
+ signedURL, err := handler.bucket.SignURL(fileInfo.SavePath, oss.HTTPPut, ttl,
+ oss.PartNumber(c.Index()+1),
+ oss.UploadID(imur.UploadID),
+ oss.ContentType("application/octet-stream"))
+ if err != nil {
+ return err
+ }
+
+ urls[c.Index()] = signedURL
+ return nil
+ })
if err != nil {
return nil, err
}
- callbackPolicyEncoded = base64.StdEncoding.EncodeToString(callbackPolicyJSON)
- policy.Conditions = append(policy.Conditions, map[string]string{"callback": callbackPolicyEncoded})
}
- // 编码上传策略
- policyJSON, err := json.Marshal(policy)
+ // 签名完成分片上传的URL
+ completeURL, err := handler.bucket.SignURL(fileInfo.SavePath, oss.HTTPPost, ttl,
+ oss.UploadID(imur.UploadID),
+ oss.Expires(time.Now().Add(time.Duration(ttl)*time.Second)),
+ oss.CompleteAll("yes"),
+ oss.ForbidOverWrite(true),
+ oss.CallbackParam(callbackPolicyEncoded))
if err != nil {
return nil, err
}
- policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
-
- // 签名上传策略
- hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey))
- _, err = io.WriteString(hmacSign, policyEncoded)
- if err != nil {
- return nil, err
- }
- signature := base64.StdEncoding.EncodeToString(hmacSign.Sum(nil))
return &serializer.UploadCredential{
- Policy: fmt.Sprintf("%s:%s", callbackPolicyEncoded, policyEncoded),
- Path: savePath,
- AccessKey: handler.Policy.AccessKey,
- Token: signature,
+ SessionID: uploadSession.Key,
+ ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
+ UploadID: imur.UploadID,
+ UploadURLs: urls,
+ CompleteURL: completeURL,
}, nil
}
// 取消上传凭证
-func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
- return nil
+func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+ return handler.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{UploadID: uploadSession.UploadID, Key: uploadSession.SavePath}, nil)
}
diff --git a/pkg/filesystem/driver/oss/handler_test.go b/pkg/filesystem/driver/oss/handler_test.go
deleted file mode 100644
index d69c9312..00000000
--- a/pkg/filesystem/driver/oss/handler_test.go
+++ /dev/null
@@ -1,354 +0,0 @@
-package oss
-
-import (
- "context"
- "io"
- "io/ioutil"
- "net/http"
- "net/url"
- "strings"
- "testing"
-
- model "github.com/cloudreve/Cloudreve/v3/models"
- "github.com/cloudreve/Cloudreve/v3/pkg/cache"
- "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
- "github.com/cloudreve/Cloudreve/v3/pkg/request"
- "github.com/stretchr/testify/assert"
- testMock "github.com/stretchr/testify/mock"
-)
-
-func TestDriver_InitOSSClient(t *testing.T) {
- asserts := assert.New(t)
- handler := Driver{
- Policy: &model.Policy{
- AccessKey: "ak",
- SecretKey: "sk",
- BucketName: "test",
- Server: "test.com",
- },
- }
-
- // 成功
- {
- asserts.NoError(handler.InitOSSClient(false))
- }
-
- // 使用内网Endpoint
- {
- handler.Policy.OptionsSerialized.ServerSideEndpoint = "endpoint2"
- asserts.NoError(handler.InitOSSClient(false))
- }
-
- // 未指定存储策略
- {
- handler := Driver{}
- asserts.Error(handler.InitOSSClient(false))
- }
-}
-
-func TestDriver_CORS(t *testing.T) {
- asserts := assert.New(t)
- handler := Driver{
- Policy: &model.Policy{
- AccessKey: "ak",
- SecretKey: "sk",
- BucketName: "test",
- Server: "test.com",
- },
- }
-
- // 失败
- {
- asserts.NotPanics(func() {
- handler.CORS()
- })
- }
-}
-
-func TestDriver_Token(t *testing.T) {
- asserts := assert.New(t)
- handler := Driver{
- Policy: &model.Policy{
- AccessKey: "ak",
- SecretKey: "sk",
- BucketName: "test",
- Server: "test.com",
- },
- }
-
- // 成功
- {
- ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
- cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
- res, err := handler.Token(ctx, 10, "key", nil)
- asserts.NoError(err)
- asserts.NotEmpty(res.Policy)
- asserts.NotEmpty(res.Token)
- asserts.Equal(handler.Policy.AccessKey, res.AccessKey)
- asserts.Equal("/123", res.Path)
- }
-
- // 上下文错误
- {
- ctx := context.Background()
- _, err := handler.Token(ctx, 10, "key", nil)
- asserts.Error(err)
- }
-
-}
-
-func TestDriver_Source(t *testing.T) {
- asserts := assert.New(t)
- handler := Driver{
- Policy: &model.Policy{
- AccessKey: "ak",
- SecretKey: "sk",
- BucketName: "test",
- Server: "test.com",
- IsPrivate: true,
- },
- }
-
- // 正常 非下载 无限速
- {
- res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0)
- asserts.NoError(err)
- resURL, err := url.Parse(res)
- asserts.NoError(err)
- query := resURL.Query()
- asserts.NotEmpty(query.Get("Signature"))
- asserts.NotEmpty(query.Get("Expires"))
- asserts.Equal("ak", query.Get("OSSAccessKeyId"))
- }
-
- // 限速 + 下载
- {
- ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"})
- res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 102401)
- asserts.NoError(err)
- resURL, err := url.Parse(res)
- asserts.NoError(err)
- query := resURL.Query()
- asserts.NotEmpty(query.Get("Signature"))
- asserts.NotEmpty(query.Get("Expires"))
- asserts.Equal("ak", query.Get("OSSAccessKeyId"))
- asserts.EqualValues("819208", query.Get("x-oss-traffic-limit"))
- asserts.NotEmpty(query.Get("response-content-disposition"))
- }
-
- // 限速超出范围 + 下载
- {
- ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"})
- res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 10)
- asserts.NoError(err)
- resURL, err := url.Parse(res)
- asserts.NoError(err)
- query := resURL.Query()
- asserts.NotEmpty(query.Get("Signature"))
- asserts.NotEmpty(query.Get("Expires"))
- asserts.Equal("ak", query.Get("OSSAccessKeyId"))
- asserts.EqualValues("819200", query.Get("x-oss-traffic-limit"))
- asserts.NotEmpty(query.Get("response-content-disposition"))
- }
-
- // 限速超出范围 + 下载
- {
- ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"})
- res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 838860801)
- asserts.NoError(err)
- resURL, err := url.Parse(res)
- asserts.NoError(err)
- query := resURL.Query()
- asserts.NotEmpty(query.Get("Signature"))
- asserts.NotEmpty(query.Get("Expires"))
- asserts.Equal("ak", query.Get("OSSAccessKeyId"))
- asserts.EqualValues("838860800", query.Get("x-oss-traffic-limit"))
- asserts.NotEmpty(query.Get("response-content-disposition"))
- }
-
- // 公共空间
- {
- handler.Policy.IsPrivate = false
- res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0)
- asserts.NoError(err)
- resURL, err := url.Parse(res)
- asserts.NoError(err)
- query := resURL.Query()
- asserts.Empty(query.Get("Signature"))
- }
-
- // 正常 指定了CDN域名
- {
- handler.Policy.BaseURL = "https://cqu.edu.cn"
- res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0)
- asserts.NoError(err)
- resURL, err := url.Parse(res)
- asserts.NoError(err)
- query := resURL.Query()
- asserts.Empty(query.Get("Signature"))
- asserts.Contains(resURL.String(), handler.Policy.BaseURL)
- }
-
- // 强制使用公网 Endpoint
- {
- handler.Policy.BaseURL = ""
- handler.Policy.OptionsSerialized.ServerSideEndpoint = "endpoint.com"
- res, err := handler.Source(context.WithValue(context.Background(), fsctx.ForceUsePublicEndpointCtx, false), "/123", url.URL{}, 10, false, 0)
- asserts.NoError(err)
- resURL, err := url.Parse(res)
- asserts.NoError(err)
- query := resURL.Query()
- asserts.Empty(query.Get("Signature"))
- asserts.Contains(resURL.String(), "endpoint.com")
- }
-}
-
-func TestDriver_Thumb(t *testing.T) {
- asserts := assert.New(t)
- handler := Driver{
- Policy: &model.Policy{
- AccessKey: "ak",
- SecretKey: "sk",
- BucketName: "test",
- Server: "test.com",
- },
- }
-
- // 上下文不存在
- {
- ctx := context.Background()
- res, err := handler.Thumb(ctx, "/123.txt")
- asserts.Error(err)
- asserts.Nil(res)
- }
-
- // 成功
- {
- cache.Set("setting_preview_timeout", "60", 0)
- ctx := context.WithValue(context.Background(), fsctx.ThumbSizeCtx, [2]uint{10, 20})
- res, err := handler.Thumb(ctx, "/123.jpg")
- asserts.NoError(err)
- resURL, err := url.Parse(res.URL)
- asserts.NoError(err)
- urlQuery := resURL.Query()
- asserts.Equal("image/resize,m_lfit,h_20,w_10", urlQuery.Get("x-oss-process"))
- }
-}
-
-func TestDriver_Delete(t *testing.T) {
- asserts := assert.New(t)
- handler := Driver{
- Policy: &model.Policy{
- AccessKey: "ak",
- SecretKey: "sk",
- BucketName: "test",
- Server: "oss-cn-shanghai.aliyuncs.com",
- },
- }
-
- // 失败
- {
- res, err := handler.Delete(context.Background(), []string{"1", "2", "3"})
- asserts.Error(err)
- asserts.Equal([]string{"1", "2", "3"}, res)
- }
-}
-
-func TestDriver_Put(t *testing.T) {
- asserts := assert.New(t)
- handler := Driver{
- Policy: &model.Policy{
- AccessKey: "ak",
- SecretKey: "sk",
- BucketName: "test",
- Server: "oss-cn-shanghai.aliyuncs.com",
- },
- }
- cache.Set("setting_upload_credential_timeout", "3600", 0)
- ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
-
- // 失败
- {
- err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("123")), "/123.txt", 3)
- asserts.Error(err)
- }
-}
-
-type ClientMock struct {
- testMock.Mock
-}
-
-func (m ClientMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
- args := m.Called(method, target, body, opts)
- return args.Get(0).(*request.Response)
-}
-
-func TestDriver_Get(t *testing.T) {
- asserts := assert.New(t)
- handler := Driver{
- Policy: &model.Policy{
- AccessKey: "ak",
- SecretKey: "sk",
- BucketName: "test",
- Server: "oss-cn-shanghai.aliyuncs.com",
- },
- HTTPClient: request.NewClient(),
- }
- cache.Set("setting_preview_timeout", "3600", 0)
-
- // 响应失败
- {
- res, err := handler.Get(context.Background(), "123.txt")
- asserts.Error(err)
- asserts.Nil(res)
- }
-
- // 响应成功
- {
- ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Size: 3})
- clientMock := ClientMock{}
- clientMock.On(
- "Request",
- "GET",
- testMock.Anything,
- testMock.Anything,
- testMock.Anything,
- ).Return(&request.Response{
- Err: nil,
- Response: &http.Response{
- StatusCode: 200,
- Body: ioutil.NopCloser(strings.NewReader(`123`)),
- },
- })
- handler.HTTPClient = clientMock
- res, err := handler.Get(ctx, "123.txt")
- clientMock.AssertExpectations(t)
- asserts.NoError(err)
- n, err := res.Seek(0, io.SeekEnd)
- asserts.NoError(err)
- asserts.EqualValues(3, n)
- content, err := ioutil.ReadAll(res)
- asserts.NoError(err)
- asserts.Equal("123", string(content))
- }
-}
-
-func TestDriver_List(t *testing.T) {
- asserts := assert.New(t)
- handler := Driver{
- Policy: &model.Policy{
- AccessKey: "ak",
- SecretKey: "sk",
- BucketName: "test",
- Server: "test.com",
- IsPrivate: true,
- },
- }
-
- // 连接失败
- {
- res, err := handler.List(context.Background(), "/", true)
- asserts.Error(err)
- asserts.Empty(res)
- }
-}
diff --git a/pkg/filesystem/driver/qiniu/handler.go b/pkg/filesystem/driver/qiniu/handler.go
index f453dd7b..22a97d06 100644
--- a/pkg/filesystem/driver/qiniu/handler.go
+++ b/pkg/filesystem/driver/qiniu/handler.go
@@ -2,6 +2,7 @@ package qiniu
import (
"context"
+ "encoding/base64"
"errors"
"fmt"
"net/http"
@@ -16,17 +17,31 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
- "github.com/qiniu/api.v7/v7/auth/qbox"
- "github.com/qiniu/api.v7/v7/storage"
+ "github.com/qiniu/go-sdk/v7/auth/qbox"
+ "github.com/qiniu/go-sdk/v7/storage"
)
// Driver 本地策略适配器
type Driver struct {
Policy *model.Policy
+ mac *qbox.Mac
+ cfg *storage.Config
+ bucket *storage.BucketManager
+}
+
+func NewDriver(policy *model.Policy) *Driver {
+ mac := qbox.NewMac(policy.AccessKey, policy.SecretKey)
+ cfg := &storage.Config{UseHTTPS: true}
+ return &Driver{
+ Policy: policy,
+ mac: mac,
+ cfg: cfg,
+ bucket: storage.NewBucketManager(mac, cfg),
+ }
}
// List 列出给定路径下的文件
-func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
+func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
base = strings.TrimPrefix(base, "/")
if base != "" {
base += "/"
@@ -42,14 +57,8 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
delimiter = "/"
}
- mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
- cfg := storage.Config{
- UseHTTPS: true,
- }
- bucketManager := storage.NewBucketManager(mac, &cfg)
-
for {
- entries, folders, nextMarker, hashNext, err := bucketManager.ListFiles(
+ entries, folders, nextMarker, hashNext, err := handler.bucket.ListFiles(
handler.Policy.BucketName,
base, delimiter, marker, 1000)
if err != nil {
@@ -99,7 +108,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
}
// Get 获取文件
-func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
+func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 给文件名加上随机参数以强制拉取
path = fmt.Sprintf("%s?v=%d", path, time.Now().UnixNano())
@@ -143,17 +152,22 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// Put 将文件流保存到指定目录
-func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
+func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
// 凭证有效期
- credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
+ credentialTTL := model.GetIntSetting("upload_session_timeout", 3600)
// 生成上传策略
fileInfo := file.Info()
+ scope := handler.Policy.BucketName
+ if fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite {
+ scope = fmt.Sprintf("%s:%s", handler.Policy.BucketName, fileInfo.SavePath)
+ }
+
putPolicy := storage.PutPolicy{
// 指定为覆盖策略
- Scope: fmt.Sprintf("%s:%s", handler.Policy.BucketName, fileInfo.SavePath),
+ Scope: scope,
SaveKey: fileInfo.SavePath,
ForceSaveKey: true,
FsizeLimit: int64(fileInfo.Size),
@@ -164,7 +178,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
}
// 生成上传凭证
- token, err := handler.getUploadCredential(ctx, putPolicy, int64(credentialTTL))
+ token, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, int64(credentialTTL), false)
if err != nil {
return err
}
@@ -178,7 +192,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
}
// 开始上传
- err = formUploader.Put(ctx, &ret, token.Token, fileInfo.SavePath, file, int64(fileInfo.Size), &putExtra)
+ err = formUploader.Put(ctx, &ret, token.Credential, fileInfo.SavePath, file, int64(fileInfo.Size), &putExtra)
if err != nil {
return err
}
@@ -188,19 +202,14 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
// Delete 删除一个或多个文件，
// 返回未删除的文件
-func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
+func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// TODO 大于一千个文件需要分批发送
deleteOps := make([]string, 0, len(files))
for _, key := range files {
deleteOps = append(deleteOps, storage.URIDelete(handler.Policy.BucketName, key))
}
- mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
- cfg := storage.Config{
- UseHTTPS: true,
- }
- bucketManager := storage.NewBucketManager(mac, &cfg)
- rets, err := bucketManager.Batch(deleteOps)
+ rets, err := handler.bucket.Batch(deleteOps)
// 处理删除结果
if err != nil {
@@ -217,7 +226,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// Thumb 获取文件缩略图
-func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
+func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
var (
thumbSize = [2]uint{400, 300}
ok = false
@@ -238,7 +247,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
}
// Source 获取外链URL
-func (handler Driver) Source(
+func (handler *Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
@@ -261,12 +270,11 @@ func (handler Driver) Source(
return handler.signSourceURL(ctx, path, ttl), nil
}
-func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64) string {
+func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64) string {
var sourceURL string
if handler.Policy.IsPrivate {
- mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
deadline := time.Now().Add(time.Second * time.Duration(ttl)).Unix()
- sourceURL = storage.MakePrivateURL(mac, handler.Policy.BaseURL, path, deadline)
+ sourceURL = storage.MakePrivateURL(handler.mac, handler.Policy.BaseURL, path, deadline)
} else {
sourceURL = storage.MakePublicURL(handler.Policy.BaseURL, path)
}
@@ -274,19 +282,20 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64)
}
// Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
+func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 创建上传策略
+ fileInfo := file.Info()
putPolicy := storage.PutPolicy{
Scope: handler.Policy.BucketName,
CallbackURL: apiURL.String(),
- CallbackBody: `{"name":"$(fname)","source_name":"$(key)","size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`,
+ CallbackBody: `{"size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`,
CallbackBodyType: "application/json",
- SaveKey: file.Info().SavePath,
+ SaveKey: fileInfo.SavePath,
ForceSaveKey: true,
FsizeLimit: int64(handler.Policy.MaxSize),
}
@@ -295,21 +304,46 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType
}
- return handler.getUploadCredential(ctx, putPolicy, ttl)
+ credential, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, ttl, true)
+ if err != nil {
+ return nil, fmt.Errorf("failed to init parts: %w", err)
+ }
+
+ credential.SessionID = uploadSession.Key
+ credential.ChunkSize = handler.Policy.OptionsSerialized.ChunkSize
+
+ uploadSession.UploadURL = credential.UploadURLs[0]
+ uploadSession.Credential = credential.Credential
+
+ return credential, nil
}
-// getUploadCredential 签名上传策略
-func (handler Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, TTL int64) (*serializer.UploadCredential, error) {
+// getUploadCredential 签名上传策略并创建上传会话
+func (handler *Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, file *fsctx.UploadTaskInfo, TTL int64, resume bool) (*serializer.UploadCredential, error) {
+ // 上传凭证
policy.Expires = uint64(TTL)
- mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
- upToken := policy.UploadToken(mac)
+ upToken := policy.UploadToken(handler.mac)
+
+ // 初始化分片上传
+ resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
+ upHost, err := resumeUploader.UpHost(handler.Policy.AccessKey, handler.Policy.BucketName)
+ if err != nil {
+ return nil, err
+ }
+
+ ret := &storage.InitPartsRet{}
+ if resume {
+ err = resumeUploader.InitParts(ctx, upToken, upHost, handler.Policy.BucketName, file.SavePath, true, ret)
+ }
return &serializer.UploadCredential{
- Token: upToken,
- }, nil
+ UploadURLs: []string{upHost + "/buckets/" + handler.Policy.BucketName + "/objects/" + base64.URLEncoding.EncodeToString([]byte(file.SavePath)) + "/uploads/" + ret.UploadID},
+ Credential: upToken,
+ }, err
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
- return nil
+ resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
+ return resumeUploader.Client.CallWith(ctx, nil, "DELETE", uploadSession.UploadURL, http.Header{"Authorization": {"UpToken " + uploadSession.Credential}}, nil, 0)
}
diff --git a/pkg/filesystem/driver/remote/client.go b/pkg/filesystem/driver/remote/client.go
index ab764f51..2f267eb7 100644
--- a/pkg/filesystem/driver/remote/client.go
+++ b/pkg/filesystem/driver/remote/client.go
@@ -90,9 +90,9 @@ func (c *remoteClient) Upload(ctx context.Context, file fsctx.FileHeader) error
// Initial chunk groups
chunks := chunk.NewChunkGroup(file, c.policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
- Max: model.GetIntSetting("onedrive_chunk_retries", 1),
+ Max: model.GetIntSetting("chunk_retries", 5),
Sleep: chunkRetrySleep,
- })
+ }, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
return c.uploadChunk(ctx, session.Key, current.Index(), content, overwrite, current.Length())
diff --git a/pkg/filesystem/driver/remote/client_test.go b/pkg/filesystem/driver/remote/client_test.go
new file mode 100644
index 00000000..c195521a
--- /dev/null
+++ b/pkg/filesystem/driver/remote/client_test.go
@@ -0,0 +1,262 @@
+package remote
+
+import (
+ "context"
+ "errors"
+ model "github.com/cloudreve/Cloudreve/v3/models"
+ "github.com/cloudreve/Cloudreve/v3/pkg/cache"
+ "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
+ "github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock"
+ "github.com/cloudreve/Cloudreve/v3/pkg/request"
+ "github.com/stretchr/testify/assert"
+ testMock "github.com/stretchr/testify/mock"
+ "io/ioutil"
+ "net/http"
+ "strings"
+ "testing"
+)
+
+func TestNewClient(t *testing.T) {
+ a := assert.New(t)
+ policy := &model.Policy{}
+
+ // 无法解析服务端url
+ {
+ policy.Server = string([]byte{0x7f})
+ c, err := NewClient(policy)
+ a.Error(err)
+ a.Nil(c)
+ }
+
+ // 成功
+ {
+ policy.Server = ""
+ c, err := NewClient(policy)
+ a.NoError(err)
+ a.NotNil(c)
+ }
+}
+
+func TestRemoteClient_Upload(t *testing.T) {
+ a := assert.New(t)
+ c, _ := NewClient(&model.Policy{})
+
+ // 无法创建上传会话
+ {
+ clientMock := requestmock.RequestMock{}
+ c.(*remoteClient).httpClient = &clientMock
+ clientMock.On(
+ "Request",
+ "PUT",
+ "upload",
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: errors.New("error"),
+ })
+ err := c.Upload(context.Background(), &fsctx.FileStream{})
+ a.Error(err)
+ a.Contains(err.Error(), "error")
+ clientMock.AssertExpectations(t)
+ }
+
+ // 分片上传失败，成功删除上传会话
+ {
+ cache.Set("setting_chunk_retries", "1", 0)
+ clientMock := requestmock.RequestMock{}
+ c.(*remoteClient).httpClient = &clientMock
+ clientMock.On(
+ "Request",
+ "PUT",
+ "upload",
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: nil,
+ Response: &http.Response{
+ StatusCode: 200,
+ Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
+ },
+ })
+ clientMock.On(
+ "Request",
+ "POST",
+ testMock.Anything,
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: errors.New("error"),
+ })
+ clientMock.On(
+ "Request",
+ "DELETE",
+ testMock.Anything,
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: nil,
+ Response: &http.Response{
+ StatusCode: 200,
+ Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
+ },
+ })
+ err := c.Upload(context.Background(), &fsctx.FileStream{})
+ a.Error(err)
+ a.Contains(err.Error(), "error")
+ clientMock.AssertExpectations(t)
+ }
+
+ // 分片上传失败，无法删除上传会话
+ {
+ cache.Set("setting_chunk_retries", "1", 0)
+ clientMock := requestmock.RequestMock{}
+ c.(*remoteClient).httpClient = &clientMock
+ clientMock.On(
+ "Request",
+ "PUT",
+ "upload",
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: nil,
+ Response: &http.Response{
+ StatusCode: 200,
+ Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
+ },
+ })
+ clientMock.On(
+ "Request",
+ "POST",
+ testMock.Anything,
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: errors.New("error"),
+ })
+ clientMock.On(
+ "Request",
+ "DELETE",
+ testMock.Anything,
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: errors.New("error2"),
+ Response: &http.Response{
+ StatusCode: 200,
+ Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
+ },
+ })
+ err := c.Upload(context.Background(), &fsctx.FileStream{})
+ a.Error(err)
+ a.Contains(err.Error(), "error")
+ clientMock.AssertExpectations(t)
+ }
+
+ // 成功
+ {
+ cache.Set("setting_chunk_retries", "1", 0)
+ clientMock := requestmock.RequestMock{}
+ c.(*remoteClient).httpClient = &clientMock
+ clientMock.On(
+ "Request",
+ "PUT",
+ "upload",
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: nil,
+ Response: &http.Response{
+ StatusCode: 200,
+ Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
+ },
+ })
+ clientMock.On(
+ "Request",
+ "POST",
+ testMock.Anything,
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Response: &http.Response{
+ StatusCode: 200,
+ Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
+ },
+ })
+ err := c.Upload(context.Background(), &fsctx.FileStream{})
+ a.NoError(err)
+ clientMock.AssertExpectations(t)
+ }
+}
+
+func TestRemoteClient_CreateUploadSessionFailed(t *testing.T) {
+ a := assert.New(t)
+ c, _ := NewClient(&model.Policy{})
+
+ clientMock := requestmock.RequestMock{}
+ c.(*remoteClient).httpClient = &clientMock
+ clientMock.On(
+ "Request",
+ "PUT",
+ "upload",
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: nil,
+ Response: &http.Response{
+ StatusCode: 200,
+ Body: ioutil.NopCloser(strings.NewReader(`{"code":500,"msg":"error"}`)),
+ },
+ })
+ err := c.Upload(context.Background(), &fsctx.FileStream{})
+ a.Error(err)
+ a.Contains(err.Error(), "error")
+ clientMock.AssertExpectations(t)
+}
+
+func TestRemoteClient_UploadChunkFailed(t *testing.T) {
+ a := assert.New(t)
+ c, _ := NewClient(&model.Policy{})
+
+ clientMock := requestmock.RequestMock{}
+ c.(*remoteClient).httpClient = &clientMock
+ clientMock.On(
+ "Request",
+ "POST",
+ testMock.Anything,
+ testMock.Anything,
+ testMock.Anything,
+ ).Return(&request.Response{
+ Err: nil,
+ Response: &http.Response{
+ StatusCode: 200,
+ Body: ioutil.NopCloser(strings.NewReader(`{"code":500,"msg":"error"}`)),
+ },
+ })
+ err := c.(*remoteClient).uploadChunk(context.Background(), "", 0, strings.NewReader(""), false, 0)
+ a.Error(err)
+ a.Contains(err.Error(), "error")
+ clientMock.AssertExpectations(t)
+}
+
+func TestRemoteClient_GetUploadURL(t *testing.T) {
+ a := assert.New(t)
+ c, _ := NewClient(&model.Policy{})
+
+ // url 解析失败
+ {
+ c.(*remoteClient).policy.Server = string([]byte{0x7f})
+ res, sign, err := c.GetUploadURL(0, "")
+ a.Error(err)
+ a.Empty(res)
+ a.Empty(sign)
+ }
+
+ // 成功
+ {
+ c.(*remoteClient).policy.Server = ""
+ res, sign, err := c.GetUploadURL(0, "")
+ a.NoError(err)
+ a.NotEmpty(res)
+ a.NotEmpty(sign)
+ }
+}
diff --git a/pkg/filesystem/driver/remote/handler_test.go b/pkg/filesystem/driver/remote/handler_test.go
index 478b2905..9320bf2d 100644
--- a/pkg/filesystem/driver/remote/handler_test.go
+++ b/pkg/filesystem/driver/remote/handler_test.go
@@ -2,6 +2,9 @@ package remote
import (
"context"
+ "errors"
+ "github.com/cloudreve/Cloudreve/v3/pkg/mocks/remoteclientmock"
+ "github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"io"
"io/ioutil"
"net/http"
@@ -14,45 +17,26 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
- "github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
)
-func TestHandler_Token(t *testing.T) {
- asserts := assert.New(t)
- handler := Driver{
- Policy: &model.Policy{
- MaxSize: 10,
- AutoRename: true,
- DirNameRule: "dir",
- FileNameRule: "file",
- OptionsSerialized: model.PolicyOption{
- FileType: []string{"txt"},
- },
- Server: "http://test.com",
- },
- AuthInstance: auth.HMACAuth{},
+func TestNewDriver(t *testing.T) {
+ a := assert.New(t)
+
+ // remoteClient 初始化失败
+ {
+ d, err := NewDriver(&model.Policy{Server: string([]byte{0x7f})})
+ a.Error(err)
+ a.Nil(d)
}
- ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
- auth.General = auth.HMACAuth{SecretKey: []byte("test")}
// 成功
{
- cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
- credential, err := handler.Token(ctx, 10, "123", nil)
- asserts.NoError(err)
- policy, err := serializer.DecodeUploadPolicy(credential.Policy)
- asserts.NoError(err)
- asserts.Equal("http://test.cloudreve.org/api/v3/callback/remote/123", policy.CallbackURL)
- asserts.Equal(uint64(10), policy.MaxSize)
- asserts.Equal(true, policy.AutoRename)
- asserts.Equal("dir", policy.SavePath)
- asserts.Equal("file", policy.FileName)
- asserts.Equal("file", policy.FileName)
- asserts.Equal([]string{"txt"}, policy.AllowedExtension)
+ d, err := NewDriver(&model.Policy{})
+ a.NoError(err)
+ a.NotNil(d)
}
-
}
func TestHandler_Source(t *testing.T) {
@@ -369,6 +353,20 @@ func TestHandler_Get(t *testing.T) {
}
func TestHandler_Put(t *testing.T) {
+ a := assert.New(t)
+ handler, _ := NewDriver(&model.Policy{
+ Type: "remote",
+ SecretKey: "test",
+ Server: "http://test.com",
+ })
+ clientMock := &remoteclientmock.RemoteClientMock{}
+ handler.uploadClient = clientMock
+ clientMock.On("Upload", testMock.Anything, testMock.Anything).Return(errors.New("error"))
+ a.Error(handler.Put(context.Background(), &fsctx.FileStream{}))
+ clientMock.AssertExpectations(t)
+}
+
+func TestHandler_Thumb(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
@@ -379,92 +377,65 @@ func TestHandler_Put(t *testing.T) {
AuthInstance: auth.HMACAuth{},
}
ctx := context.Background()
- asserts.NoError(cache.Set("setting_upload_credential_timeout", "3600", 0))
+ asserts.NoError(cache.Set("setting_preview_timeout", "60", 0))
+ resp, err := handler.Thumb(ctx, "/1.txt")
+ asserts.NoError(err)
+ asserts.True(resp.Redirect)
+}
- // 成功
+func TestHandler_Token(t *testing.T) {
+ a := assert.New(t)
+ handler, _ := NewDriver(&model.Policy{})
+
+ // 无法创建上传会话
{
- ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
- clientMock := ClientMock{}
- clientMock.On(
- "Request",
- "POST",
- "http://test.com/api/v3/slave/upload",
- testMock.Anything,
- testMock.Anything,
- ).Return(&request.Response{
- Err: nil,
- Response: &http.Response{
- StatusCode: 200,
- Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
- },
- })
- handler.Client = clientMock
- err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
+ clientMock := &remoteclientmock.RemoteClientMock{}
+ handler.uploadClient = clientMock
+ clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10)).Return(errors.New("error"))
+ res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
+ a.Error(err)
+ a.Contains(err.Error(), "error")
+ a.Nil(res)
clientMock.AssertExpectations(t)
- asserts.NoError(err)
}
- // 请求失败
+ // 无法创建上传地址
{
- ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
- clientMock := ClientMock{}
- clientMock.On(
- "Request",
- "POST",
- "http://test.com/api/v3/slave/upload",
- testMock.Anything,
- testMock.Anything,
- ).Return(&request.Response{
- Err: nil,
- Response: &http.Response{
- StatusCode: 404,
- Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
- },
- })
- handler.Client = clientMock
- err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
+ clientMock := &remoteclientmock.RemoteClientMock{}
+ handler.uploadClient = clientMock
+ clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10)).Return(nil)
+ clientMock.On("GetUploadURL", int64(10), "").Return("", "", errors.New("error"))
+ res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
+ a.Error(err)
+ a.Contains(err.Error(), "error")
+ a.Nil(res)
clientMock.AssertExpectations(t)
- asserts.Error(err)
}
- // 返回错误
+ // 成功
{
- ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
- clientMock := ClientMock{}
- clientMock.On(
- "Request",
- "POST",
- "http://test.com/api/v3/slave/upload",
- testMock.Anything,
- testMock.Anything,
- ).Return(&request.Response{
- Err: nil,
- Response: &http.Response{
- StatusCode: 200,
- Body: ioutil.NopCloser(strings.NewReader(`{"code":1}`)),
- },
- })
- handler.Client = clientMock
- err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
+ clientMock := &remoteclientmock.RemoteClientMock{}
+ handler.uploadClient = clientMock
+ clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10)).Return(nil)
+ clientMock.On("GetUploadURL", int64(10), "").Return("1", "2", nil)
+ res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
+ a.NoError(err)
+ a.NotNil(res)
+ a.Equal("1", res.UploadURLs[0])
+ a.Equal("2", res.Credential)
clientMock.AssertExpectations(t)
- asserts.Error(err)
}
-
}
-func TestHandler_Thumb(t *testing.T) {
- asserts := assert.New(t)
- handler := Driver{
- Policy: &model.Policy{
- Type: "remote",
- SecretKey: "test",
- Server: "http://test.com",
- },
- AuthInstance: auth.HMACAuth{},
- }
- ctx := context.Background()
- asserts.NoError(cache.Set("setting_preview_timeout", "60", 0))
- resp, err := handler.Thumb(ctx, "/1.txt")
- asserts.NoError(err)
- asserts.True(resp.Redirect)
+func TestDriver_CancelToken(t *testing.T) {
+ a := assert.New(t)
+ handler, _ := NewDriver(&model.Policy{})
+
+ clientMock := &remoteclientmock.RemoteClientMock{}
+ handler.uploadClient = clientMock
+ clientMock.On("DeleteUploadSession", testMock.Anything, "key").Return(errors.New("error"))
+ err := handler.CancelToken(context.Background(), &serializer.UploadSession{Key: "key"})
+ a.Error(err)
+ a.Contains(err.Error(), "error")
+ clientMock.AssertExpectations(t)
}
diff --git a/pkg/filesystem/driver/s3/handler.go b/pkg/filesystem/driver/s3/handler.go
index 120bcf1e..ba9ce60a 100644
--- a/pkg/filesystem/driver/s3/handler.go
+++ b/pkg/filesystem/driver/s3/handler.go
@@ -2,13 +2,12 @@ package s3
import (
"context"
- "crypto/hmac"
- "crypto/sha256"
- "encoding/base64"
- "encoding/hex"
- "encoding/json"
"errors"
+ "fmt"
+ "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
+ "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
+ "io"
"net/http"
"net/url"
"path"
@@ -47,6 +46,14 @@ type MetaData struct {
Etag string
}
+func NewDriver(policy *model.Policy) (*Driver, error) {
+ driver := &Driver{
+ Policy: policy,
+ }
+
+ return driver, driver.InitS3Client()
+}
+
// InitS3Client 初始化S3会话
func (handler *Driver) InitS3Client() error {
if handler.Policy == nil {
@@ -72,13 +79,7 @@ func (handler *Driver) InitS3Client() error {
}
// List 列出给定路径下的文件
-func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
-
- // 初始化客户端
- if err := handler.InitS3Client(); err != nil {
- return nil, err
- }
-
+func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// 初始化列目录参数
base = strings.TrimPrefix(base, "/")
if base != "" {
@@ -155,8 +156,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
}
// Get 获取文件
-func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
-
+func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 获取文件源地址
downloadURL, err := handler.Source(
ctx,
@@ -197,7 +197,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// Put 将文件流保存到指定目录
-func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
+func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
// 初始化客户端
@@ -205,13 +205,15 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
return err
}
- uploader := s3manager.NewUploader(handler.sess)
+ uploader := s3manager.NewUploader(handler.sess, func(u *s3manager.Uploader) {
+ u.PartSize = int64(handler.Policy.OptionsSerialized.ChunkSize)
+ })
dst := file.Info().SavePath
_, err := uploader.Upload(&s3manager.UploadInput{
Bucket: &handler.Policy.BucketName,
Key: &dst,
- Body: file,
+ Body: io.LimitReader(file, int64(file.Info().Size)),
})
if err != nil {
@@ -223,13 +225,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
// Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误
-func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
-
- // 初始化客户端
- if err := handler.InitS3Client(); err != nil {
- return files, err
- }
-
+func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
failed := make([]string, 0, len(files))
deleted := make([]string, 0, len(files))
@@ -263,12 +259,12 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// Thumb 获取文件缩略图
-func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
+func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
return nil, errors.New("未实现")
}
// Source 获取外链URL
-func (handler Driver) Source(
+func (handler *Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
@@ -325,42 +321,75 @@ func (handler Driver) Source(
}
// Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
- // 生成回调地址
- siteURL := model.GetSiteURL()
- apiBaseURI, _ := url.Parse("/api/v3/callback/s3/" + uploadSession.Key)
- apiURL := siteURL.ResolveReference(apiBaseURI)
-
- // 上传策略
- savePath := file.Info().SavePath
- putPolicy := UploadPolicy{
- Expiration: time.Now().UTC().Add(time.Duration(ttl) * time.Second).Format(time.RFC3339),
- Conditions: []interface{}{
- map[string]string{"bucket": handler.Policy.BucketName},
- []string{"starts-with", "$key", savePath},
- []string{"starts-with", "$success_action_redirect", apiURL.String()},
- []string{"starts-with", "$name", ""},
- []string{"starts-with", "$Content-Type", ""},
- map[string]string{"x-amz-algorithm": "AWS4-HMAC-SHA256"},
- },
+func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
+ // 检查文件是否存在
+ fileInfo := file.Info()
+ if _, err := handler.Meta(ctx, fileInfo.SavePath); err == nil {
+ return nil, fmt.Errorf("file already exist")
}
- if handler.Policy.MaxSize > 0 {
- putPolicy.Conditions = append(putPolicy.Conditions,
- []interface{}{"content-length-range", 0, handler.Policy.MaxSize})
+ // 创建分片上传
+ expires := time.Now().Add(time.Duration(ttl) * time.Second)
+ res, err := handler.svc.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
+ Bucket: &handler.Policy.BucketName,
+ Key: &fileInfo.SavePath,
+ Expires: &expires,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("failed to create multipart upload: %w", err)
}
- // 生成上传凭证
- return handler.getUploadCredential(ctx, putPolicy, apiURL, savePath)
-}
+ uploadSession.UploadID = *res.UploadId
+
+ // 为每个分片签名上传 URL
+ chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false)
+ urls := make([]string, chunks.Num())
+ for chunks.Next() {
+ err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error {
+ signedReq, _ := handler.svc.UploadPartRequest(&s3.UploadPartInput{
+ Bucket: &handler.Policy.BucketName,
+ Key: &fileInfo.SavePath,
+ PartNumber: aws.Int64(int64(c.Index() + 1)),
+ UploadId: res.UploadId,
+ })
+
+ signedURL, err := signedReq.Presign(time.Duration(ttl) * time.Second)
+ if err != nil {
+ return err
+ }
+
+ urls[c.Index()] = signedURL
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ }
-// Meta 获取文件信息
-func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
- // 初始化客户端
- if err := handler.InitS3Client(); err != nil {
+ // 签名完成分片上传的请求URL
+ signedReq, _ := handler.svc.CompleteMultipartUploadRequest(&s3.CompleteMultipartUploadInput{
+ Bucket: &handler.Policy.BucketName,
+ Key: &fileInfo.SavePath,
+ UploadId: res.UploadId,
+ })
+
+ signedURL, err := signedReq.Presign(time.Duration(ttl) * time.Second)
+ if err != nil {
return nil, err
}
+ // 生成上传凭证
+ return &serializer.UploadCredential{
+ SessionID: uploadSession.Key,
+ ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
+ UploadID: *res.UploadId,
+ UploadURLs: urls,
+ CompleteURL: signedURL,
+ }, nil
+}
+
+// Meta 获取文件信息
+func (handler *Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
res, err := handler.svc.GetObject(
&s3.GetObjectInput{
Bucket: &handler.Policy.BucketName,
@@ -378,52 +407,8 @@ func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error)
}
-func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback *url.URL, savePath string) (*serializer.UploadCredential, error) {
-
- longDate := time.Now().UTC().Format("20060102T150405Z")
- shortDate := time.Now().UTC().Format("20060102")
-
- credential := handler.Policy.AccessKey + "/" + shortDate + "/" + handler.Policy.OptionsSerialized.Region + "/s3/aws4_request"
- policy.Conditions = append(policy.Conditions, map[string]string{"x-amz-credential": credential})
- policy.Conditions = append(policy.Conditions, map[string]string{"x-amz-date": longDate})
-
- // 编码上传策略
- policyJSON, err := json.Marshal(policy)
- if err != nil {
- return nil, err
- }
- policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
-
- //签名
- signature := getHMAC([]byte("AWS4"+handler.Policy.SecretKey), []byte(shortDate))
- signature = getHMAC(signature, []byte(handler.Policy.OptionsSerialized.Region))
- signature = getHMAC(signature, []byte("s3"))
- signature = getHMAC(signature, []byte("aws4_request"))
- signature = getHMAC(signature, []byte(policyEncoded))
-
- return &serializer.UploadCredential{
- Policy: policyEncoded,
- Callback: callback.String(),
- Token: hex.EncodeToString(signature),
- AccessKey: credential,
- Path: savePath,
- KeyTime: longDate,
- }, nil
-}
-
-func getHMAC(key []byte, data []byte) []byte {
- hash := hmac.New(sha256.New, key)
- hash.Write(data)
- return hash.Sum(nil)
-}
-
// CORS 创建跨域策略
-func (handler Driver) CORS() error {
- // 初始化客户端
- if err := handler.InitS3Client(); err != nil {
- return err
- }
-
+func (handler *Driver) CORS() error {
rule := s3.CORSRule{
AllowedMethods: aws.StringSlice([]string{
"GET",
@@ -434,6 +419,7 @@ func (handler Driver) CORS() error {
}),
AllowedOrigins: aws.StringSlice([]string{"*"}),
AllowedHeaders: aws.StringSlice([]string{"*"}),
+ ExposeHeaders: aws.StringSlice([]string{"ETag"}),
MaxAgeSeconds: aws.Int64(3600),
}
@@ -448,6 +434,11 @@ func (handler Driver) CORS() error {
}
// 取消上传凭证
-func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
- return nil
+func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+ _, err := handler.svc.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
+ UploadId: &uploadSession.UploadID,
+ Bucket: &handler.Policy.BucketName,
+ Key: &uploadSession.SavePath,
+ })
+ return err
}
diff --git a/pkg/filesystem/driver/upyun/handler.go b/pkg/filesystem/driver/upyun/handler.go
index ab033969..0357f720 100644
--- a/pkg/filesystem/driver/upyun/handler.go
+++ b/pkg/filesystem/driver/upyun/handler.go
@@ -311,8 +311,6 @@ func (handler Driver) signURL(ctx context.Context, path *url.URL, TTL int64) (st
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
- // 检查文件大小
-
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/upyun/" + uploadSession.Key)
@@ -332,17 +330,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
}
// 生成上传凭证
- return handler.getUploadCredential(ctx, putPolicy)
-}
-
-// 取消上传凭证
-func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
- return nil
-}
-
-func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy) (*serializer.UploadCredential, error) {
- // 生成上传策略
- policyJSON, err := json.Marshal(policy)
+ policyJSON, err := json.Marshal(putPolicy)
if err != nil {
return nil, err
}
@@ -353,11 +341,18 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli
signStr := handler.Sign(ctx, elements)
return &serializer.UploadCredential{
- Policy: policyEncoded,
- Token: signStr,
+ SessionID: uploadSession.Key,
+ Policy: policyEncoded,
+ Credential: signStr,
+ UploadURLs: []string{"https://v0.api.upyun.com/" + handler.Policy.BucketName},
}, nil
}
+// 取消上传凭证
+func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+ return nil
+}
+
// Sign 计算又拍云的签名头
func (handler Driver) Sign(ctx context.Context, elements []string) string {
password := fmt.Sprintf("%x", md5.Sum([]byte(handler.Policy.SecretKey)))
diff --git a/pkg/filesystem/file.go b/pkg/filesystem/file.go
index 3d244e44..0aa67800 100644
--- a/pkg/filesystem/file.go
+++ b/pkg/filesystem/file.go
@@ -48,9 +48,6 @@ func (fs *FileSystem) AddFile(ctx context.Context, parent *model.Folder, file fs
// 添加文件记录前的钩子
err := fs.Trigger(ctx, "BeforeAddFile", file)
if err != nil {
- if err := fs.Trigger(ctx, "BeforeAddFileFailed", file); err != nil {
- util.Log().Debug("BeforeAddFileFailed 钩子执行失败，%s", err)
- }
return nil, err
}
@@ -180,6 +177,7 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m
for policyID, toBeDeletedFiles := range files {
// 列举出需要物理删除的文件的物理路径
sourceNamesAll := make([]string, 0, len(toBeDeletedFiles))
+ uploadSessions := make([]*serializer.UploadSession, 0, len(toBeDeletedFiles))
for i := 0; i < len(toBeDeletedFiles); i++ {
sourceNamesAll = append(sourceNamesAll, toBeDeletedFiles[i].SourceName)
@@ -187,11 +185,7 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m
if toBeDeletedFiles[i].UploadSessionID != nil {
if session, ok := cache.Get(UploadSessionCachePrefix + *toBeDeletedFiles[i].UploadSessionID); ok {
uploadSession := session.(serializer.UploadSession)
- if err := fs.Handler.CancelToken(ctx, &uploadSession); err != nil {
- util.Log().Warning("无法取消 [%s] 的上传会话: %s", err)
- }
-
- cache.Deletes([]string{*toBeDeletedFiles[i].UploadSessionID}, UploadSessionCachePrefix)
+ uploadSessions = append(uploadSessions, &uploadSession)
}
}
@@ -205,6 +199,15 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m
continue
}
+ // 取消上传会话
+ for _, upSession := range uploadSessions {
+ if err := fs.Handler.CancelToken(ctx, upSession); err != nil {
+ util.Log().Warning("无法取消 [%s] 的上传会话: %s", upSession.Name, err)
+ }
+
+ cache.Deletes([]string{upSession.Key}, UploadSessionCachePrefix)
+ }
+
// 执行删除
failedFile, _ := fs.Handler.Delete(ctx, sourceNamesAll)
failed[policyID] = failedFile
diff --git a/pkg/filesystem/file_test.go b/pkg/filesystem/file_test.go
index cb3d24f6..5e495c75 100644
--- a/pkg/filesystem/file_test.go
+++ b/pkg/filesystem/file_test.go
@@ -2,6 +2,7 @@ package filesystem
import (
"context"
+ "errors"
"os"
"testing"
@@ -19,8 +20,9 @@ import (
func TestFileSystem_AddFile(t *testing.T) {
asserts := assert.New(t)
file := fsctx.FileStream{
- Size: 5,
- Name: "1.png",
+ Size: 5,
+ Name: "1.png",
+ SavePath: "/Uploads/1_sad.png",
}
folder := model.Folder{
Model: gorm.Model{
@@ -39,24 +41,55 @@ func TestFileSystem_AddFile(t *testing.T) {
},
},
},
+ Policy: &model.Policy{Type: "cos"},
}
- ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
- ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/Uploads/1_sad.png")
- _, err := fs.AddFile(ctx, &folder)
+ _, err := fs.AddFile(context.Background(), &folder, &file)
asserts.Error(err)
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
- f, err := fs.AddFile(ctx, &folder)
+ f, err := fs.AddFile(context.Background(), &folder, &file)
asserts.NoError(err)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Equal("/Uploads/1_sad.png", f.SourceName)
asserts.NotEmpty(f.PicInfo)
+
+ // 前置钩子执行失败
+ {
+ hookExecuted := false
+ fs.Use("BeforeAddFile", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
+ hookExecuted = true
+ return errors.New("error")
+ })
+ f, err := fs.AddFile(context.Background(), &folder, &file)
+ asserts.Error(err)
+ asserts.Nil(f)
+ asserts.True(hookExecuted)
+ }
+
+ // 后置钩子执行失败
+ {
+ hookExecuted := false
+ mock.ExpectBegin()
+ mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
+ mock.ExpectRollback()
+ fs.Hooks = map[string][]Hook{}
+ fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
+ hookExecuted = true
+ return errors.New("error")
+ })
+ f, err := fs.AddFile(context.Background(), &folder, &file)
+ asserts.Error(err)
+ asserts.Nil(f)
+ asserts.True(hookExecuted)
+ asserts.NoError(mock.ExpectationsWereMet())
+ }
}
func TestFileSystem_GetContent(t *testing.T) {
@@ -263,6 +296,22 @@ func TestFileSystem_deleteGroupedFile(t *testing.T) {
3: {},
}, failed)
}
+ // 包含上传会话文件
+ {
+ sessionID := "session"
+ cache.Set(UploadSessionCachePrefix+sessionID, serializer.UploadSession{Key: sessionID}, 0)
+ files[1].Policy.Type = "local"
+ files[3].Policy.Type = "local"
+ files[0].UploadSessionID = &sessionID
+ failed := fs.deleteGroupedFile(ctx, fs.GroupFileByPolicy(ctx, files))
+ asserts.Equal(map[uint][]string{
+ 1: {},
+ 2: {},
+ 3: {},
+ }, failed)
+ _, ok := cache.Get(UploadSessionCachePrefix + sessionID)
+ asserts.False(ok)
+ }
}
func TestFileSystem_GetSource(t *testing.T) {
diff --git a/pkg/filesystem/filesystem.go b/pkg/filesystem/filesystem.go
index 48e03784..b0da90e7 100644
--- a/pkg/filesystem/filesystem.go
+++ b/pkg/filesystem/filesystem.go
@@ -121,12 +121,11 @@ func NewAnonymousFileSystem() (*FileSystem, error) {
// DispatchHandler 根据存储策略分配文件适配器
func (fs *FileSystem) DispatchHandler() error {
- currentPolicy := fs.Policy
- policyType := currentPolicy.Type
-
- if currentPolicy == nil {
- return ErrUnknownPolicyType
+ if fs.Policy == nil {
+ return errors.New("未设置存储策略")
}
+ policyType := fs.Policy.Type
+ currentPolicy := fs.Policy
switch policyType {
case "mock", "anonymous":
@@ -144,16 +143,12 @@ func (fs *FileSystem) DispatchHandler() error {
fs.Handler = handler
case "qiniu":
- fs.Handler = qiniu.Driver{
- Policy: currentPolicy,
- }
+ fs.Handler = qiniu.NewDriver(currentPolicy)
return nil
case "oss":
- fs.Handler = oss.Driver{
- Policy: currentPolicy,
- HTTPClient: request.NewClient(),
- }
- return nil
+ handler, err := oss.NewDriver(currentPolicy)
+ fs.Handler = handler
+ return err
case "upyun":
fs.Handler = upyun.Driver{
Policy: currentPolicy,
@@ -178,10 +173,9 @@ func (fs *FileSystem) DispatchHandler() error {
}
return nil
case "s3":
- fs.Handler = s3.Driver{
- Policy: currentPolicy,
- }
- return nil
+ handler, err := s3.NewDriver(currentPolicy)
+ fs.Handler = handler
+ return err
default:
return ErrUnknownPolicyType
}
diff --git a/pkg/filesystem/filesystem_test.go b/pkg/filesystem/filesystem_test.go
index 0c558d79..8b7aae37 100644
--- a/pkg/filesystem/filesystem_test.go
+++ b/pkg/filesystem/filesystem_test.go
@@ -2,16 +2,16 @@ package filesystem
import (
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
+ "github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/masterinslave"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/slaveinmaster"
+ "github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"net/http/httptest"
"github.com/DATA-DOG/go-sqlmock"
model "github.com/cloudreve/Cloudreve/v3/models"
- "github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/remote"
- "github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
@@ -36,7 +36,7 @@ func TestNewFileSystem(t *testing.T) {
fs, err = NewFileSystem(&user)
asserts.NoError(err)
asserts.NotNil(fs.Handler)
- asserts.IsType(remote.Driver{}, fs.Handler)
+ asserts.IsType(&remote.Driver{}, fs.Handler)
user.Policy.Type = "unknown"
fs, err = NewFileSystem(&user)
@@ -64,9 +64,10 @@ func TestNewFileSystemFromContext(t *testing.T) {
func TestDispatchHandler(t *testing.T) {
asserts := assert.New(t)
fs := &FileSystem{
- User: &model.User{Policy: model.Policy{
+ User: &model.User{},
+ Policy: &model.Policy{
Type: "local",
- }},
+ },
}
// 未指定，使用用户默认
@@ -95,7 +96,7 @@ func TestDispatchHandler(t *testing.T) {
err = fs.DispatchHandler()
asserts.NoError(err)
- fs.Policy = &model.Policy{Type: "oss"}
+ fs.Policy = &model.Policy{Type: "oss", Server: "https://s.com", BucketName: "1234"}
err = fs.DispatchHandler()
asserts.NoError(err)
@@ -140,23 +141,6 @@ func TestNewFileSystemFromCallback(t *testing.T) {
asserts.Error(err)
}
- // 找不到上传策略
- {
- c, _ := gin.CreateTestContext(httptest.NewRecorder())
- c.Set("user", &model.User{
- Policy: model.Policy{
- Type: "local",
- },
- })
- c.Set("callbackSession", &serializer.UploadSession{PolicyID: 138})
- cache.Deletes([]string{"138"}, "policy_")
- mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}))
- fs, err := NewFileSystemFromCallback(c)
- asserts.NoError(mock.ExpectationsWereMet())
- asserts.Nil(fs)
- asserts.Error(err)
- }
-
// 成功
{
c, _ := gin.CreateTestContext(httptest.NewRecorder())
@@ -165,11 +149,8 @@ func TestNewFileSystemFromCallback(t *testing.T) {
Type: "local",
},
})
- c.Set("callbackSession", &serializer.UploadSession{PolicyID: 138})
- cache.Deletes([]string{"138"}, "policy_")
- mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id", "type", "options"}).AddRow(138, "local", "{}"))
+ c.Set(UploadSessionCtx, &serializer.UploadSession{Policy: model.Policy{Type: "local"}})
fs, err := NewFileSystemFromCallback(c)
- asserts.NoError(mock.ExpectationsWereMet())
asserts.NotNil(fs)
asserts.NoError(err)
}
@@ -234,6 +215,16 @@ func TestNewAnonymousFileSystem(t *testing.T) {
asserts.Error(err)
asserts.Nil(fs)
}
+
+ // 从机
+ {
+ conf.SystemConfig.Mode = "slave"
+ fs, err := NewAnonymousFileSystem()
+ asserts.NoError(mock.ExpectationsWereMet())
+ asserts.NoError(err)
+ asserts.NotNil(fs)
+ asserts.NotNil(fs.Handler)
+ }
}
func TestFileSystem_Recycle(t *testing.T) {
diff --git a/pkg/filesystem/fsctx/stream.go b/pkg/filesystem/fsctx/stream.go
index c51d28c2..4cf48e3d 100644
--- a/pkg/filesystem/fsctx/stream.go
+++ b/pkg/filesystem/fsctx/stream.go
@@ -1,6 +1,7 @@
package fsctx
import (
+ "errors"
"io"
"time"
)
@@ -75,7 +76,11 @@ func (file *FileStream) Close() error {
}
func (file *FileStream) Seek(offset int64, whence int) (int64, error) {
- return file.Seeker.Seek(offset, whence)
+ if file.Seekable() {
+ return file.Seeker.Seek(offset, whence)
+ }
+
+ return 0, errors.New("no seeker")
}
func (file *FileStream) Seekable() bool {
diff --git a/pkg/filesystem/fsctx/stream_test.go b/pkg/filesystem/fsctx/stream_test.go
index 8cc0c859..1ef6e1fa 100644
--- a/pkg/filesystem/fsctx/stream_test.go
+++ b/pkg/filesystem/fsctx/stream_test.go
@@ -1,30 +1,15 @@
package fsctx
import (
+ model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/stretchr/testify/assert"
+ "io"
"io/ioutil"
+ "os"
"strings"
"testing"
)
-func TestFileStream_GetFileName(t *testing.T) {
- asserts := assert.New(t)
- file := FileStream{Name: "123"}
- asserts.Equal("123", file.GetFileName())
-}
-
-func TestFileStream_GetMIMEType(t *testing.T) {
- asserts := assert.New(t)
- file := FileStream{MIMEType: "123"}
- asserts.Equal("123", file.GetMIMEType())
-}
-
-func TestFileStream_GetSize(t *testing.T) {
- asserts := assert.New(t)
- file := FileStream{Size: 123}
- asserts.Equal(uint64(123), file.GetSize())
-}
-
func TestFileStream_Read(t *testing.T) {
asserts := assert.New(t)
file := FileStream{
@@ -40,9 +25,54 @@ func TestFileStream_Read(t *testing.T) {
func TestFileStream_Close(t *testing.T) {
asserts := assert.New(t)
- file := FileStream{
- File: ioutil.NopCloser(strings.NewReader("123")),
+ {
+ file := FileStream{
+ File: ioutil.NopCloser(strings.NewReader("123")),
+ }
+ err := file.Close()
+ asserts.NoError(err)
+ }
+
+ {
+ file := FileStream{}
+ err := file.Close()
+ asserts.NoError(err)
+ }
+}
+
+func TestFileStream_Seek(t *testing.T) {
+ asserts := assert.New(t)
+ f, _ := os.CreateTemp("", "*")
+ defer func() {
+ f.Close()
+ os.Remove(f.Name())
+ }()
+ {
+ file := FileStream{
+ File: f,
+ Seeker: f,
+ }
+ res, err := file.Seek(0, io.SeekStart)
+ asserts.NoError(err)
+ asserts.EqualValues(0, res)
}
- err := file.Close()
- asserts.NoError(err)
+
+ {
+ file := FileStream{}
+ res, err := file.Seek(0, io.SeekStart)
+ asserts.Error(err)
+ asserts.EqualValues(0, res)
+ }
+}
+
+func TestFileStream_Info(t *testing.T) {
+ a := assert.New(t)
+ file := FileStream{}
+ a.NotNil(file.Info())
+
+ file.SetSize(10)
+ a.EqualValues(10, file.Info().Size)
+
+ file.SetModel(&model.File{})
+ a.NotNil(file.Info().Model)
}
diff --git a/pkg/filesystem/fsctx/taskinfo/taskinfo.go b/pkg/filesystem/fsctx/taskinfo/taskinfo.go
deleted file mode 100644
index 1899c1bf..00000000
--- a/pkg/filesystem/fsctx/taskinfo/taskinfo.go
+++ /dev/null
@@ -1 +0,0 @@
-package taskinfo
diff --git a/pkg/filesystem/hooks.go b/pkg/filesystem/hooks.go
index e73fa33a..fc4a68fa 100644
--- a/pkg/filesystem/hooks.go
+++ b/pkg/filesystem/hooks.go
@@ -5,7 +5,6 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
- "github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
@@ -149,7 +148,6 @@ func HookCancelContext(ctx context.Context, fs *FileSystem, file fsctx.FileHeade
}
// HookUpdateSourceName 更新文件SourceName
-// TODO：测试
func HookUpdateSourceName(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File)
if !ok {
@@ -245,7 +243,7 @@ func HookGenerateThumb(ctx context.Context, fs *FileSystem, fileHeader fsctx.Fil
fs.recycleLock.Lock()
go func() {
defer fs.recycleLock.Unlock()
- _, _ = fs.Handler.Delete(ctx, []string{fileMode.SourceName + conf.ThumbConfig.FileSuffix})
+ _, _ = fs.Handler.Delete(ctx, []string{fileMode.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")})
fs.GenerateThumbnail(ctx, fileMode)
}()
}
diff --git a/pkg/filesystem/hooks_test.go b/pkg/filesystem/hooks_test.go
index a247490e..0daa8ec6 100644
--- a/pkg/filesystem/hooks_test.go
+++ b/pkg/filesystem/hooks_test.go
@@ -3,22 +3,20 @@ package filesystem
import (
"context"
"errors"
- "io"
- "io/ioutil"
- "net/http"
- "os"
- "strings"
- "sync"
- "testing"
-
"github.com/DATA-DOG/go-sqlmock"
- model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
+ "github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
+ "io/ioutil"
+ "net/http"
+ "strings"
+ "testing"
+
+ model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
@@ -26,78 +24,72 @@ import (
func TestGenericBeforeUpload(t *testing.T) {
asserts := assert.New(t)
- file := fsctx.FileStream{
+ file := &fsctx.FileStream{
Size: 5,
Name: "1.txt",
}
+ ctx := context.Background()
cache.Set("pack_size_0", uint64(0), 0)
- ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
fs := FileSystem{
User: &model.User{
Storage: 0,
Group: model.Group{
MaxStorage: 11,
},
- Policy: model.Policy{
- MaxSize: 4,
- OptionsSerialized: model.PolicyOption{
- FileType: []string{"txt"},
- },
+ },
+ Policy: &model.Policy{
+ MaxSize: 4,
+ OptionsSerialized: model.PolicyOption{
+ FileType: []string{"txt"},
},
},
}
- asserts.Error(HookValidateFile(ctx, &fs))
+ asserts.Error(HookValidateFile(ctx, &fs, file))
file.Size = 1
file.Name = "1"
- ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
- asserts.Error(HookValidateFile(ctx, &fs))
+ asserts.Error(HookValidateFile(ctx, &fs, file))
file.Name = "1.txt"
- ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
- asserts.NoError(HookValidateFile(ctx, &fs))
+ asserts.NoError(HookValidateFile(ctx, &fs, file))
file.Name = "1.t/xt"
- ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
- asserts.Error(HookValidateFile(ctx, &fs))
+ asserts.Error(HookValidateFile(ctx, &fs, file))
}
func TestGenericAfterUploadCanceled(t *testing.T) {
asserts := assert.New(t)
- f, err := os.Create("TestGenericAfterUploadCanceled")
- asserts.NoError(err)
- f.Close()
- file := fsctx.FileStream{
- Size: 5,
- Name: "TestGenericAfterUploadCanceled",
+ file := &fsctx.FileStream{
+ Size: 5,
+ Name: "TestGenericAfterUploadCanceled",
+ SavePath: "TestGenericAfterUploadCanceled",
}
- ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "TestGenericAfterUploadCanceled")
- ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
+ ctx := context.Background()
fs := FileSystem{
- User: &model.User{Storage: 5},
- Handler: local.Driver{},
+ User: &model.User{},
}
// 成功
- err = HookDeleteTempFile(ctx, &fs)
- asserts.NoError(err)
- err = HookGiveBackCapacity(ctx, &fs)
- asserts.NoError(err)
- asserts.Equal(uint64(0), fs.User.Storage)
-
- f, err = os.Create("TestGenericAfterUploadCanceled")
- asserts.NoError(err)
- f.Close()
+ {
+ mockHandler := &FileHeaderMock{}
+ fs.Handler = mockHandler
+ mockHandler.On("Delete", testMock.Anything, testMock.Anything).Return([]string{}, nil)
+ err := HookDeleteTempFile(ctx, &fs, file)
+ asserts.NoError(err)
+ mockHandler.AssertExpectations(t)
+ }
- // 容量不能再降低
- err = HookGiveBackCapacity(ctx, &fs)
- asserts.Error(err)
+ // 失败
+ {
+ mockHandler := &FileHeaderMock{}
+ fs.Handler = mockHandler
+ mockHandler.On("Delete", testMock.Anything, testMock.Anything).Return([]string{}, errors.New(""))
+ err := HookDeleteTempFile(ctx, &fs, file)
+ asserts.NoError(err)
+ mockHandler.AssertExpectations(t)
+ }
- //文件不存在
- fs.User.Storage = 5
- err = HookDeleteTempFile(ctx, &fs)
- asserts.NoError(err)
}
func TestGenericAfterUpload(t *testing.T) {
@@ -108,13 +100,14 @@ func TestGenericAfterUpload(t *testing.T) {
ID: 1,
},
},
+ Policy: &model.Policy{},
}
- ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{
+ ctx := context.Background()
+ file := &fsctx.FileStream{
VirtualPath: "/我的文件",
Name: "test.txt",
- })
- ctx = context.WithValue(ctx, fsctx.SavePathCtx, "")
+ }
// 正常
mock.ExpectQuery("SELECT(.+)").
@@ -127,9 +120,10 @@ func TestGenericAfterUpload(t *testing.T) {
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found"))
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
- err := GenericAfterUpload(ctx, &fs)
+ err := GenericAfterUpload(ctx, &fs, file)
asserts.NoError(err)
asserts.NoError(mock.ExpectationsWereMet())
@@ -137,7 +131,7 @@ func TestGenericAfterUpload(t *testing.T) {
mock.ExpectQuery("SELECT(.+)folders(.+)").WillReturnRows(
mock.NewRows([]string{"name"}),
)
- err = GenericAfterUpload(ctx, &fs)
+ err = GenericAfterUpload(ctx, &fs, file)
asserts.Equal(ErrRootProtected, err)
asserts.NoError(mock.ExpectationsWereMet())
@@ -152,10 +146,25 @@ func TestGenericAfterUpload(t *testing.T) {
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnRows(
mock.NewRows([]string{"name"}).AddRow("test.txt"),
)
- err = GenericAfterUpload(ctx, &fs)
+ err = GenericAfterUpload(ctx, &fs, file)
asserts.Equal(ErrFileExisted, err)
asserts.NoError(mock.ExpectationsWereMet())
+ // 文件已存在, 且为上传占位符
+ mock.ExpectQuery("SELECT(.+)").
+ WithArgs(1).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
+ // 1
+ mock.ExpectQuery("SELECT(.+)").
+ WithArgs(1, 1, "我的文件").
+ WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1))
+ mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnRows(
+ mock.NewRows([]string{"name", "upload_session_id"}).AddRow("test.txt", "1"),
+ )
+ err = GenericAfterUpload(ctx, &fs, file)
+ asserts.Equal(ErrFileUploadSessionExisted, err)
+ asserts.NoError(mock.ExpectationsWereMet())
+
// 插入失败
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
@@ -170,7 +179,7 @@ func TestGenericAfterUpload(t *testing.T) {
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
- err = GenericAfterUpload(ctx, &fs)
+ err = GenericAfterUpload(ctx, &fs, file)
asserts.Equal(ErrInsertFileRecord, err)
asserts.NoError(mock.ExpectationsWereMet())
@@ -180,7 +189,7 @@ func TestFileSystem_Use(t *testing.T) {
asserts := assert.New(t)
fs := FileSystem{}
- hook := func(ctx context.Context, fs *FileSystem) error {
+ hook := func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
return nil
}
@@ -215,77 +224,79 @@ func TestFileSystem_Trigger(t *testing.T) {
}
ctx := context.Background()
- hook := func(ctx context.Context, fs *FileSystem) error {
+ hook := func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
fs.User.Storage++
return nil
}
// 一个
fs.Use("BeforeUpload", hook)
- err := fs.Trigger(ctx, "BeforeUpload")
+ err := fs.Trigger(ctx, "BeforeUpload", nil)
asserts.NoError(err)
asserts.Equal(uint64(1), fs.User.Storage)
// 多个
fs.Use("BeforeUpload", hook)
fs.Use("BeforeUpload", hook)
- err = fs.Trigger(ctx, "BeforeUpload")
+ err = fs.Trigger(ctx, "BeforeUpload", nil)
asserts.NoError(err)
asserts.Equal(uint64(4), fs.User.Storage)
+
+ // 多个,有失败
+ fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
+ return errors.New("error")
+ })
+ fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
+ asserts.Fail("following hooks executed")
+ return nil
+ })
+ err = fs.Trigger(ctx, "BeforeUpload", nil)
+ asserts.Error(err)
}
-func TestHookIsFileExist(t *testing.T) {
+func TestHookValidateCapacity(t *testing.T) {
asserts := assert.New(t)
+ cache.Set("pack_size_1", uint64(0), 0)
fs := &FileSystem{User: &model.User{
- Model: gorm.Model{
- ID: 1,
+ Model: gorm.Model{ID: 1},
+ Storage: 0,
+ Group: model.Group{
+ MaxStorage: 11,
},
}}
- ctx := context.WithValue(context.Background(), fsctx.PathCtx, "/test.txt")
+ ctx := context.Background()
+ file := &fsctx.FileStream{Size: 11}
{
- mock.ExpectQuery("SELECT(.+)").
- WithArgs(1).
- WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
- mock.ExpectQuery("SELECT(.+)").WithArgs(uint(1), "test.txt").WillReturnRows(
- sqlmock.NewRows([]string{"Name"}).AddRow("s"),
- )
- err := HookIsFileExist(ctx, fs)
- asserts.NoError(mock.ExpectationsWereMet())
+ err := HookValidateCapacity(ctx, fs, file)
asserts.NoError(err)
}
{
- mock.ExpectQuery("SELECT(.+)").
- WithArgs(1).
- WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
- mock.ExpectQuery("SELECT(.+)").WithArgs(uint(1), "test.txt").WillReturnRows(
- sqlmock.NewRows([]string{"Name"}),
- )
- err := HookIsFileExist(ctx, fs)
- asserts.NoError(mock.ExpectationsWereMet())
+ file.Size = 12
+ err := HookValidateCapacity(ctx, fs, file)
asserts.Error(err)
}
-
}
-func TestHookValidateCapacity(t *testing.T) {
- asserts := assert.New(t)
- cache.Set("pack_size_1", uint64(0), 0)
+func TestHookValidateCapacityDiff(t *testing.T) {
+ a := assert.New(t)
fs := &FileSystem{User: &model.User{
- Model: gorm.Model{ID: 1},
- Storage: 0,
Group: model.Group{
MaxStorage: 11,
},
}}
- ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{Size: 10})
+ file := model.File{Size: 10}
+ ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
+
+ // 无需操作
{
- err := HookValidateCapacity(ctx, fs)
- asserts.NoError(err)
+ a.NoError(HookValidateCapacityDiff(ctx, fs, &fsctx.FileStream{Size: 10}))
}
+
+ // 需要验证
{
- err := HookValidateCapacity(ctx, fs)
- asserts.Error(err)
+ a.Error(HookValidateCapacityDiff(ctx, fs, &fsctx.FileStream{Size: 12}))
}
+
}
func TestHookResetPolicy(t *testing.T) {
@@ -301,7 +312,7 @@ func TestHookResetPolicy(t *testing.T) {
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "type"}).AddRow(2, "local"))
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
- err := HookResetPolicy(ctx, fs)
+ err := HookResetPolicy(ctx, fs, nil)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
@@ -310,76 +321,22 @@ func TestHookResetPolicy(t *testing.T) {
{
cache.Deletes([]string{"2"}, "policy_")
ctx := context.Background()
- err := HookResetPolicy(ctx, fs)
+ err := HookResetPolicy(ctx, fs, nil)
asserts.Error(err)
}
}
-func TestHookChangeCapacity(t *testing.T) {
- asserts := assert.New(t)
- cache.Set("pack_size_1", uint64(0), 0)
-
- // 容量增加 失败
- {
- fs := &FileSystem{User: &model.User{
- Model: gorm.Model{ID: 1},
- }}
-
- newFile := fsctx.FileStream{Size: 10}
- oldFile := model.File{Size: 9}
- ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile)
- ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
- err := HookChangeCapacity(ctx, fs)
- asserts.Equal(ErrInsufficientCapacity, err)
- }
-
- // 容量增加 成功
- {
- fs := &FileSystem{User: &model.User{
- Model: gorm.Model{ID: 1},
- Group: model.Group{MaxStorage: 1},
- }}
-
- newFile := fsctx.FileStream{Size: 10}
- oldFile := model.File{Size: 9}
- ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile)
- ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
- mock.ExpectBegin()
- mock.ExpectExec("UPDATE(.+)").WithArgs(1, sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
- err := HookChangeCapacity(ctx, fs)
- asserts.NoError(mock.ExpectationsWereMet())
- asserts.NoError(err)
- asserts.Equal(uint64(1), fs.User.Storage)
- }
-
- // 容量减少
- {
- fs := &FileSystem{User: &model.User{
- Model: gorm.Model{ID: 1},
- Storage: 1,
- }}
-
- newFile := fsctx.FileStream{Size: 9}
- oldFile := model.File{Size: 10}
- ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile)
- ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
- err := HookChangeCapacity(ctx, fs)
- asserts.NoError(err)
- asserts.Equal(uint64(0), fs.User.Storage)
- }
-}
-
func TestHookCleanFileContent(t *testing.T) {
asserts := assert.New(t)
fs := &FileSystem{User: &model.User{
Model: gorm.Model{ID: 1},
}}
- ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "123/123")
+ file := &fsctx.FileStream{SavePath: "123/123"}
handlerMock := FileHeaderMock{}
- handlerMock.On("Put", testMock.Anything, testMock.Anything, "123/123").Return(errors.New("error"))
+ handlerMock.On("Put", testMock.Anything, testMock.Anything).Return(errors.New("error"))
fs.Handler = handlerMock
- err := HookCleanFileContent(ctx, fs)
+ err := HookCleanFileContent(context.Background(), fs, file)
asserts.Error(err)
handlerMock.AssertExpectations(t)
}
@@ -395,14 +352,17 @@ func TestHookClearFileSize(t *testing.T) {
ctx := context.WithValue(
context.Background(),
fsctx.FileModelCtx,
- model.File{Model: gorm.Model{ID: 1}},
+ model.File{Model: gorm.Model{ID: 1}, Size: 10},
)
mock.ExpectBegin()
- mock.ExpectExec("UPDATE(.+)").
- WithArgs(0, sqlmock.AnyArg(), 1).
+ mock.ExpectExec("UPDATE(.+)files(.+)").
+ WithArgs(0, sqlmock.AnyArg(), 1, 10).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE(.+)users(.+)").
+ WithArgs(10, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
- err := HookClearFileSize(ctx, fs)
+ err := HookClearFileSize(ctx, fs, nil)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
@@ -410,7 +370,7 @@ func TestHookClearFileSize(t *testing.T) {
// 上下文对象不存在
{
ctx := context.Background()
- err := HookClearFileSize(ctx, fs)
+ err := HookClearFileSize(ctx, fs, nil)
asserts.Error(err)
}
@@ -432,7 +392,7 @@ func TestHookUpdateSourceName(t *testing.T) {
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WithArgs("new.txt", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
- err := HookUpdateSourceName(ctx, fs)
+ err := HookUpdateSourceName(ctx, fs, nil)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
@@ -440,7 +400,7 @@ func TestHookUpdateSourceName(t *testing.T) {
// 上下文错误
{
ctx := context.Background()
- err := HookUpdateSourceName(ctx, fs)
+ err := HookUpdateSourceName(ctx, fs, nil)
asserts.Error(err)
}
}
@@ -457,41 +417,32 @@ func TestGenericAfterUpdate(t *testing.T) {
Model: gorm.Model{ID: 1},
PicInfo: "1,1",
}
- newFile := fsctx.FileStream{Size: 10}
+ newFile := &fsctx.FileStream{Size: 10}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
- ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
handlerMock := FileHeaderMock{}
handlerMock.On("Delete", testMock.Anything, []string{"._thumb"}).Return([]string{}, nil)
fs.Handler = handlerMock
mock.ExpectBegin()
- mock.ExpectExec("UPDATE(.+)").
- WithArgs(10, sqlmock.AnyArg(), 1).
+ mock.ExpectExec("UPDATE(.+)files(.+)").
+ WithArgs(10, sqlmock.AnyArg(), 1, 0).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE(.+)users(.+)").
+ WithArgs(10, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
- err := GenericAfterUpdate(ctx, fs)
+ err := GenericAfterUpdate(ctx, fs, newFile)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
- // 新文件上下文不存在
- {
- originFile := model.File{
- Model: gorm.Model{ID: 1},
- PicInfo: "1,1",
- }
- ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
- err := GenericAfterUpdate(ctx, fs)
- asserts.Error(err)
- }
-
// 原始文件上下文不存在
{
- newFile := fsctx.FileStream{Size: 10}
- ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, newFile)
- err := GenericAfterUpdate(ctx, fs)
+ newFile := &fsctx.FileStream{Size: 10}
+ ctx := context.Background()
+ err := GenericAfterUpdate(ctx, fs, newFile)
asserts.Error(err)
}
@@ -502,91 +453,41 @@ func TestGenericAfterUpdate(t *testing.T) {
Model: gorm.Model{ID: 1},
PicInfo: "1,1",
}
- newFile := fsctx.FileStream{Size: 10}
+ newFile := &fsctx.FileStream{Size: 10}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
- ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").
- WithArgs(10, sqlmock.AnyArg(), 1).
+ WithArgs(10, sqlmock.AnyArg(), 1, 0).
WillReturnError(errors.New("error"))
mock.ExpectRollback()
- err := GenericAfterUpdate(ctx, fs)
+ err := GenericAfterUpdate(ctx, fs, newFile)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
}
}
-func TestHookSlaveUploadValidate(t *testing.T) {
- asserts := assert.New(t)
- conf.SystemConfig.Mode = "slave"
- fs, err := NewAnonymousFileSystem()
- conf.SystemConfig.Mode = "master"
- asserts.NoError(err)
-
- // 正常
- {
- policy := serializer.UploadPolicy{
- SavePath: "",
- MaxSize: 10,
- AllowedExtension: nil,
- }
- file := fsctx.FileStream{Name: "1.txt", Size: 10}
- ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
- ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
- asserts.NoError(HookSlaveUploadValidate(ctx, fs))
- }
-
- // 尺寸太大
- {
- policy := serializer.UploadPolicy{
- SavePath: "",
- MaxSize: 10,
- AllowedExtension: nil,
- }
- file := fsctx.FileStream{Name: "1.txt", Size: 11}
- ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
- ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
- asserts.Equal(ErrFileSizeTooBig, HookSlaveUploadValidate(ctx, fs))
- }
-
- // 文件名非法
- {
- policy := serializer.UploadPolicy{
- SavePath: "",
- MaxSize: 10,
- AllowedExtension: nil,
- }
- file := fsctx.FileStream{Name: "/1.txt", Size: 10}
- ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
- ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
- asserts.Equal(ErrIllegalObjectName, HookSlaveUploadValidate(ctx, fs))
- }
-
- // 扩展名非法
- {
- policy := serializer.UploadPolicy{
- SavePath: "",
- MaxSize: 10,
- AllowedExtension: []string{"jpg"},
- }
- file := fsctx.FileStream{Name: "1.txt", Size: 10}
- ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
- ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
- asserts.Equal(ErrFileExtensionNotAllowed, HookSlaveUploadValidate(ctx, fs))
+func TestHookGenerateThumb(t *testing.T) {
+ a := assert.New(t)
+ mockHandler := &FileHeaderMock{}
+ fs := &FileSystem{
+ User: &model.User{
+ Model: gorm.Model{ID: 1},
+ },
+ Handler: mockHandler,
+ Policy: &model.Policy{Type: "local"},
}
-}
-
-type ClientMock struct {
- testMock.Mock
-}
-
-func (m ClientMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
- args := m.Called(method, target, body, opts)
- return args.Get(0).(*request.Response)
+ mockHandler.On("Delete", testMock.Anything, []string{"1.txt._thumb"}).Return([]string{}, nil)
+ a.NoError(HookGenerateThumb(context.Background(), fs, &fsctx.FileStream{
+ Model: &model.File{
+ SourceName: "1.txt",
+ },
+ }))
+ fs.Recycle()
+ mockHandler.AssertExpectations(t)
}
func TestSlaveAfterUpload(t *testing.T) {
@@ -598,7 +499,7 @@ func TestSlaveAfterUpload(t *testing.T) {
// 成功
{
- clientMock := ClientMock{}
+ clientMock := requestmock.RequestMock{}
clientMock.On(
"Request",
"POST",
@@ -613,19 +514,28 @@ func TestSlaveAfterUpload(t *testing.T) {
},
})
request.GeneralClient = clientMock
- ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{
+ file := &fsctx.FileStream{
Size: 10,
VirtualPath: "/my",
Name: "test.txt",
- })
- ctx = context.WithValue(ctx, fsctx.UploadPolicyCtx, serializer.UploadPolicy{
- CallbackURL: "http://test/callbakc",
- })
- ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/not_exist")
- err := SlaveAfterUpload(ctx, fs)
+ SavePath: "/not_exist",
+ }
+ err := SlaveAfterUpload(&serializer.UploadSession{Callback: "http://test/callbakc"})(context.Background(), fs, file)
clientMock.AssertExpectations(t)
asserts.NoError(err)
}
+
+ // 跳过回调
+ {
+ file := &fsctx.FileStream{
+ Size: 10,
+ VirtualPath: "/my",
+ Name: "test.txt",
+ SavePath: "/not_exist",
+ }
+ err := SlaveAfterUpload(&serializer.UploadSession{})(context.Background(), fs, file)
+ asserts.NoError(err)
+ }
}
func TestFileSystem_CleanHooks(t *testing.T) {
@@ -663,7 +573,7 @@ func TestHookCancelContext(t *testing.T) {
// empty ctx
{
- asserts.NoError(HookCancelContext(ctx, fs))
+ asserts.NoError(HookCancelContext(ctx, fs, nil))
select {
case <-ctx.Done():
t.Errorf("Channel should not be closed")
@@ -675,62 +585,99 @@ func TestHookCancelContext(t *testing.T) {
// with cancel ctx
{
ctx = context.WithValue(ctx, fsctx.CancelFuncCtx, cancel)
- asserts.NoError(HookCancelContext(ctx, fs))
+ asserts.NoError(HookCancelContext(ctx, fs, nil))
_, ok := <-ctx.Done()
asserts.False(ok)
}
}
-func TestHookGiveBackCapacity(t *testing.T) {
- asserts := assert.New(t)
- fs := &FileSystem{
- User: &model.User{
- Model: gorm.Model{ID: 1},
- Storage: 10,
+func TestHookClearFileHeaderSize(t *testing.T) {
+ a := assert.New(t)
+ fs := &FileSystem{}
+ file := &fsctx.FileStream{Size: 10}
+ a.NoError(HookClearFileHeaderSize(context.Background(), fs, file))
+ a.EqualValues(0, file.Size)
+}
+
+func TestHookTruncateFileTo(t *testing.T) {
+ a := assert.New(t)
+ fs := &FileSystem{}
+ file := &fsctx.FileStream{}
+ a.NoError(HookTruncateFileTo(0)(context.Background(), fs, file))
+
+ fs.Handler = local.Driver{}
+ a.Error(HookTruncateFileTo(0)(context.Background(), fs, file))
+}
+
+func TestHookChunkUploaded(t *testing.T) {
+ a := assert.New(t)
+ fs := &FileSystem{}
+ file := &fsctx.FileStream{
+ AppendStart: 10,
+ Size: 10,
+ Model: &model.File{
+ Model: gorm.Model{ID: 1},
},
}
- ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{Size: 1})
- // without once limit
- {
- asserts.NoError(HookGiveBackCapacity(ctx, fs))
- asserts.EqualValues(9, fs.User.Storage)
- asserts.NoError(HookGiveBackCapacity(ctx, fs))
- asserts.EqualValues(8, fs.User.Storage)
- }
+ mock.ExpectBegin()
+ mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(20, sqlmock.AnyArg(), 1, 0).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE(.+)users(.+)").
+ WithArgs(20, sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectCommit()
+ a.NoError(HookChunkUploaded(context.Background(), fs, file))
+ a.NoError(mock.ExpectationsWereMet())
+}
- // with once limit
- {
- ctx = context.WithValue(ctx, fsctx.ValidateCapacityOnceCtx, &sync.Once{})
- asserts.NoError(HookGiveBackCapacity(ctx, fs))
- asserts.EqualValues(7, fs.User.Storage)
- asserts.NoError(HookGiveBackCapacity(ctx, fs))
- asserts.EqualValues(7, fs.User.Storage)
+func TestHookChunkUploadFailed(t *testing.T) {
+ a := assert.New(t)
+ fs := &FileSystem{}
+ file := &fsctx.FileStream{
+ AppendStart: 10,
+ Size: 10,
+ Model: &model.File{
+ Model: gorm.Model{ID: 1},
+ },
}
+
+ mock.ExpectBegin()
+ mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(10, sqlmock.AnyArg(), 1, 0).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE(.+)users(.+)").
+ WithArgs(10, sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectCommit()
+ a.NoError(HookChunkUploadFailed(context.Background(), fs, file))
+ a.NoError(mock.ExpectationsWereMet())
}
-func TestHookValidateCapacityWithoutIncrease(t *testing.T) {
+func TestHookPopPlaceholderToFile(t *testing.T) {
a := assert.New(t)
- fs := &FileSystem{
- User: &model.User{
- Model: gorm.Model{ID: 1},
- Storage: 10,
- Group: model.Group{},
+ fs := &FileSystem{}
+ file := &fsctx.FileStream{
+ Model: &model.File{
+ Model: gorm.Model{ID: 1},
},
}
- ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fsctx.FileStream{Size: 1})
- // not enough
- {
- fs.User.Group.MaxStorage = 10
- a.Error(HookValidateCapacity(ctx, fs))
- a.EqualValues(10, fs.User.Storage)
- }
+ mock.ExpectBegin()
+ mock.ExpectExec("UPDATE(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectCommit()
+ a.NoError(HookPopPlaceholderToFile("1,1")(context.Background(), fs, file))
+ a.NoError(mock.ExpectationsWereMet())
+}
- // enough
- {
- fs.User.Group.MaxStorage = 11
- a.NoError(HookValidateCapacity(ctx, fs))
- a.EqualValues(10, fs.User.Storage)
+func TestHookDeleteUploadSession(t *testing.T) {
+ a := assert.New(t)
+ fs := &FileSystem{}
+ file := &fsctx.FileStream{
+ Model: &model.File{
+ Model: gorm.Model{ID: 1},
+ },
}
+
+ cache.Set(UploadSessionCachePrefix+"TestHookDeleteUploadSession", "", 0)
+ a.NoError(HookDeleteUploadSession("TestHookDeleteUploadSession")(context.Background(), fs, file))
+ _, ok := cache.Get(UploadSessionCachePrefix + "TestHookDeleteUploadSession")
+ a.False(ok)
}
diff --git a/pkg/filesystem/image.go b/pkg/filesystem/image.go
index 759ebca1..30e80bf4 100644
--- a/pkg/filesystem/image.go
+++ b/pkg/filesystem/image.go
@@ -3,7 +3,6 @@ package filesystem
import (
"context"
"fmt"
- "strconv"
"sync"
"runtime"
@@ -65,7 +64,7 @@ type Pool struct {
// Init 初始化任务池
func getThumbWorker() *Pool {
once.Do(func() {
- maxWorker := conf.ThumbConfig.MaxTaskCount
+ maxWorker := model.GetIntSetting("thumb_max_task_count", -1)
if maxWorker <= 0 {
maxWorker = runtime.GOMAXPROCS(0)
}
@@ -118,9 +117,9 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
// 生成缩略图
image.GetThumb(fs.GenerateThumbnailSize(w, h))
// 保存到文件
- err = image.Save(util.RelativePath(file.SourceName + conf.ThumbConfig.FileSuffix))
+ err = image.Save(util.RelativePath(file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")))
image = nil
- if conf.ThumbConfig.GCAfterGen {
+ if model.IsTrueVal(model.GetSettingByName("thumb_gc_after_gen")) {
util.Log().Debug("GenerateThumbnail runtime.GC")
runtime.GC()
}
@@ -139,17 +138,11 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
// 失败时删除缩略图文件
if err != nil {
- _, _ = fs.Handler.Delete(newCtx, []string{file.SourceName + conf.ThumbConfig.FileSuffix})
+ _, _ = fs.Handler.Delete(newCtx, []string{file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")})
}
}
// GenerateThumbnailSize 获取要生成的缩略图的尺寸
func (fs *FileSystem) GenerateThumbnailSize(w, h int) (uint, uint) {
- if conf.SystemConfig.Mode == "master" {
- options := model.GetSettingByNames("thumb_width", "thumb_height")
- w, _ := strconv.ParseUint(options["thumb_width"], 10, 32)
- h, _ := strconv.ParseUint(options["thumb_height"], 10, 32)
- return uint(w), uint(h)
- }
- return conf.ThumbConfig.MaxWidth, conf.ThumbConfig.MaxHeight
+ return uint(model.GetIntSetting("thumb_width", 400)), uint(model.GetIntSetting("thumb_width", 300))
}
diff --git a/pkg/filesystem/image_test.go b/pkg/filesystem/image_test.go
index 9678df92..cc33b190 100644
--- a/pkg/filesystem/image_test.go
+++ b/pkg/filesystem/image_test.go
@@ -2,13 +2,15 @@ package filesystem
import (
"context"
- "testing"
-
+ "errors"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
- "github.com/stretchr/testify/assert"
+ "github.com/cloudreve/Cloudreve/v3/pkg/request"
testMock "github.com/stretchr/testify/mock"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
)
func TestFileSystem_GetThumb(t *testing.T) {
@@ -47,3 +49,22 @@ func TestFileSystem_ThumbWorker(t *testing.T) {
getThumbWorker().releaseWorker()
})
}
+
+func TestFileSystem_GenerateThumbnail(t *testing.T) {
+ fs := &FileSystem{User: &model.User{}}
+
+ // 无法生成缩略图
+ {
+ fs.SetTargetFile(&[]model.File{{}})
+ fs.GenerateThumbnail(context.Background(), &model.File{})
+ }
+
+ // 无法获取文件数据
+ {
+ testHandller := new(FileHeaderMock)
+ testHandller.On("Get", testMock.Anything, "").Return(request.NopRSCloser{}, errors.New("error"))
+ fs.Handler = testHandller
+ fs.GenerateThumbnail(context.Background(), &model.File{Name: "test.png"})
+ testHandller.AssertExpectations(t)
+ }
+}
diff --git a/pkg/filesystem/manage.go b/pkg/filesystem/manage.go
index ddf83e6e..ad327457 100644
--- a/pkg/filesystem/manage.go
+++ b/pkg/filesystem/manage.go
@@ -221,6 +221,15 @@ func (fs *FileSystem) ListDeleteDirs(ctx context.Context, ids []uint) error {
if err != nil {
return ErrDBListObjects.WithError(err)
}
+
+ // 忽略根目录
+ for i := 0; i < len(folders); i++ {
+ if folders[i].ParentID == nil {
+ folders = append(folders[:i], folders[i+1:]...)
+ break
+ }
+ }
+
fs.SetTargetDir(&folders)
// 检索目录下的子文件
diff --git a/pkg/filesystem/manage_test.go b/pkg/filesystem/manage_test.go
index f3186ef7..da91691c 100644
--- a/pkg/filesystem/manage_test.go
+++ b/pkg/filesystem/manage_test.go
@@ -3,6 +3,8 @@ package filesystem
import (
"context"
"errors"
+ "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
+ testMock "github.com/stretchr/testify/mock"
"os"
"testing"
@@ -11,12 +13,10 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
- "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
- testMock "github.com/stretchr/testify/mock"
)
func TestFileSystem_ListPhysical(t *testing.T) {
@@ -296,12 +296,12 @@ func TestFileSystem_ListDeleteDirs(t *testing.T) {
{
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
- sqlmock.NewRows([]string{"id"}).
- AddRow(1).
- AddRow(2).
- AddRow(3),
+ sqlmock.NewRows([]string{"id", "parent_id"}).
+ AddRow(1, 0).
+ AddRow(2, 0).
+ AddRow(3, 0),
)
- mock.ExpectQuery("SELECT(.+)").
+ mock.ExpectQuery("SELECT(.+)files(.+)").
WithArgs(1, 2, 3).
WillReturnRows(
sqlmock.NewRows([]string{"id", "name"}).
@@ -316,21 +316,47 @@ func TestFileSystem_ListDeleteDirs(t *testing.T) {
asserts.NoError(mock.ExpectationsWereMet())
}
+ // 成功,忽略根目录
+ {
+ mock.ExpectQuery("SELECT(.+)").
+ WillReturnRows(
+ sqlmock.NewRows([]string{"id", "parent_id"}).
+ AddRow(1, 0).
+ AddRow(2, nil).
+ AddRow(3, 0),
+ )
+ mock.ExpectQuery("SELECT(.+)files(.+)").
+ WithArgs(1, 3).
+ WillReturnRows(
+ sqlmock.NewRows([]string{"id", "name"}).
+ AddRow(4, "1.txt").
+ AddRow(5, "2.txt").
+ AddRow(6, "3.txt"),
+ )
+ fs.CleanTargets()
+ err := fs.ListDeleteDirs(context.Background(), []uint{1})
+ asserts.NoError(err)
+ asserts.Len(fs.FileTarget, 3)
+ asserts.Len(fs.DirTarget, 2)
+ asserts.NoError(mock.ExpectationsWereMet())
+ }
+
// 检索文件发生错误
{
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
- sqlmock.NewRows([]string{"id"}).
- AddRow(1).
- AddRow(2).
- AddRow(3),
+ sqlmock.NewRows([]string{"id", "parent_id"}).
+ AddRow(1, 0).
+ AddRow(2, 0).
+ AddRow(3, 0),
)
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 2, 3).
WillReturnError(errors.New("error"))
+ fs.CleanTargets()
err := fs.ListDeleteDirs(context.Background(), []uint{1})
asserts.Error(err)
- asserts.Len(fs.DirTarget, 6)
+ asserts.Len(fs.DirTarget, 3)
asserts.NoError(mock.ExpectationsWereMet())
}
// 检索目录发生错误
@@ -349,7 +375,7 @@ func TestFileSystem_Delete(t *testing.T) {
cache.Set("pack_size_1", uint64(0), 0)
fs := &FileSystem{User: &model.User{
Model: gorm.Model{
- ID: 1,
+ ID: 0,
},
Storage: 3,
Group: model.Group{MaxStorage: 3},
@@ -361,10 +387,10 @@ func TestFileSystem_Delete(t *testing.T) {
fs.CleanTargets()
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
- sqlmock.NewRows([]string{"id"}).
- AddRow(1).
- AddRow(2).
- AddRow(3),
+ sqlmock.NewRows([]string{"id", "parent_id"}).
+ AddRow(1, 0).
+ AddRow(2, 0).
+ AddRow(3, 0),
)
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 2, 3).
@@ -380,18 +406,16 @@ func TestFileSystem_Delete(t *testing.T) {
// 删除文件记录
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
- WillReturnResult(sqlmock.NewResult(0, 3))
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectExec("DELETE(.+)").
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectExec("UPDATE(.+)users(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
// 删除对应分享
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)shares").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
- // 归还容量
- mock.ExpectBegin()
- mock.ExpectExec("UPDATE(.+)").
- WillReturnResult(sqlmock.NewResult(0, 3))
- mock.ExpectCommit()
// 删除目录
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
@@ -407,7 +431,6 @@ func TestFileSystem_Delete(t *testing.T) {
fs.DirTarget = []model.Folder{}
err := fs.Delete(ctx, []uint{1}, []uint{1}, true)
asserts.NoError(err)
- asserts.Equal(uint64(0), fs.User.Storage)
}
//全部成功
{
@@ -419,10 +442,10 @@ func TestFileSystem_Delete(t *testing.T) {
asserts.NoError(err)
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
- sqlmock.NewRows([]string{"id"}).
- AddRow(1).
- AddRow(2).
- AddRow(3),
+ sqlmock.NewRows([]string{"id", "parent_id"}).
+ AddRow(1, 0).
+ AddRow(2, 0).
+ AddRow(3, 0),
)
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 2, 3).
@@ -438,18 +461,16 @@ func TestFileSystem_Delete(t *testing.T) {
// 删除文件记录
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
- WillReturnResult(sqlmock.NewResult(0, 3))
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectExec("DELETE(.+)").
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectExec("UPDATE(.+)users(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
// 删除对应分享
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)shares").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
- // 归还容量
- mock.ExpectBegin()
- mock.ExpectExec("UPDATE(.+)").
- WillReturnResult(sqlmock.NewResult(0, 3))
- mock.ExpectCommit()
// 删除目录
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
@@ -465,7 +486,6 @@ func TestFileSystem_Delete(t *testing.T) {
fs.DirTarget = []model.Folder{}
err = fs.Delete(ctx, []uint{1}, []uint{1}, false)
asserts.NoError(err)
- asserts.Equal(uint64(0), fs.User.Storage)
}
}
@@ -573,7 +593,9 @@ func TestFileSystem_Rename(t *testing.T) {
Model: gorm.Model{
ID: 1,
},
- }}
+ },
+ Policy: &model.Policy{},
+ }
ctx := context.Background()
// 重命名文件 成功
@@ -683,7 +705,7 @@ func TestFileSystem_Rename(t *testing.T) {
// 新名字是文件，扩展名不合法
{
- fs.User.Policy.OptionsSerialized.FileType = []string{"txt"}
+ fs.Policy.OptionsSerialized.FileType = []string{"txt"}
err := fs.Rename(ctx, []uint{}, []uint{10}, "1.jpg")
asserts.Error(err)
asserts.Equal(ErrIllegalObjectName, err)
@@ -691,7 +713,7 @@ func TestFileSystem_Rename(t *testing.T) {
// 新名字是目录，不应该检测扩展名
{
- fs.User.Policy.OptionsSerialized.FileType = []string{"txt"}
+ fs.Policy.OptionsSerialized.FileType = []string{"txt"}
mock.ExpectQuery("SELECT(.+)folders(.+)").
WithArgs(10, 1).
WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
diff --git a/pkg/filesystem/upload.go b/pkg/filesystem/upload.go
index 38d42eff..5ff90a18 100644
--- a/pkg/filesystem/upload.go
+++ b/pkg/filesystem/upload.go
@@ -91,40 +91,17 @@ func (fs *FileSystem) Upload(ctx context.Context, file *fsctx.FileStream) (err e
// TODO 完善测试
func (fs *FileSystem) GenerateSavePath(ctx context.Context, file fsctx.FileHeader) string {
fileInfo := file.Info()
-
- if fs.User.Model.ID != 0 {
- return path.Join(
- fs.Policy.GeneratePath(
- fs.User.Model.ID,
- fileInfo.VirtualPath,
- ),
- fs.Policy.GenerateFileName(
- fs.User.Model.ID,
- fileInfo.FileName,
- ),
- )
- }
-
- // 匿名文件系统尝试根据上下文中的上传策略生成路径
- var anonymousPolicy model.Policy
- if policy, ok := ctx.Value(fsctx.UploadPolicyCtx).(serializer.UploadPolicy); ok {
- anonymousPolicy = model.Policy{
- Type: "remote",
- AutoRename: policy.AutoRename,
- DirNameRule: policy.SavePath,
- FileNameRule: policy.FileName,
- }
- }
return path.Join(
- anonymousPolicy.GeneratePath(
- 0,
- "",
+ fs.Policy.GeneratePath(
+ fs.User.Model.ID,
+ fileInfo.VirtualPath,
),
- anonymousPolicy.GenerateFileName(
- 0,
+ fs.Policy.GenerateFileName(
+ fs.User.Model.ID,
fileInfo.FileName,
),
)
+
}
// CancelUpload 监测客户端取消上传
@@ -174,9 +151,6 @@ func (fs *FileSystem) CreateUploadSession(ctx context.Context, file *fsctx.FileS
fs.Use("BeforeUpload", HookValidateFile)
fs.Use("BeforeUpload", HookValidateCapacity)
- if !fs.Policy.IsUploadPlaceholderWithSize() {
- fs.Use("AfterUpload", HookClearFileHeaderSize)
- }
// 验证文件规格
if err := fs.Upload(ctx, file); err != nil {
@@ -202,6 +176,9 @@ func (fs *FileSystem) CreateUploadSession(ctx context.Context, file *fsctx.FileS
}
// 创建占位符
+ if !fs.Policy.IsUploadPlaceholderWithSize() {
+ fs.Use("AfterUpload", HookClearFileHeaderSize)
+ }
fs.Use("AfterUpload", GenericAfterUpload)
if err := fs.Upload(ctx, file); err != nil {
return nil, err
diff --git a/pkg/filesystem/upload_test.go b/pkg/filesystem/upload_test.go
index 74f7abe3..30f24ccd 100644
--- a/pkg/filesystem/upload_test.go
+++ b/pkg/filesystem/upload_test.go
@@ -3,14 +3,7 @@ package filesystem
import (
"context"
"errors"
- "io"
- "io/ioutil"
- "net/http"
- "net/http/httptest"
- "net/url"
- "strings"
- "testing"
-
+ "github.com/DATA-DOG/go-sqlmock"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
@@ -20,12 +13,33 @@ import (
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "testing"
)
type FileHeaderMock struct {
testMock.Mock
}
+func (m FileHeaderMock) Put(ctx context.Context, file fsctx.FileHeader) error {
+ args := m.Called(ctx, file)
+ return args.Error(0)
+}
+
+func (m FileHeaderMock) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
+ args := m.Called(ctx, ttl, uploadSession, file)
+ return args.Get(0).(*serializer.UploadCredential), args.Error(1)
+}
+
+func (m FileHeaderMock) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+ args := m.Called(ctx, uploadSession)
+ return args.Error(0)
+}
+
func (m FileHeaderMock) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
args := m.Called(ctx, path, recursive)
return args.Get(0).([]response.Object), args.Error(1)
@@ -36,11 +50,6 @@ func (m FileHeaderMock) Get(ctx context.Context, path string) (response.RSCloser
return args.Get(0).(response.RSCloser), args.Error(1)
}
-func (m FileHeaderMock) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
- args := m.Called(ctx, file, dst)
- return args.Error(0)
-}
-
func (m FileHeaderMock) Delete(ctx context.Context, files []string) ([]string, error) {
args := m.Called(ctx, files)
return args.Get(0).([]string), args.Error(1)
@@ -56,11 +65,6 @@ func (m FileHeaderMock) Source(ctx context.Context, path string, url url.URL, ex
return args.Get(0).(string), args.Error(1)
}
-func (m FileHeaderMock) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
- args := m.Called(ctx, ttl, uploadSession)
- return args.Get(0).(serializer.UploadCredential), args.Error(1)
-}
-
func TestFileSystem_Upload(t *testing.T) {
asserts := assert.New(t)
@@ -73,10 +77,10 @@ func TestFileSystem_Upload(t *testing.T) {
Model: gorm.Model{
ID: 1,
},
- Policy: model.Policy{
- AutoRename: false,
- DirNameRule: "{path}",
- },
+ },
+ Policy: &model.Policy{
+ AutoRename: false,
+ DirNameRule: "{path}",
},
}
ctx, cancel := context.WithCancel(context.Background())
@@ -84,7 +88,7 @@ func TestFileSystem_Upload(t *testing.T) {
c.Request, _ = http.NewRequest("POST", "/", nil)
ctx = context.WithValue(ctx, fsctx.GinCtx, c)
cancel()
- file := fsctx.FileStream{
+ file := &fsctx.FileStream{
Size: 5,
VirtualPath: "/",
Name: "1.txt",
@@ -94,17 +98,17 @@ func TestFileSystem_Upload(t *testing.T) {
// 正常，上下文已指定源文件
testHandler = new(FileHeaderMock)
- testHandler.On("Put", testMock.Anything, testMock.Anything, "123/123.txt").Return(nil)
+ testHandler.On("Put", testMock.Anything, testMock.Anything).Return(nil)
fs = &FileSystem{
Handler: testHandler,
User: &model.User{
Model: gorm.Model{
ID: 1,
},
- Policy: model.Policy{
- AutoRename: false,
- DirNameRule: "{path}",
- },
+ },
+ Policy: &model.Policy{
+ AutoRename: false,
+ DirNameRule: "{path}",
},
}
ctx, cancel = context.WithCancel(context.Background())
@@ -113,7 +117,7 @@ func TestFileSystem_Upload(t *testing.T) {
ctx = context.WithValue(ctx, fsctx.GinCtx, c)
ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{SourceName: "123/123.txt"})
cancel()
- file = fsctx.FileStream{
+ file = &fsctx.FileStream{
Size: 5,
VirtualPath: "/",
Name: "1.txt",
@@ -123,7 +127,7 @@ func TestFileSystem_Upload(t *testing.T) {
asserts.NoError(err)
// BeforeUpload 返回错误
- fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem) error {
+ fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
return errors.New("error")
})
err = fs.Upload(ctx, file)
@@ -133,7 +137,7 @@ func TestFileSystem_Upload(t *testing.T) {
// 上传文件失败
testHandler2 := new(FileHeaderMock)
- testHandler2.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(errors.New("error"))
+ testHandler2.On("Put", testMock.Anything, testMock.Anything).Return(errors.New("error"))
fs.Handler = testHandler2
err = fs.Upload(ctx, file)
asserts.Error(err)
@@ -141,12 +145,12 @@ func TestFileSystem_Upload(t *testing.T) {
// AfterUpload失败
testHandler3 := new(FileHeaderMock)
- testHandler3.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(nil)
+ testHandler3.On("Put", testMock.Anything, testMock.Anything).Return(nil)
fs.Handler = testHandler3
- fs.Use("AfterUpload", func(ctx context.Context, fs *FileSystem) error {
+ fs.Use("AfterUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
return errors.New("error")
})
- fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem) error {
+ fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
return errors.New("error")
})
err = fs.Upload(ctx, file)
@@ -155,43 +159,39 @@ func TestFileSystem_Upload(t *testing.T) {
}
-func TestFileSystem_GenerateSavePath_Anonymous(t *testing.T) {
- asserts := assert.New(t)
- fs := FileSystem{User: &model.User{}}
- ctx := context.WithValue(
- context.Background(),
- fsctx.UploadPolicyCtx,
- serializer.UploadPolicy{
- SavePath: "{randomkey16}",
- AutoRename: false,
- },
- )
-
- savePath := fs.GenerateSavePath(ctx, fsctx.FileStream{
- Name: "test.test",
- })
- asserts.Len(savePath, 26)
- asserts.Contains(savePath, "test.test")
-}
-
func TestFileSystem_GetUploadToken(t *testing.T) {
asserts := assert.New(t)
- fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}}
+ fs := FileSystem{
+ User: &model.User{Model: gorm.Model{ID: 1}},
+ Policy: &model.Policy{},
+ }
ctx := context.Background()
// 成功
{
cache.SetSettings(map[string]string{
- "upload_credential_timeout": "10",
- "upload_session_timeout": "10",
+ "upload_session_timeout": "10",
}, "setting_")
testHandler := new(FileHeaderMock)
- testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{Token: "test"}, nil)
+ testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything, testMock.Anything).Return(&serializer.UploadCredential{Credential: "test"}, nil)
fs.Handler = testHandler
- res, err := fs.CreateUploadSession(ctx, "/", 10, "123")
+ mock.ExpectQuery("SELECT(.+)").
+ WithArgs(1).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
+ mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found"))
+ mock.ExpectBegin()
+ mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectCommit()
+ res, err := fs.CreateUploadSession(ctx, &fsctx.FileStream{
+ Size: 0,
+ Name: "file",
+ VirtualPath: "/",
+ })
+ asserts.NoError(mock.ExpectationsWereMet())
testHandler.AssertExpectations(t)
asserts.NoError(err)
- asserts.Equal("test", res.Token)
+ asserts.Equal("test", res.Credential)
}
// 无法获取上传凭证
@@ -201,9 +201,22 @@ func TestFileSystem_GetUploadToken(t *testing.T) {
"upload_session_timeout": "10",
}, "setting_")
testHandler := new(FileHeaderMock)
- testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{}, errors.New("error"))
+ testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything, testMock.Anything).Return(&serializer.UploadCredential{}, errors.New("error"))
fs.Handler = testHandler
- _, err := fs.CreateUploadSession(ctx, "/", 10, "123")
+ mock.ExpectQuery("SELECT(.+)").
+ WithArgs(1).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
+ mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found"))
+ mock.ExpectBegin()
+ mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectCommit()
+ _, err := fs.CreateUploadSession(ctx, &fsctx.FileStream{
+ Size: 0,
+ Name: "file",
+ VirtualPath: "/",
+ })
+ asserts.NoError(mock.ExpectationsWereMet())
testHandler.AssertExpectations(t)
asserts.Error(err)
}
@@ -211,27 +224,41 @@ func TestFileSystem_GetUploadToken(t *testing.T) {
func TestFileSystem_UploadFromStream(t *testing.T) {
asserts := assert.New(t)
- fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}}
+ fs := FileSystem{
+ User: &model.User{
+ Model: gorm.Model{ID: 1},
+ Policy: model.Policy{Type: "mock"},
+ },
+ Policy: &model.Policy{Type: "mock"},
+ }
ctx := context.Background()
- err := fs.UploadFromStream(ctx, ioutil.NopCloser(strings.NewReader("123")), "/1.txt", 1)
+ err := fs.UploadFromStream(ctx, &fsctx.FileStream{
+ File: ioutil.NopCloser(strings.NewReader("123")),
+ }, true)
asserts.Error(err)
}
func TestFileSystem_UploadFromPath(t *testing.T) {
asserts := assert.New(t)
- fs := FileSystem{User: &model.User{Policy: model.Policy{Type: "mock"}, Model: gorm.Model{ID: 1}}}
+ fs := FileSystem{
+ User: &model.User{
+ Model: gorm.Model{ID: 1},
+ Policy: model.Policy{Type: "mock"},
+ },
+ Policy: &model.Policy{Type: "mock"},
+ }
ctx := context.Background()
// 文件不存在
{
- err := fs.UploadFromPath(ctx, "test/not_exist", "/", true)
+ err := fs.UploadFromPath(ctx, "test/not_exist", "/", fsctx.Overwrite)
asserts.Error(err)
}
// 文存在,上传失败
{
- err := fs.UploadFromPath(ctx, "tests/test.zip", "/", true)
+ err := fs.UploadFromPath(ctx, "tests/test.zip", "/", fsctx.Overwrite)
asserts.Error(err)
}
}
diff --git a/pkg/filesystem/validator_test.go b/pkg/filesystem/validator_test.go
index 8a39ae28..8f685f27 100644
--- a/pkg/filesystem/validator_test.go
+++ b/pkg/filesystem/validator_test.go
@@ -68,10 +68,9 @@ func TestFileSystem_ValidateFileSize(t *testing.T) {
asserts := assert.New(t)
ctx := context.Background()
fs := FileSystem{
- User: &model.User{
- Policy: model.Policy{
- MaxSize: 10,
- },
+ User: &model.User{},
+ Policy: &model.Policy{
+ MaxSize: 10,
},
}
@@ -80,7 +79,7 @@ func TestFileSystem_ValidateFileSize(t *testing.T) {
asserts.False(fs.ValidateFileSize(ctx, 11))
// 无限制
- fs.User.Policy.MaxSize = 0
+ fs.Policy.MaxSize = 0
asserts.True(fs.ValidateFileSize(ctx, 11))
}
@@ -88,11 +87,10 @@ func TestFileSystem_ValidateExtension(t *testing.T) {
asserts := assert.New(t)
ctx := context.Background()
fs := FileSystem{
- User: &model.User{
- Policy: model.Policy{
- OptionsSerialized: model.PolicyOption{
- FileType: nil,
- },
+ User: &model.User{},
+ Policy: &model.Policy{
+ OptionsSerialized: model.PolicyOption{
+ FileType: nil,
},
},
}
@@ -100,11 +98,11 @@ func TestFileSystem_ValidateExtension(t *testing.T) {
asserts.True(fs.ValidateExtension(ctx, "1"))
asserts.True(fs.ValidateExtension(ctx, "1.txt"))
- fs.User.Policy.OptionsSerialized.FileType = []string{}
+ fs.Policy.OptionsSerialized.FileType = []string{}
asserts.True(fs.ValidateExtension(ctx, "1"))
asserts.True(fs.ValidateExtension(ctx, "1.txt"))
- fs.User.Policy.OptionsSerialized.FileType = []string{"txt", "jpg"}
+ fs.Policy.OptionsSerialized.FileType = []string{"txt", "jpg"}
asserts.False(fs.ValidateExtension(ctx, "1"))
asserts.False(fs.ValidateExtension(ctx, "1.jpg.png"))
asserts.True(fs.ValidateExtension(ctx, "1.txt"))
diff --git a/pkg/mocks/mocks.go b/pkg/mocks/mocks.go
index 2b085f16..01c450b8 100644
--- a/pkg/mocks/mocks.go
+++ b/pkg/mocks/mocks.go
@@ -7,11 +7,9 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/balancer"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
- "github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/task"
testMock "github.com/stretchr/testify/mock"
- "io"
)
type NodePoolMock struct {
@@ -151,11 +149,3 @@ func (t TaskPoolMock) Add(num int) {
func (t TaskPoolMock) Submit(job task.Job) {
t.Called(job)
}
-
-type RequestMock struct {
- testMock.Mock
-}
-
-func (r RequestMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
- return r.Called(method, target, body, opts).Get(0).(*request.Response)
-}
diff --git a/pkg/mocks/remoteclientmock/mock.go b/pkg/mocks/remoteclientmock/mock.go
new file mode 100644
index 00000000..dee47352
--- /dev/null
+++ b/pkg/mocks/remoteclientmock/mock.go
@@ -0,0 +1,32 @@
+package remoteclientmock
+
+import (
+ "context"
+ "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
+ "github.com/cloudreve/Cloudreve/v3/pkg/serializer"
+ "github.com/stretchr/testify/mock"
+)
+
+type RemoteClientMock struct {
+ mock.Mock
+}
+
+func (r *RemoteClientMock) CreateUploadSession(ctx context.Context, session *serializer.UploadSession, ttl int64) error {
+ return r.Called(ctx, session, ttl).Error(0)
+}
+
+func (r *RemoteClientMock) GetUploadURL(ttl int64, sessionID string) (string, string, error) {
+ args := r.Called(ttl, sessionID)
+
+ return args.String(0), args.String(1), args.Error(2)
+}
+
+func (r *RemoteClientMock) Upload(ctx context.Context, file fsctx.FileHeader) error {
+ args := r.Called(ctx, file)
+ return args.Error(0)
+}
+
+func (r *RemoteClientMock) DeleteUploadSession(ctx context.Context, sessionID string) error {
+ args := r.Called(ctx, sessionID)
+ return args.Error(0)
+}
diff --git a/pkg/mocks/requestmock/request.go b/pkg/mocks/requestmock/request.go
new file mode 100644
index 00000000..7e6ca1b1
--- /dev/null
+++ b/pkg/mocks/requestmock/request.go
@@ -0,0 +1,15 @@
+package requestmock
+
+import (
+ "github.com/cloudreve/Cloudreve/v3/pkg/request"
+ "github.com/stretchr/testify/mock"
+ "io"
+)
+
+type RequestMock struct {
+ mock.Mock
+}
+
+func (r RequestMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
+ return r.Called(method, target, body, opts).Get(0).(*request.Response)
+}
diff --git a/pkg/request/request_test.go b/pkg/request/request_test.go
index 7b1a48d8..1f00f7e8 100644
--- a/pkg/request/request_test.go
+++ b/pkg/request/request_test.go
@@ -82,6 +82,8 @@ func TestHTTPClient_Request(t *testing.T) {
WithTimeout(time.Duration(1)*time.Microsecond),
WithCredential(auth.HMACAuth{SecretKey: []byte("123")}, 10),
WithContext(context.Background()),
+ WithoutHeader([]string{"s s", "s s"}),
+ WithMasterMeta(),
)
asserts.Error(resp.Err)
asserts.Nil(resp.Response)
diff --git a/pkg/serializer/explorer_test.go b/pkg/serializer/explorer_test.go
new file mode 100644
index 00000000..00c9efc9
--- /dev/null
+++ b/pkg/serializer/explorer_test.go
@@ -0,0 +1,15 @@
+package serializer
+
+import (
+ model "github.com/cloudreve/Cloudreve/v3/models"
+ "github.com/stretchr/testify/assert"
+ "testing"
+)
+
+func TestBuildObjectList(t *testing.T) {
+ a := assert.New(t)
+ res := BuildObjectList(1, []Object{{}, {}}, &model.Policy{})
+ a.NotEmpty(res.Parent)
+ a.NotNil(res.Policy)
+ a.Len(res.Objects, 2)
+}
diff --git a/pkg/serializer/upload.go b/pkg/serializer/upload.go
index 225c4934..4e7150b1 100644
--- a/pkg/serializer/upload.go
+++ b/pkg/serializer/upload.go
@@ -1,9 +1,7 @@
package serializer
import (
- "encoding/base64"
"encoding/gob"
- "encoding/json"
model "github.com/cloudreve/Cloudreve/v3/models"
"time"
)
@@ -20,19 +18,18 @@ type UploadPolicy struct {
// UploadCredential 返回给客户端的上传凭证
type UploadCredential struct {
- SessionID string `json:"sessionID"`
- ChunkSize uint64 `json:"chunkSize"` // 分块大小，0 为部分快
- Expires int64 `json:"expires"` // 上传凭证过期时间， Unix 时间戳
- UploadURLs []string `json:"uploadURLs"`
- Credential string `json:"credential"`
-
- Token string `json:"token"`
- Policy string `json:"policy"`
- Path string `json:"path"` // 存储路径
- AccessKey string `json:"ak"`
- KeyTime string `json:"key_time,omitempty"` // COS用有效期
- Callback string `json:"callback,omitempty"` // 回调地址
- Key string `json:"key,omitempty"` // 文件标识符，通常为回调key
+ SessionID string `json:"sessionID"`
+ ChunkSize uint64 `json:"chunkSize"` // 分块大小，0 为部分快
+ Expires int64 `json:"expires"` // 上传凭证过期时间， Unix 时间戳
+ UploadURLs []string `json:"uploadURLs,omitempty"`
+ Credential string `json:"credential,omitempty"`
+ UploadID string `json:"uploadID,omitempty"`
+ Callback string `json:"callback,omitempty"` // 回调地址
+ Path string `json:"path,omitempty"` // 存储路径
+ AccessKey string `json:"ak,omitempty"`
+ KeyTime string `json:"keyTime,omitempty"` // COS用有效期
+ Policy string `json:"policy,omitempty"`
+ CompleteURL string `json:"completeURL,omitempty"`
}
// UploadSession 上传会话
@@ -47,6 +44,9 @@ type UploadSession struct {
Policy model.Policy
Callback string // 回调 URL 地址
CallbackSecret string // 回调 URL
+ UploadURL string
+ UploadID string
+ Credential string
}
// UploadCallback 上传回调正文
@@ -62,32 +62,3 @@ type GeneralUploadCallbackFailed struct {
func init() {
gob.Register(UploadSession{})
}
-
-// DecodeUploadPolicy 反序列化Header中携带的上传策略
-func DecodeUploadPolicy(raw string) (*UploadPolicy, error) {
- var res UploadPolicy
-
- rawJSON, err := base64.StdEncoding.DecodeString(raw)
- if err != nil {
- return nil, err
- }
-
- err = json.Unmarshal(rawJSON, &res)
- if err != nil {
- return nil, err
- }
-
- return &res, err
-}
-
-// EncodeUploadPolicy 序列化Header中携带的上传策略
-func (policy *UploadPolicy) EncodeUploadPolicy() (string, error) {
- jsonRes, err := json.Marshal(policy)
- if err != nil {
- return "", err
- }
-
- res := base64.StdEncoding.EncodeToString(jsonRes)
- return res, nil
-
-}
diff --git a/pkg/serializer/upload_test.go b/pkg/serializer/upload_test.go
deleted file mode 100644
index 2ed89f1d..00000000
--- a/pkg/serializer/upload_test.go
+++ /dev/null
@@ -1,63 +0,0 @@
-package serializer
-
-import (
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-func TestDecodeUploadPolicy(t *testing.T) {
- asserts := assert.New(t)
-
- testCases := []struct {
- input string
- expectError bool
- expectNil bool
- expectRes *UploadPolicy
- }{
- {
- "错误的base64字符",
- true,
- true,
- &UploadPolicy{},
- },
- {
- "6ZSZ6K+v55qESlNPTuWtl+espg==",
- true,
- true,
- &UploadPolicy{},
- },
- {
- "e30=",
- false,
- false,
- &UploadPolicy{},
- },
- {
- "eyJjYWxsYmFja191cmwiOiJ0ZXN0In0=",
- false,
- false,
- &UploadPolicy{CallbackURL: "test"},
- },
- }
-
- for _, testCase := range testCases {
- res, err := DecodeUploadPolicy(testCase.input)
- if testCase.expectError {
- asserts.Error(err)
- }
- if testCase.expectNil {
- asserts.Nil(res)
- }
- if !testCase.expectNil {
- asserts.Equal(testCase.expectRes, res)
- }
- }
-}
-
-func TestUploadPolicy_EncodeUploadPolicy(t *testing.T) {
- asserts := assert.New(t)
- testPolicy := UploadPolicy{}
- res, err := testPolicy.EncodeUploadPolicy()
- asserts.NoError(err)
- asserts.NotEmpty(res)
-}
diff --git a/pkg/serializer/user_test.go b/pkg/serializer/user_test.go
index cb70fcc8..29421861 100644
--- a/pkg/serializer/user_test.go
+++ b/pkg/serializer/user_test.go
@@ -29,23 +29,19 @@ func TestMain(m *testing.M) {
func TestBuildUser(t *testing.T) {
asserts := assert.New(t)
- user := model.User{
- Policy: model.Policy{MaxSize: 1024 * 1024},
- }
+ user := model.User{}
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}))
res := BuildUser(user)
asserts.NoError(mock.ExpectationsWereMet())
- asserts.Equal("1.00mb", res.Policy.MaxSize)
+ asserts.NotNil(res)
}
func TestBuildUserResponse(t *testing.T) {
asserts := assert.New(t)
- user := model.User{
- Policy: model.Policy{MaxSize: 1024 * 1024},
- }
+ user := model.User{}
res := BuildUserResponse(user)
- asserts.Equal("1.00mb", res.Data.(User).Policy.MaxSize)
+ asserts.NotNil(res)
}
func TestBuildUserStorageResponse(t *testing.T) {
diff --git a/pkg/task/import_test.go b/pkg/task/import_test.go
index ea22cb83..c75e17cd 100644
--- a/pkg/task/import_test.go
+++ b/pkg/task/import_test.go
@@ -182,6 +182,7 @@ func TestImportTask_Do(t *testing.T) {
// 插入文件记录
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(2, 1))
+ mock.ExpectExec("UPDATE(.+)users(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(2, 1))
mock.ExpectCommit()
task.Do()
diff --git a/pkg/task/tranfer.go b/pkg/task/tranfer.go
index b8023245..596ae9eb 100644
--- a/pkg/task/tranfer.go
+++ b/pkg/task/tranfer.go
@@ -117,7 +117,6 @@ func (job *TransferTask) Do() {
}
// 切换为从机节点处理上传
- fs.SetPolicyFromPath(path.Dir(dst))
fs.SwitchToSlaveHandler(node)
err = fs.UploadFromStream(context.Background(), &fsctx.FileStream{
File: nil,
diff --git a/pkg/thumb/image.go b/pkg/thumb/image.go
index 7b026206..69c73a3b 100644
--- a/pkg/thumb/image.go
+++ b/pkg/thumb/image.go
@@ -12,7 +12,6 @@ import (
"strings"
model "github.com/cloudreve/Cloudreve/v3/models"
- "github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
//"github.com/nfnt/resize"
@@ -78,11 +77,11 @@ func (image *Thumb) Save(path string) (err error) {
return err
}
defer out.Close()
- switch conf.ThumbConfig.EncodeMethod {
+ switch model.GetSettingByNameWithDefault("thumb_encode_method", "jpg") {
case "png":
err = png.Encode(out, image.src)
default:
- err = jpeg.Encode(out, image.src, &jpeg.Options{Quality: conf.ThumbConfig.EncodeQuality})
+ err = jpeg.Encode(out, image.src, &jpeg.Options{Quality: model.GetIntSetting("thumb_encode_quality", 85)})
}
return err
diff --git a/pkg/util/path.go b/pkg/util/path.go
index ff51d579..2dd8aefe 100644
--- a/pkg/util/path.go
+++ b/pkg/util/path.go
@@ -56,3 +56,4 @@ func RelativePath(name string) string {
e, _ := os.Executable()
return filepath.Join(filepath.Dir(e), name)
}
+
diff --git a/pkg/webdav/internal/xml/xml.go b/pkg/webdav/internal/xml/xml.go
index 5b79cbec..be282bbd 100644
--- a/pkg/webdav/internal/xml/xml.go
+++ b/pkg/webdav/internal/xml/xml.go
@@ -1040,7 +1040,7 @@ Input:
d.buf.WriteByte(';')
n, err := strconv.ParseUint(s, base, 64)
if err == nil && n <= unicode.MaxRune {
- text = string(n)
+ text = string(rune(n))
haveText = true
}
}
@@ -1063,7 +1063,7 @@ Input:
if isName(name) {
s := string(name)
if r, ok := entity[s]; ok {
- text = string(r)
+ text = string(rune(r))
haveText = true
} else if d.Entity != nil {
text, haveText = d.Entity[s]
diff --git a/routers/controllers/callback.go b/routers/controllers/callback.go
index 9179ab58..dade5662 100644
--- a/routers/controllers/callback.go
+++ b/routers/controllers/callback.go
@@ -112,7 +112,6 @@ func COSCallback(c *gin.Context) {
// S3Callback S3上传完成客户端回调
func S3Callback(c *gin.Context) {
- c.Header("Access-Control-Allow-Origin", "*")
var callbackBody callback.S3Callback
if err := c.ShouldBindQuery(&callbackBody); err == nil {
res := callbackBody.PreProcess(c)
diff --git a/routers/controllers/file.go b/routers/controllers/file.go
index 9b0d2486..1141d7ab 100644
--- a/routers/controllers/file.go
+++ b/routers/controllers/file.go
@@ -5,7 +5,7 @@ import (
"fmt"
"net/http"
- "github.com/cloudreve/Cloudreve/v3/pkg/conf"
+ model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
@@ -174,7 +174,7 @@ func Thumb(c *gin.Context) {
}
defer resp.Content.Close()
- http.ServeContent(c.Writer, c.Request, "thumb."+conf.ThumbConfig.EncodeMethod, fs.FileTarget[0].UpdatedAt, resp.Content)
+ http.ServeContent(c.Writer, c.Request, "thumb."+model.GetSettingByNameWithDefault("thumb_encode_method", "jpg"), fs.FileTarget[0].UpdatedAt, resp.Content)
}
diff --git a/routers/controllers/site.go b/routers/controllers/site.go
index b066397c..5598605e 100644
--- a/routers/controllers/site.go
+++ b/routers/controllers/site.go
@@ -49,7 +49,7 @@ func Ping(c *gin.Context) {
c.JSON(200, serializer.Response{
Code: 0,
- Data: conf.BackendVersion + conf.IsPro,
+ Data: version,
})
}
diff --git a/routers/controllers/slave.go b/routers/controllers/slave.go
index 5a0d2774..e1e7de22 100644
--- a/routers/controllers/slave.go
+++ b/routers/controllers/slave.go
@@ -2,8 +2,6 @@ package controllers
import (
"context"
- "net/url"
- "strconv"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
diff --git a/routers/file_router_test.go b/routers/file_router_test.go
deleted file mode 100644
index 41ff0721..00000000
--- a/routers/file_router_test.go
+++ /dev/null
@@ -1,180 +0,0 @@
-package routers
-
-import (
- "bytes"
- "encoding/json"
- "net/http"
- "net/http/httptest"
- "strings"
- "testing"
-
- "github.com/cloudreve/Cloudreve/v3/middleware"
- model "github.com/cloudreve/Cloudreve/v3/models"
- "github.com/cloudreve/Cloudreve/v3/pkg/serializer"
- "github.com/cloudreve/Cloudreve/v3/service/explorer"
- "github.com/stretchr/testify/assert"
-)
-
-func TestListDirectoryRoute(t *testing.T) {
- switchToMemDB()
- asserts := assert.New(t)
- router := InitMasterRouter()
- w := httptest.NewRecorder()
-
- // 成功
- req, _ := http.NewRequest(
- "GET",
- "/api/v3/directory/",
- nil,
- )
- middleware.SessionMock = map[string]interface{}{"user_id": 1}
- router.ServeHTTP(w, req)
- asserts.Equal(200, w.Code)
- resJSON := &serializer.Response{}
- err := json.Unmarshal(w.Body.Bytes(), resJSON)
- asserts.NoError(err)
- asserts.Equal(0, resJSON.Code)
-
- w.Body.Reset()
-
-}
-
-func TestLocalFileUpload(t *testing.T) {
- switchToMemDB()
- asserts := assert.New(t)
- router := InitMasterRouter()
- w := httptest.NewRecorder()
- middleware.SessionMock = map[string]interface{}{"user_id": 1}
-
- testCases := []struct {
- GetRequest func() *http.Request
- ExpectCode int
- RollBack func()
- }{
- // 文件大小指定错误
- {
- GetRequest: func() *http.Request {
- req, _ := http.NewRequest(
- "POST",
- "/api/v3/file/upload",
- nil,
- )
- req.Header.Add("Content-Length", "ddf")
- return req
- },
- ExpectCode: 40001,
- },
- // 返回错误
- {
- GetRequest: func() *http.Request {
- req, _ := http.NewRequest(
- "POST",
- "/api/v3/file/upload",
- strings.NewReader("2333"),
- )
- req.Header.Add("Content-Length", "4")
- req.Header.Add("X-Cr-FileName", "大地的%sfsf")
- return req
- },
- ExpectCode: 40002,
- },
- // 成功
- {
- GetRequest: func() *http.Request {
- req, _ := http.NewRequest(
- "POST",
- "/api/v3/file/upload",
- strings.NewReader("2333"),
- )
- req.Header.Add("Content-Length", "4")
- req.Header.Add("X-Cr-FileName", "TestFileUploadRoute.txt")
- req.Header.Add("X-Cr-Path", "/")
- return req
- },
- ExpectCode: 0,
- },
- }
-
- for key, testCase := range testCases {
- req := testCase.GetRequest()
- router.ServeHTTP(w, req)
- asserts.Equal(200, w.Code)
- resJSON := &serializer.Response{}
- err := json.Unmarshal(w.Body.Bytes(), resJSON)
- asserts.NoError(err, "测试用例%d", key)
- asserts.Equal(testCase.ExpectCode, resJSON.Code, "测试用例%d", key)
- if testCase.RollBack != nil {
- testCase.RollBack()
- }
- w.Body.Reset()
- }
-
-}
-
-func TestObjectDelete(t *testing.T) {
- asserts := assert.New(t)
- router := InitMasterRouter()
- w := httptest.NewRecorder()
- middleware.SessionMock = map[string]interface{}{"user_id": 1}
-
- testCases := []struct {
- Mock []string
- GetRequest func() *http.Request
- ExpectCode int
- RollBack []string
- }{
- // 路径不存在，返回无错误
- {
- GetRequest: func() *http.Request {
- body := explorer.ItemService{
- Items: []uint{1},
- }
- bodyStr, _ := json.Marshal(body)
- req, _ := http.NewRequest(
- "DELETE",
- "/api/v3/object",
- bytes.NewReader(bodyStr),
- )
- return req
- },
- ExpectCode: 0,
- },
- // 文件删除失败，返回203
- {
- Mock: []string{"INSERT INTO `files` (`id`, `created_at`, `updated_at`, `deleted_at`, `name`, `source_name`, `user_id`, `size`, `pic_info`, `folder_id`, `policy_id`) VALUES(5, '2019-11-30 07:08:33', '2019-11-30 07:08:33', NULL, 'pigeon.zip', '65azil3B_pigeon.zip', 1, 1667217, '', 1, 1);"},
- GetRequest: func() *http.Request {
- body := explorer.ItemService{
- Items: []uint{5},
- }
- bodyStr, _ := json.Marshal(body)
- req, _ := http.NewRequest(
- "DELETE",
- "/api/v3/object",
- bytes.NewReader(bodyStr),
- )
- return req
- },
- RollBack: []string{"DELETE FROM `v3_files` WHERE `id`=5"},
- ExpectCode: 203,
- },
- }
-
- for key, testCase := range testCases {
- for _, value := range testCase.Mock {
- model.DB.Exec(value)
- }
- req := testCase.GetRequest()
- router.ServeHTTP(w, req)
- asserts.Equal(200, w.Code)
- resJSON := &serializer.Response{}
- err := json.Unmarshal(w.Body.Bytes(), resJSON)
- asserts.NoError(err, "测试用例%d", key)
- asserts.Equal(testCase.ExpectCode, resJSON.Code, "测试用例%d", key)
-
- for _, value := range testCase.RollBack {
- model.DB.Exec(value)
- }
-
- w.Body.Reset()
- }
-}
diff --git a/routers/router.go b/routers/router.go
index 21f42ae8..962179f9 100644
--- a/routers/router.go
+++ b/routers/router.go
@@ -244,19 +244,22 @@ func InitMasterRouter() *gin.Engine {
)
// 七牛策略上传回调
callback.POST(
- "qiniu/:key",
+ "qiniu/:sessionID",
+ middleware.UseUploadSession("qiniu"),
middleware.QiniuCallbackAuth(),
controllers.QiniuCallback,
)
// 阿里云OSS策略上传回调
callback.POST(
- "oss/:key",
+ "oss/:sessionID",
+ middleware.UseUploadSession("oss"),
middleware.OSSCallbackAuth(),
controllers.OSSCallback,
)
// 又拍云策略上传回调
callback.POST(
- "upyun/:key",
+ "upyun/:sessionID",
+ middleware.UseUploadSession("upyun"),
middleware.UpyunCallbackAuth(),
controllers.UpyunCallback,
)
@@ -264,11 +267,12 @@ func InitMasterRouter() *gin.Engine {
{
// 文件上传完成
onedrive.POST(
- "finish/:key",
+ "finish/:sessionID",
+ middleware.UseUploadSession("onedrive"),
middleware.OneDriveCallbackAuth(),
controllers.OneDriveCallback,
)
- // 文件上传完成
+ // OAuth 完成
onedrive.GET(
"auth",
controllers.OneDriveOAuth,
@@ -276,14 +280,14 @@ func InitMasterRouter() *gin.Engine {
}
// 腾讯云COS策略上传回调
callback.GET(
- "cos/:key",
- middleware.COSCallbackAuth(),
+ "cos/:sessionID",
+ middleware.UseUploadSession("cos"),
controllers.COSCallback,
)
// AWS S3策略上传回调
callback.GET(
- "s3/:key",
- middleware.S3CallbackAuth(),
+ "s3/:sessionID",
+ middleware.UseUploadSession("s3"),
controllers.S3Callback,
)
}
diff --git a/routers/router_test.go b/routers/router_test.go
index 4c65d4e4..2476de6a 100644
--- a/routers/router_test.go
+++ b/routers/router_test.go
@@ -1,6 +1,7 @@
package routers
import (
+ "github.com/cloudreve/Cloudreve/v3/pkg/conf"
"net/http"
"net/http/httptest"
"testing"
@@ -19,7 +20,7 @@ func TestPing(t *testing.T) {
router.ServeHTTP(w, req)
assert.Equal(t, 200, w.Code)
- asserts.Contains(w.Body.String(), "Pong")
+ asserts.Contains(w.Body.String(), conf.BackendVersion)
}
func TestCaptcha(t *testing.T) {
diff --git a/service/admin/policy.go b/service/admin/policy.go
index 5c648ecd..a9151d53 100644
--- a/service/admin/policy.go
+++ b/service/admin/policy.go
@@ -149,9 +149,9 @@ func (service *PolicyService) AddCORS() serializer.Response {
switch policy.Type {
case "oss":
- handler := oss.Driver{
- Policy: &policy,
- HTTPClient: request.NewClient(),
+ handler, err := oss.NewDriver(&policy)
+ if err != nil {
+ return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err)
}
if err := handler.CORS(); err != nil {
return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err)
@@ -169,13 +169,16 @@ func (service *PolicyService) AddCORS() serializer.Response {
},
}),
}
+
if err := handler.CORS(); err != nil {
return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err)
}
case "s3":
- handler := s3.Driver{
- Policy: &policy,
+ handler, err := s3.NewDriver(&policy)
+ if err != nil {
+ return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err)
}
+
if err := handler.CORS(); err != nil {
return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err)
}
@@ -207,8 +210,12 @@ func (service *SlavePingService) Test() serializer.Response {
return serializer.ParamErr("从机无法向主机发送回调请求，请检查主机端 参数设置 - 站点信息 - 站点URL设置，并确保从机可以连接到此地址，"+err.Error(), nil)
}
- if res.Data.(string) != conf.BackendVersion {
- return serializer.ParamErr("Cloudreve版本不一致，主机："+res.Data.(string)+"，从机："+conf.BackendVersion, nil)
+ version := conf.BackendVersion
+ if conf.IsPro == "true" {
+ version += "-pro"
+ }
+ if res.Data.(string) != version {
+ return serializer.ParamErr("Cloudreve版本不一致，主机："+res.Data.(string)+"，从机："+version, nil)
}
return serializer.Response{}
diff --git a/service/callback/upload.go b/service/callback/upload.go
index df5d1807..fb94675a 100644
--- a/service/callback/upload.go
+++ b/service/callback/upload.go
@@ -50,7 +50,6 @@ type UpyunCallbackService struct {
// OneDriveCallback OneDrive 客户端回调正文
type OneDriveCallback struct {
- ID string `json:"id" binding:"required"`
Meta *onedrive.FileInfo
}
@@ -62,9 +61,6 @@ type COSCallback struct {
// S3Callback S3 客户端回调正文
type S3Callback struct {
- Bucket string `form:"bucket"`
- Etag string `form:"etag"`
- Key string `form:"key"`
}
// GetBody 返回回调正文
@@ -165,22 +161,21 @@ func (service *OneDriveCallback) PreProcess(c *gin.Context) serializer.Response
defer fs.Recycle()
// 获取回调会话
- callbackSessionRaw, _ := c.Get("callbackSession")
- callbackSession := callbackSessionRaw.(*serializer.UploadSession)
+ uploadSession := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
// 获取文件信息
- info, err := fs.Handler.(onedrive.Driver).Client.Meta(context.Background(), service.ID, "")
+ info, err := fs.Handler.(onedrive.Driver).Client.Meta(context.Background(), "", uploadSession.SavePath)
if err != nil {
return serializer.Err(serializer.CodeUploadFailed, "文件元信息查询失败", err)
}
// 验证与回调会话中是否一致
- actualPath := strings.TrimPrefix(callbackSession.SavePath, "/")
- isSizeCheckFailed := callbackSession.Size != info.Size
+ actualPath := strings.TrimPrefix(uploadSession.SavePath, "/")
+ isSizeCheckFailed := uploadSession.Size != info.Size
- // SharePoint 会对 Office 文档增加 meta data 导致文件大小不一致，这里增加 10 KB 宽容
+ // SharePoint 会对 Office 文档增加 meta data 导致文件大小不一致，这里增加 100 KB 宽容
// See: https://github.com/OneDrive/onedrive-api-docs/issues/935
- if strings.Contains(fs.Policy.OptionsSerialized.OdDriver, "sharepoint.com") && isSizeCheckFailed && (info.Size > callbackSession.Size) && (info.Size-callbackSession.Size <= 10240) {
+ if strings.Contains(fs.Policy.OptionsSerialized.OdDriver, "sharepoint.com") && isSizeCheckFailed && (info.Size > uploadSession.Size) && (info.Size-uploadSession.Size <= 102400) {
isSizeCheckFailed = false
}
@@ -202,17 +197,16 @@ func (service *COSCallback) PreProcess(c *gin.Context) serializer.Response {
defer fs.Recycle()
// 获取回调会话
- callbackSessionRaw, _ := c.Get("callbackSession")
- callbackSession := callbackSessionRaw.(*serializer.UploadSession)
+ uploadSession := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
// 获取文件信息
- info, err := fs.Handler.(cos.Driver).Meta(context.Background(), callbackSession.SavePath)
+ info, err := fs.Handler.(cos.Driver).Meta(context.Background(), uploadSession.SavePath)
if err != nil {
return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err)
}
// 验证实际文件信息与回调会话中是否一致
- if callbackSession.Size != info.Size || callbackSession.Key != info.CallbackKey {
+ if uploadSession.Size != info.Size || uploadSession.Key != info.CallbackKey {
return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err)
}
@@ -229,19 +223,39 @@ func (service *S3Callback) PreProcess(c *gin.Context) serializer.Response {
defer fs.Recycle()
// 获取回调会话
- callbackSessionRaw, _ := c.Get("callbackSession")
- callbackSession := callbackSessionRaw.(*serializer.UploadSession)
+ uploadSession := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
// 获取文件信息
- info, err := fs.Handler.(s3.Driver).Meta(context.Background(), callbackSession.SavePath)
+ info, err := fs.Handler.(*s3.Driver).Meta(context.Background(), uploadSession.SavePath)
if err != nil {
return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err)
}
// 验证实际文件信息与回调会话中是否一致
- if callbackSession.Size != info.Size || service.Etag != info.Etag {
+ if uploadSession.Size != info.Size {
return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err)
}
return ProcessCallback(service, c)
}
+
+// PreProcess 对OneDrive客户端回调进行预处理验证
+func (service *UploadCallbackService) PreProcess(c *gin.Context) serializer.Response {
+ // 创建文件系统
+ fs, err := filesystem.NewFileSystemFromCallback(c)
+ if err != nil {
+ return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
+ }
+ defer fs.Recycle()
+
+ // 获取回调会话
+ uploadSession := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
+
+ // 验证文件大小
+ if uploadSession.Size != service.Size {
+ fs.Handler.Delete(context.Background(), []string{uploadSession.SavePath})
+ return serializer.Err(serializer.CodeUploadFailed, "文件大小不一致", nil)
+ }
+
+ return ProcessCallback(service, c)
+}