feat(workflow): HTTP auth and custom file name for remote downloads (#3072)

- DownloadWorkflowService accepts optional file_name/username/password
- Per-task options persisted in RemoteDownloadTaskState for resumability
- aria2: mapped to out/http-user/http-passwd; qBittorrent: rename plus
  URL userinfo credentials
- File name sanitized against path separators; creds only applied to
  plain HTTP(S) sources
- New fields in the remote download dialog (en-US, zh-CN)

Authored By: TDvorak <info@tdvorak.dev>

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
pull/3582/head
Tomas Dvorak 2 weeks ago
parent 24c751961e
commit d165acae77

@ -561,6 +561,10 @@
"remoteDownloadURL": "Download target URL", "remoteDownloadURL": "Download target URL",
"remoteDownloadURLDescription": "Paste the download URL, one URL per line", "remoteDownloadURLDescription": "Paste the download URL, one URL per line",
"remoteDownloadDst": "Download to", "remoteDownloadDst": "Download to",
"remoteDownloadFileName": "File name (optional)",
"remoteDownloadFileNameDescription": "Custom output file name",
"remoteDownloadHttpUser": "HTTP username (optional)",
"remoteDownloadHttpPassword": "HTTP password (optional)",
"processNode": "Target node", "processNode": "Target node",
"remoteDownloadNodeAuto": "Auto dispatch", "remoteDownloadNodeAuto": "Auto dispatch",
"createTask": "Create task", "createTask": "Create task",

@ -561,6 +561,10 @@
"remoteDownloadURL": "下载链接", "remoteDownloadURL": "下载链接",
"remoteDownloadURLDescription": "输入文件下载地址,一行一个", "remoteDownloadURLDescription": "输入文件下载地址,一行一个",
"remoteDownloadDst": "下载至", "remoteDownloadDst": "下载至",
"remoteDownloadFileName": "文件名(可选)",
"remoteDownloadFileNameDescription": "自定义保存文件名",
"remoteDownloadHttpUser": "HTTP 用户名(可选)",
"remoteDownloadHttpPassword": "HTTP 密码(可选)",
"processNode": "处理节点", "processNode": "处理节点",
"remoteDownloadNodeAuto": "自动分配", "remoteDownloadNodeAuto": "自动分配",
"createTask": "创建任务", "createTask": "创建任务",

@ -101,6 +101,9 @@ export interface DownloadWorkflowService {
src?: string[]; src?: string[];
src_file?: string; src_file?: string;
dst: string; dst: string;
file_name?: string;
username?: string;
password?: string;
} }
export interface ImportWorkflowService { export interface ImportWorkflowService {

@ -13,7 +13,10 @@ import { OutlineIconTextField } from "../../Common/Form/OutlineIconTextField.tsx
import { PathSelectorForm } from "../../Common/Form/PathSelectorForm.tsx"; import { PathSelectorForm } from "../../Common/Form/PathSelectorForm.tsx";
import { ViewTaskAction } from "../../Common/Snackbar/snackbar.tsx"; import { ViewTaskAction } from "../../Common/Snackbar/snackbar.tsx";
import DraggableDialog from "../../Dialogs/DraggableDialog.tsx"; import DraggableDialog from "../../Dialogs/DraggableDialog.tsx";
import Edit from "../../Icons/Edit.tsx";
import Link from "../../Icons/Link.tsx"; import Link from "../../Icons/Link.tsx";
import LockClosedKey from "../../Icons/LockClosedKey.tsx";
import PersonOutlined from "../../Icons/PersonOutlined.tsx";
import { FileManagerIndex } from "../FileManager.tsx"; import { FileManagerIndex } from "../FileManager.tsx";
const CreateRemoteDownload = () => { const CreateRemoteDownload = () => {
@ -26,6 +29,9 @@ const CreateRemoteDownload = () => {
const [loading, setLoading] = useState(false); const [loading, setLoading] = useState(false);
const [path, setPath] = useState(""); const [path, setPath] = useState("");
const [url, setUrl] = useState(""); const [url, setUrl] = useState("");
const [fileName, setFileName] = useState("");
const [username, setUsername] = useState("");
const [password, setPassword] = useState("");
const open = useAppSelector((state) => state.globalState.remoteDownloadDialogOpen); const open = useAppSelector((state) => state.globalState.remoteDownloadDialogOpen);
const target = useAppSelector((state) => state.globalState.remoteDownloadDialogFile); const target = useAppSelector((state) => state.globalState.remoteDownloadDialogFile);
@ -37,6 +43,9 @@ const CreateRemoteDownload = () => {
const fs = initialPath.fs(); const fs = initialPath.fs();
setPath(fs == Filesystem.shared_with_me || fs == Filesystem.trash ? defaultPath : initialPath.toString()); setPath(fs == Filesystem.shared_with_me || fs == Filesystem.trash ? defaultPath : initialPath.toString());
setUrl(""); setUrl("");
setFileName("");
setUsername("");
setPassword("");
} }
}, [open]); }, [open]);
@ -55,6 +64,9 @@ const CreateRemoteDownload = () => {
src_file: target ? getFileLinkedUri(target) : undefined, src_file: target ? getFileLinkedUri(target) : undefined,
dst: path, dst: path,
src: url ? url.split("\n") : undefined, src: url ? url.split("\n") : undefined,
file_name: fileName || undefined,
username: username || undefined,
password: password || undefined,
}), }),
) )
.then(() => { .then(() => {
@ -68,7 +80,7 @@ const CreateRemoteDownload = () => {
.finally(() => { .finally(() => {
setLoading(false); setLoading(false);
}); });
}, [target, url, path]); }, [target, url, path, fileName, username, password]);
return ( return (
<DraggableDialog <DraggableDialog
@ -111,6 +123,38 @@ const CreateRemoteDownload = () => {
label={t("modals.remoteDownloadDst")} label={t("modals.remoteDownloadDst")}
/> />
</Stack> </Stack>
<Stack spacing={3} direction={isMobile ? "column" : "row"}>
<OutlineIconTextField
icon={<Edit />}
variant="outlined"
value={fileName}
onChange={(e) => setFileName(e.target.value)}
label={t("application:modals.remoteDownloadFileName")}
placeholder={t("modals.remoteDownloadFileNameDescription")}
fullWidth
/>
</Stack>
{!target && (
<Stack spacing={3} direction={isMobile ? "column" : "row"}>
<OutlineIconTextField
icon={<PersonOutlined />}
variant="outlined"
value={username}
onChange={(e) => setUsername(e.target.value)}
label={t("application:modals.remoteDownloadHttpUser")}
fullWidth
/>
<OutlineIconTextField
icon={<LockClosedKey />}
variant="outlined"
type="password"
value={password}
onChange={(e) => setPassword(e.target.value)}
label={t("application:modals.remoteDownloadHttpPassword")}
fullWidth
/>
</Stack>
)}
</Stack> </Stack>
</DialogContent> </DialogContent>
</DraggableDialog> </DraggableDialog>

@ -5,6 +5,8 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"maps"
"net/url"
"os" "os"
"path" "path"
"path/filepath" "path/filepath"
@ -45,6 +47,9 @@ type (
SrcFileUri string `json:"src_file_uri,omitempty"` SrcFileUri string `json:"src_file_uri,omitempty"`
SrcUri string `json:"src_uri,omitempty"` SrcUri string `json:"src_uri,omitempty"`
Dst string `json:"dst,omitempty"` Dst string `json:"dst,omitempty"`
FileName string `json:"file_name,omitempty"`
HTTPUsername string `json:"http_username,omitempty"`
HTTPPassword string `json:"http_password,omitempty"`
Handle *downloader.TaskHandle `json:"handle,omitempty"` Handle *downloader.TaskHandle `json:"handle,omitempty"`
Status *downloader.TaskStatus `json:"status,omitempty"` Status *downloader.TaskStatus `json:"status,omitempty"`
NodeState `json:",inline"` NodeState `json:",inline"`
@ -81,14 +86,28 @@ func init() {
queue.RegisterResumableTaskFactory(queue.RemoteDownloadTaskType, NewRemoteDownloadTaskFromModel) queue.RegisterResumableTaskFactory(queue.RemoteDownloadTaskType, NewRemoteDownloadTaskFromModel)
} }
// RemoteDownloadTaskOption carries optional per-task parameters supplied by
// the user: a custom output file name and HTTP basic-auth credentials for
// plain HTTP(S) sources.
type RemoteDownloadTaskOption struct {
FileName string
HTTPUsername string
HTTPPassword string
}
// NewRemoteDownloadTask creates a new RemoteDownloadTask // NewRemoteDownloadTask creates a new RemoteDownloadTask
func NewRemoteDownloadTask(ctx context.Context, src string, srcFile, dst string) (queue.Task, error) { func NewRemoteDownloadTask(ctx context.Context, src string, srcFile, dst string, opts *RemoteDownloadTaskOption) (queue.Task, error) {
state := &RemoteDownloadTaskState{ state := &RemoteDownloadTaskState{
SrcUri: src, SrcUri: src,
SrcFileUri: srcFile, SrcFileUri: srcFile,
Dst: dst, Dst: dst,
NodeState: NodeState{}, NodeState: NodeState{},
} }
if opts != nil {
state.FileName = sanitizeFileName(opts.FileName)
state.HTTPUsername = opts.HTTPUsername
state.HTTPPassword = opts.HTTPPassword
}
stateBytes, err := json.Marshal(state) stateBytes, err := json.Marshal(state)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to marshal state: %w", err) return nil, fmt.Errorf("failed to marshal state: %w", err)
@ -214,8 +233,10 @@ func (m *RemoteDownloadTask) createDownloadTask(ctx context.Context, dep depende
torrentUrl = torrentUrls[0].Url torrentUrl = torrentUrls[0].Url
} }
options, taskUrl := m.buildDownloadOptions(ctx, user.Edges.Group.Settings.RemoteDownloadOptions, torrentUrl)
// Create download task // Create download task
handle, err := m.d.CreateTask(ctx, torrentUrl, user.Edges.Group.Settings.RemoteDownloadOptions) handle, err := m.d.CreateTask(ctx, taskUrl, options)
if err != nil { if err != nil {
return task.StatusError, fmt.Errorf("failed to create download task: %w", err) return task.StatusError, fmt.Errorf("failed to create download task: %w", err)
} }
@ -225,6 +246,44 @@ func (m *RemoteDownloadTask) createDownloadTask(ctx context.Context, dep depende
return task.StatusSuspending, nil return task.StatusSuspending, nil
} }
// buildDownloadOptions overlays per-task options (custom file name, HTTP
// credentials) onto the group's remote-download options. Custom name and
// credentials only apply to plain HTTP(S) source URLs on aria2; qBittorrent
// accepts a torrent rename and carries HTTP auth in the URL userinfo.
func (m *RemoteDownloadTask) buildDownloadOptions(ctx context.Context, base map[string]interface{}, srcUrl string) (map[string]interface{}, string) {
if m.state.FileName == "" && m.state.HTTPUsername == "" {
return base, srcUrl
}
options := maps.Clone(base)
if options == nil {
options = map[string]interface{}{}
}
isHttpSrc := m.state.SrcFileUri == "" && (strings.HasPrefix(m.state.SrcUri, "http://") || strings.HasPrefix(m.state.SrcUri, "https://"))
switch m.node.Settings(ctx).Provider {
case types.DownloaderProviderQBittorrent:
if m.state.FileName != "" {
options["rename"] = m.state.FileName
}
if m.state.HTTPUsername != "" && isHttpSrc {
if u, err := url.Parse(srcUrl); err == nil {
u.User = url.UserPassword(m.state.HTTPUsername, m.state.HTTPPassword)
srcUrl = u.String()
}
}
default:
if m.state.FileName != "" && isHttpSrc {
options["out"] = m.state.FileName
}
if m.state.HTTPUsername != "" && isHttpSrc {
options["http-user"] = m.state.HTTPUsername
options["http-passwd"] = m.state.HTTPPassword
}
}
return options, srcUrl
}
// buildSSRFOptions composes the SSRF policy for a download: the assigned // buildSSRFOptions composes the SSRF policy for a download: the assigned
// node's URLValidation settings, plus the operator-configured site URL hosts // node's URLValidation settings, plus the operator-configured site URL hosts
// (always allowlisted so users can fetch files served by Cloudreve itself). // (always allowlisted so users can fetch files served by Cloudreve itself).
@ -679,6 +738,6 @@ func (m *RemoteDownloadTask) Progress(ctx context.Context) queue.Progresses {
} }
func sanitizeFileName(name string) string { func sanitizeFileName(name string) string {
r := strings.NewReplacer("\\", "_", ":", "_", "*", "_", "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_") r := strings.NewReplacer("\\", "_", "/", "_", ":", "_", "*", "_", "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_")
return r.Replace(name) return r.Replace(name)
} }

@ -0,0 +1,120 @@
package workflows
import (
"context"
"encoding/json"
"testing"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/cluster"
"github.com/stretchr/testify/assert"
)
type stubNode struct {
cluster.Node
settings *types.NodeSetting
}
func (s *stubNode) Settings(ctx context.Context) *types.NodeSetting {
return s.settings
}
func newRemoteDownloadTaskForOptions(state *RemoteDownloadTaskState, provider types.DownloaderProvider) *RemoteDownloadTask {
return &RemoteDownloadTask{
state: state,
node: &stubNode{settings: &types.NodeSetting{Provider: provider}},
}
}
func TestBuildDownloadOptionsAria2(t *testing.T) {
a := assert.New(t)
m := newRemoteDownloadTaskForOptions(&RemoteDownloadTaskState{
SrcUri: "https://example.com/file.zip",
FileName: "renamed.zip",
HTTPUsername: "user",
HTTPPassword: "pass",
}, types.DownloaderProviderAria2)
base := map[string]interface{}{"max-connection-per-server": "4"}
opts, taskUrl := m.buildDownloadOptions(context.Background(), base, "https://example.com/file.zip")
a.Equal("renamed.zip", opts["out"])
a.Equal("user", opts["http-user"])
a.Equal("pass", opts["http-passwd"])
a.Equal("4", opts["max-connection-per-server"])
a.Equal("https://example.com/file.zip", taskUrl)
_, exists := base["out"]
a.False(exists, "base options must not be mutated")
}
func TestBuildDownloadOptionsQBittorrent(t *testing.T) {
a := assert.New(t)
m := newRemoteDownloadTaskForOptions(&RemoteDownloadTaskState{
SrcUri: "https://example.com/file.torrent",
FileName: "renamed",
HTTPUsername: "user",
HTTPPassword: "p@ss:word",
}, types.DownloaderProviderQBittorrent)
opts, taskUrl := m.buildDownloadOptions(context.Background(), nil, "https://example.com/file.torrent")
a.Equal("renamed", opts["rename"])
_, exists := opts["out"]
a.False(exists)
a.Equal("https://user:p%40ss%3Aword@example.com/file.torrent", taskUrl)
}
func TestBuildDownloadOptionsNonHttpSrc(t *testing.T) {
a := assert.New(t)
// Torrent source file: aria2 "out" and HTTP credentials must not apply.
m := newRemoteDownloadTaskForOptions(&RemoteDownloadTaskState{
SrcFileUri: "cloudreve://my/file.torrent",
FileName: "renamed",
HTTPUsername: "user",
HTTPPassword: "pass",
}, types.DownloaderProviderAria2)
opts, taskUrl := m.buildDownloadOptions(context.Background(), nil, "https://slave.example.com/entity/1")
_, hasOut := opts["out"]
a.False(hasOut)
_, hasUser := opts["http-user"]
a.False(hasUser)
a.Equal("https://slave.example.com/entity/1", taskUrl)
// Magnet link: same for aria2.
m = newRemoteDownloadTaskForOptions(&RemoteDownloadTaskState{
SrcUri: "magnet:?xt=urn:btih:abc",
FileName: "renamed",
HTTPUsername: "user",
}, types.DownloaderProviderAria2)
opts, _ = m.buildDownloadOptions(context.Background(), nil, "magnet:?xt=urn:btih:abc")
_, hasOut = opts["out"]
a.False(hasOut)
_, hasUser = opts["http-user"]
a.False(hasUser)
}
func TestBuildDownloadOptionsNoExtras(t *testing.T) {
a := assert.New(t)
m := newRemoteDownloadTaskForOptions(&RemoteDownloadTaskState{
SrcUri: "https://example.com/file.zip",
}, types.DownloaderProviderAria2)
base := map[string]interface{}{"k": "v"}
opts, taskUrl := m.buildDownloadOptions(context.Background(), base, "https://example.com/file.zip")
a.Equal("v", opts["k"])
a.Equal("https://example.com/file.zip", taskUrl)
}
func TestNewRemoteDownloadTaskSanitizesFileName(t *testing.T) {
a := assert.New(t)
tsk, err := NewRemoteDownloadTask(context.Background(), "https://example.com/f", "", "cloudreve://my/dst", &RemoteDownloadTaskOption{
FileName: "../../etc/passwd",
})
a.NoError(err)
state := &RemoteDownloadTaskState{}
a.NoError(json.Unmarshal([]byte(tsk.(*RemoteDownloadTask).Task.PrivateState), state))
a.Equal(".._.._etc_passwd", state.FileName)
}

@ -73,9 +73,12 @@ func init() {
type ( type (
DownloadWorkflowService struct { DownloadWorkflowService struct {
Src []string `json:"src"` Src []string `json:"src"`
SrcFile string `json:"src_file"` SrcFile string `json:"src_file"`
Dst string `json:"dst" binding:"required"` Dst string `json:"dst" binding:"required"`
FileName string `json:"file_name" binding:"omitempty,max=255"`
Username string `json:"username" binding:"omitempty,max=255"`
Password string `json:"password" binding:"omitempty,max=255"`
} }
CreateDownloadParamCtx struct{} CreateDownloadParamCtx struct{}
) )
@ -131,6 +134,20 @@ func (service *DownloadWorkflowService) CreateDownloadTask(c *gin.Context) ([]*T
} }
} }
// Custom file name only applies to single-source tasks; HTTP credentials
// only apply to plain HTTP(S) source URLs.
taskOpts := &workflows.RemoteDownloadTaskOption{
HTTPUsername: service.Username,
HTTPPassword: service.Password,
}
if len(service.Src) <= 1 {
taskOpts.FileName = service.FileName
}
if service.SrcFile != "" {
taskOpts.HTTPUsername = ""
taskOpts.HTTPPassword = ""
}
// batch creating tasks // batch creating tasks
ae := serializer.NewAggregateError() ae := serializer.NewAggregateError()
tasks := make([]queue.Task, 0, len(service.Src)) tasks := make([]queue.Task, 0, len(service.Src))
@ -139,7 +156,7 @@ func (service *DownloadWorkflowService) CreateDownloadTask(c *gin.Context) ([]*T
continue continue
} }
t, err := workflows.NewRemoteDownloadTask(c, src, service.SrcFile, service.Dst) t, err := workflows.NewRemoteDownloadTask(c, src, service.SrcFile, service.Dst, taskOpts)
if err != nil { if err != nil {
ae.Add(src, err) ae.Add(src, err)
continue continue
@ -153,7 +170,7 @@ func (service *DownloadWorkflowService) CreateDownloadTask(c *gin.Context) ([]*T
} }
if service.SrcFile != "" { if service.SrcFile != "" {
t, err := workflows.NewRemoteDownloadTask(c, "", service.SrcFile, service.Dst) t, err := workflows.NewRemoteDownloadTask(c, "", service.SrcFile, service.Dst, taskOpts)
if err != nil { if err != nil {
ae.Add(service.SrcFile, err) ae.Add(service.SrcFile, err)
} }

Loading…
Cancel
Save