feat(workflow): custom request headers for remote downloads (#3491)

- DownloadWorkflowService accepts headers[] ("Name: value" lines,
  CRLF rejected, max 32 entries)
- aria2 maps them to the header option; qBittorrent maps a Cookie:
  line onto its cookie field (the only credential its add API accepts)
- Persisted in task private state for resumability; only applied to
  plain HTTP(S) sources

Authored By: TDvorak <info@tdvorak.dev>

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
pull/3582/head
Tomas Dvorak 2 weeks ago
parent d165acae77
commit 791764a66f

@ -565,6 +565,8 @@
"remoteDownloadFileNameDescription": "Custom output file name", "remoteDownloadFileNameDescription": "Custom output file name",
"remoteDownloadHttpUser": "HTTP username (optional)", "remoteDownloadHttpUser": "HTTP username (optional)",
"remoteDownloadHttpPassword": "HTTP password (optional)", "remoteDownloadHttpPassword": "HTTP password (optional)",
"remoteDownloadHeaders": "Request headers (optional)",
"remoteDownloadHeadersDescription": "One header per line, e.g. Cookie: sid=...",
"processNode": "Target node", "processNode": "Target node",
"remoteDownloadNodeAuto": "Auto dispatch", "remoteDownloadNodeAuto": "Auto dispatch",
"createTask": "Create task", "createTask": "Create task",

@ -565,6 +565,8 @@
"remoteDownloadFileNameDescription": "自定义保存文件名", "remoteDownloadFileNameDescription": "自定义保存文件名",
"remoteDownloadHttpUser": "HTTP 用户名(可选)", "remoteDownloadHttpUser": "HTTP 用户名(可选)",
"remoteDownloadHttpPassword": "HTTP 密码(可选)", "remoteDownloadHttpPassword": "HTTP 密码(可选)",
"remoteDownloadHeaders": "请求头(可选)",
"remoteDownloadHeadersDescription": "每行一个请求头,例如 Cookie: sid=...",
"processNode": "处理节点", "processNode": "处理节点",
"remoteDownloadNodeAuto": "自动分配", "remoteDownloadNodeAuto": "自动分配",
"createTask": "创建任务", "createTask": "创建任务",

@ -104,6 +104,7 @@ export interface DownloadWorkflowService {
file_name?: string; file_name?: string;
username?: string; username?: string;
password?: string; password?: string;
headers?: string[];
} }
export interface ImportWorkflowService { export interface ImportWorkflowService {

@ -32,6 +32,7 @@ const CreateRemoteDownload = () => {
const [fileName, setFileName] = useState(""); const [fileName, setFileName] = useState("");
const [username, setUsername] = useState(""); const [username, setUsername] = useState("");
const [password, setPassword] = useState(""); const [password, setPassword] = useState("");
const [headers, setHeaders] = useState("");
const open = useAppSelector((state) => state.globalState.remoteDownloadDialogOpen); const open = useAppSelector((state) => state.globalState.remoteDownloadDialogOpen);
const target = useAppSelector((state) => state.globalState.remoteDownloadDialogFile); const target = useAppSelector((state) => state.globalState.remoteDownloadDialogFile);
@ -46,6 +47,7 @@ const CreateRemoteDownload = () => {
setFileName(""); setFileName("");
setUsername(""); setUsername("");
setPassword(""); setPassword("");
setHeaders("");
} }
}, [open]); }, [open]);
@ -67,6 +69,7 @@ const CreateRemoteDownload = () => {
file_name: fileName || undefined, file_name: fileName || undefined,
username: username || undefined, username: username || undefined,
password: password || undefined, password: password || undefined,
headers: headers ? headers.split("\n").filter((h) => h.trim()) : undefined,
}), }),
) )
.then(() => { .then(() => {
@ -80,7 +83,7 @@ const CreateRemoteDownload = () => {
.finally(() => { .finally(() => {
setLoading(false); setLoading(false);
}); });
}, [target, url, path, fileName, username, password]); }, [target, url, path, fileName, username, password, headers]);
return ( return (
<DraggableDialog <DraggableDialog
@ -155,6 +158,21 @@ const CreateRemoteDownload = () => {
/> />
</Stack> </Stack>
)} )}
{!target && (
<Stack spacing={3} direction={isMobile ? "column" : "row"}>
<OutlineIconTextField
icon={<Link />}
variant="outlined"
value={headers}
multiline
minRows={2}
onChange={(e) => setHeaders(e.target.value)}
placeholder={t("modals.remoteDownloadHeadersDescription")}
label={t("application:modals.remoteDownloadHeaders")}
fullWidth
/>
</Stack>
)}
</Stack> </Stack>
</DialogContent> </DialogContent>
</DraggableDialog> </DraggableDialog>

@ -50,6 +50,7 @@ type (
FileName string `json:"file_name,omitempty"` FileName string `json:"file_name,omitempty"`
HTTPUsername string `json:"http_username,omitempty"` HTTPUsername string `json:"http_username,omitempty"`
HTTPPassword string `json:"http_password,omitempty"` HTTPPassword string `json:"http_password,omitempty"`
HTTPHeaders []string `json:"http_headers,omitempty"`
Handle *downloader.TaskHandle `json:"handle,omitempty"` Handle *downloader.TaskHandle `json:"handle,omitempty"`
Status *downloader.TaskStatus `json:"status,omitempty"` Status *downloader.TaskStatus `json:"status,omitempty"`
NodeState `json:",inline"` NodeState `json:",inline"`
@ -93,6 +94,7 @@ type RemoteDownloadTaskOption struct {
FileName string FileName string
HTTPUsername string HTTPUsername string
HTTPPassword string HTTPPassword string
HTTPHeaders []string
} }
// NewRemoteDownloadTask creates a new RemoteDownloadTask // NewRemoteDownloadTask creates a new RemoteDownloadTask
@ -107,6 +109,7 @@ func NewRemoteDownloadTask(ctx context.Context, src string, srcFile, dst string,
state.FileName = sanitizeFileName(opts.FileName) state.FileName = sanitizeFileName(opts.FileName)
state.HTTPUsername = opts.HTTPUsername state.HTTPUsername = opts.HTTPUsername
state.HTTPPassword = opts.HTTPPassword state.HTTPPassword = opts.HTTPPassword
state.HTTPHeaders = opts.HTTPHeaders
} }
stateBytes, err := json.Marshal(state) stateBytes, err := json.Marshal(state)
if err != nil { if err != nil {
@ -251,7 +254,7 @@ func (m *RemoteDownloadTask) createDownloadTask(ctx context.Context, dep depende
// credentials only apply to plain HTTP(S) source URLs on aria2; qBittorrent // credentials only apply to plain HTTP(S) source URLs on aria2; qBittorrent
// accepts a torrent rename and carries HTTP auth in the URL userinfo. // accepts a torrent rename and carries HTTP auth in the URL userinfo.
func (m *RemoteDownloadTask) buildDownloadOptions(ctx context.Context, base map[string]interface{}, srcUrl string) (map[string]interface{}, string) { func (m *RemoteDownloadTask) buildDownloadOptions(ctx context.Context, base map[string]interface{}, srcUrl string) (map[string]interface{}, string) {
if m.state.FileName == "" && m.state.HTTPUsername == "" { if m.state.FileName == "" && m.state.HTTPUsername == "" && len(m.state.HTTPHeaders) == 0 {
return base, srcUrl return base, srcUrl
} }
@ -266,20 +269,34 @@ func (m *RemoteDownloadTask) buildDownloadOptions(ctx context.Context, base map[
if m.state.FileName != "" { if m.state.FileName != "" {
options["rename"] = m.state.FileName options["rename"] = m.state.FileName
} }
if m.state.HTTPUsername != "" && isHttpSrc { if isHttpSrc {
if m.state.HTTPUsername != "" {
if u, err := url.Parse(srcUrl); err == nil { if u, err := url.Parse(srcUrl); err == nil {
u.User = url.UserPassword(m.state.HTTPUsername, m.state.HTTPPassword) u.User = url.UserPassword(m.state.HTTPUsername, m.state.HTTPPassword)
srcUrl = u.String() srcUrl = u.String()
} }
} }
// qBittorrent's add API only accepts a cookie field, not arbitrary
// headers — pass through any Cookie: line the user supplied.
for _, h := range m.state.HTTPHeaders {
if k, v, ok := strings.Cut(h, ":"); ok && strings.EqualFold(strings.TrimSpace(k), "cookie") {
options["cookie"] = strings.TrimSpace(v)
}
}
}
default: default:
if m.state.FileName != "" && isHttpSrc { if isHttpSrc {
if m.state.FileName != "" {
options["out"] = m.state.FileName options["out"] = m.state.FileName
} }
if m.state.HTTPUsername != "" && isHttpSrc { if m.state.HTTPUsername != "" {
options["http-user"] = m.state.HTTPUsername options["http-user"] = m.state.HTTPUsername
options["http-passwd"] = m.state.HTTPPassword options["http-passwd"] = m.state.HTTPPassword
} }
if len(m.state.HTTPHeaders) > 0 {
options["header"] = m.state.HTTPHeaders
}
}
} }
return options, srcUrl return options, srcUrl
} }

@ -107,6 +107,28 @@ func TestBuildDownloadOptionsNoExtras(t *testing.T) {
a.Equal("https://example.com/file.zip", taskUrl) a.Equal("https://example.com/file.zip", taskUrl)
} }
func TestBuildDownloadOptionsHeaders(t *testing.T) {
a := assert.New(t)
// aria2: arbitrary headers pass through as a list.
m := newRemoteDownloadTaskForOptions(&RemoteDownloadTaskState{
SrcUri: "https://example.com/f.zip",
HTTPHeaders: []string{"Cookie: sid=abc", "Referer: https://example.com/"},
}, types.DownloaderProviderAria2)
opts, _ := m.buildDownloadOptions(context.Background(), nil, "https://example.com/f.zip")
a.Equal([]string{"Cookie: sid=abc", "Referer: https://example.com/"}, opts["header"])
// qBittorrent: only the Cookie line maps onto the cookie field.
m = newRemoteDownloadTaskForOptions(&RemoteDownloadTaskState{
SrcUri: "https://example.com/f.torrent",
HTTPHeaders: []string{"Referer: https://x/", "Cookie: sid=abc"},
}, types.DownloaderProviderQBittorrent)
opts, _ = m.buildDownloadOptions(context.Background(), nil, "https://example.com/f.torrent")
a.Equal("sid=abc", opts["cookie"])
_, hasHeader := opts["header"]
a.False(hasHeader)
}
func TestNewRemoteDownloadTaskSanitizesFileName(t *testing.T) { func TestNewRemoteDownloadTaskSanitizesFileName(t *testing.T) {
a := assert.New(t) a := assert.New(t)
tsk, err := NewRemoteDownloadTask(context.Background(), "https://example.com/f", "", "cloudreve://my/dst", &RemoteDownloadTaskOption{ tsk, err := NewRemoteDownloadTask(context.Background(), "https://example.com/f", "", "cloudreve://my/dst", &RemoteDownloadTaskOption{

@ -5,6 +5,7 @@ import (
"encoding/gob" "encoding/gob"
"fmt" "fmt"
"github.com/cloudreve/Cloudreve/v4/pkg/hashid" "github.com/cloudreve/Cloudreve/v4/pkg/hashid"
"strings"
"time" "time"
"github.com/cloudreve/Cloudreve/v4/application/dependency" "github.com/cloudreve/Cloudreve/v4/application/dependency"
@ -79,6 +80,7 @@ type (
FileName string `json:"file_name" binding:"omitempty,max=255"` FileName string `json:"file_name" binding:"omitempty,max=255"`
Username string `json:"username" binding:"omitempty,max=255"` Username string `json:"username" binding:"omitempty,max=255"`
Password string `json:"password" binding:"omitempty,max=255"` Password string `json:"password" binding:"omitempty,max=255"`
Headers []string `json:"headers" binding:"omitempty,max=32,dive,max=2048"`
} }
CreateDownloadParamCtx struct{} CreateDownloadParamCtx struct{}
) )
@ -134,11 +136,20 @@ func (service *DownloadWorkflowService) CreateDownloadTask(c *gin.Context) ([]*T
} }
} }
// Validate custom request headers: "Name: value" lines, no CRLF injection.
for _, h := range service.Headers {
name, _, ok := strings.Cut(h, ":")
if !ok || strings.TrimSpace(name) == "" || strings.ContainsAny(h, "\r\n") {
return nil, serializer.NewError(serializer.CodeParamErr, "Invalid header", nil)
}
}
// Custom file name only applies to single-source tasks; HTTP credentials // Custom file name only applies to single-source tasks; HTTP credentials
// only apply to plain HTTP(S) source URLs. // and headers only apply to plain HTTP(S) source URLs.
taskOpts := &workflows.RemoteDownloadTaskOption{ taskOpts := &workflows.RemoteDownloadTaskOption{
HTTPUsername: service.Username, HTTPUsername: service.Username,
HTTPPassword: service.Password, HTTPPassword: service.Password,
HTTPHeaders: service.Headers,
} }
if len(service.Src) <= 1 { if len(service.Src) <= 1 {
taskOpts.FileName = service.FileName taskOpts.FileName = service.FileName
@ -146,6 +157,7 @@ func (service *DownloadWorkflowService) CreateDownloadTask(c *gin.Context) ([]*T
if service.SrcFile != "" { if service.SrcFile != "" {
taskOpts.HTTPUsername = "" taskOpts.HTTPUsername = ""
taskOpts.HTTPPassword = "" taskOpts.HTTPPassword = ""
taskOpts.HTTPHeaders = nil
} }
// batch creating tasks // batch creating tasks

Loading…
Cancel
Save