Merge pull request #196 from Dvorinka/feat/download-route-shuffle

feat: shuffle download URLs across site URL and CDN routes (#173)

ROADMAP.md

@@ -207,6 +207,7 @@ Order = user-visible value first; each ships with backend + UI + tests.
 - #3454 (PG FK on upload) is **Pro-only** — `audit_logs` doesn't exist in this codebase. When B.5 adds our own audit log: insert the audit row in the same tx *after* the file row, never before.
 - [x] `desloppify` pass — 73 review items dispositioned (46 fixed, 27 honestly skipped), strict score 77.1 (was 18.9); scorecard lives in README. `security-reviewer` pass done incrementally per batch (OAuth secrets, SSRF, process exec, path safety)
 - [x] Tag management page (upstream #2962) — owner-scoped `tag:` metadata stats/rename/recolor/delete in `inventory.FileClient`, `GET/PATCH/DELETE /file/tag` routes, Settings → Tags tab with merge-on-rename semantics
+- [x] Download URL shuffling (#173) — `download_cdn_shuffle` distributes generated download URLs randomly across the site URL + `download_cdn_routes` endpoints (`setting.DownloadURLBase`, honors `UseFirstSiteUrl`); covers entity downloads, archive sessions, and redirect-type direct links; manual route picker hidden client-side while active
 - [x] Private space / vault (upstream #3447) — opt-in root folder flagged `sys:vault`; ancestry-based membership (zero flag maintenance; chain-less search results resolved lazily via `file_children`); `vaultNavigator` decorator gating `To`/`Children`/`Walk`/`ExecuteHook`; separate vault password (`salt:sha256`, sensitive) + 30-min cache-backed unlock session, unlock rate-limited 10/h; vault content never shareable and never direct-linkable; search filtered while locked; `vault_enabled`/`vault_unlocked` in user settings; unlock prompt in `ExplorerError`, Private space section in security settings, lock badge on vault folder (#195)
 - [x] Share `hide_readme` option (upstream #2729 item 6) — `ShareProps.HideReadMe` (only meaningful with `ShowReadMe`); share navigator filters `README.md`/`README.txt` (case-insensitive) from listings while direct-path resolution stays open for the readme viewer; `detectReadMe` URI fallback now probes unconditionally; owner-only `hide_readme` on share responses; Share dialog nested checkbox
 - [x] 2FA recovery codes (upstream #2729 item 3) — `user.two_factor_backup_codes` sensitive JSON of `salt:sha256` digests; `PUT /user/setting/2fa/backup` regenerates 10 one-time codes behind a valid TOTP (rate-limited 5/h); `Verify2FA` falls back to single-use code consumption on TOTP failure; codes invalidated on secret rotation/disable; login phase gains a recovery-code input mode; security settings show remaining count + regenerate dialog

frontend/public/locales/en-US/dashboard.json

@@ -854,6 +854,8 @@
     "ssoRegisterEnabledDes": "Automatically create a local account when a user signs in via SSO for the first time. The sign-up email filter below also applies.",
     "downloadCdnRoutes": "Download CDN routes",
     "downloadCdnRoutesDes": "Alternative download endpoints offered to users, one per line in name=url format (e.g. Line 1=https://cdn1.example.com). Routes must proxy the full request path and query back to this site so signed URLs stay valid; CORS headers are required for in-browser downloads.",
+    "downloadCdnShuffle": "Shuffle download routes",
+    "downloadCdnShuffleDes": "Randomly distribute each generated download link across the site URL and all configured CDN routes, so batch exports to download managers hit different endpoints. The manual route picker is hidden while enabled.",
     "ssoAutoRedirect": "Auto redirect to SSO",
     "ssoAutoRedirectDes": "Skip the login form and send visitors straight to the identity provider. Append <0>?nosso=1</0> to the login URL to reach the password form (e.g. for admin recovery).",
     "ssoCallbackUrl": "Callback URL",

frontend/public/locales/zh-CN/dashboard.json

@@ -854,6 +854,8 @@
     "ssoRegisterEnabledDes": "用户首次通过 SSO 登录时自动创建本地账号。下方的注册邮箱过滤规则同样适用。",
     "downloadCdnRoutes": "下载 CDN 线路",
     "downloadCdnRoutesDes": "供用户选择的备用下载端点，每行一条，格式为 名称=URL（例如 线路1=https://cdn1.example.com）。线路需将完整的请求路径与查询串代理回本站以保证签名有效；浏览器内下载需要线路配置 CORS 头。",
+    "downloadCdnShuffle": "随机分发下载线路",
+    "downloadCdnShuffleDes": "启用后，每条生成的下载链接将随机分发到本站地址或任一已配置的 CDN 线路，批量导出到下载器时不同链接会命中不同端点。启用期间将隐藏手动选择线路。",
     "ssoAutoRedirect": "自动跳转至 SSO",
     "ssoAutoRedirectDes": "跳过登录表单，直接跳转至身份提供商。在登录地址后附加 <0>?nosso=1</0> 可进入密码登录表单（例如管理员账户恢复）。",
     "ssoCallbackUrl": "回调地址",

frontend/src/api/site.ts

@@ -34,6 +34,7 @@ export interface SiteConfig {
   sso_auto_redirect?: boolean;
   qq_connect_enabled?: boolean;
   download_cdn_routes?: { name: string; url: string }[];
+  download_cdn_shuffle?: boolean;
   abuse_captcha?: boolean;
   upload_dedup?: boolean;
   allow_select_node?: boolean;

frontend/src/component/Admin/Settings/Settings.tsx

@@ -165,6 +165,7 @@ const Settings = () => {
                     "siteDes",
                     "siteURL",
                     "download_cdn_routes",
+                    "download_cdn_shuffle",
                     "siteScript",
                     "pwa_small_icon",
                     "pwa_medium_icon",

frontend/src/component/Admin/Settings/SiteInformation/SiteInformation.tsx

@@ -65,6 +65,24 @@ const SiteInformation = () => {
                 <NoMarginHelperText>{t("settings.downloadCdnRoutesDes")}</NoMarginHelperText>
               </FormControl>
             </SettingForm>
+            <SettingForm lgWidth={5}>
+              <FormControl fullWidth>
+                <FormControlLabel
+                  control={
+                    <Switch
+                      checked={isTrueVal(values.download_cdn_shuffle)}
+                      onChange={(e) =>
+                        setSettings({
+                          download_cdn_shuffle: e.target.checked ? "1" : "0",
+                        })
+                      }
+                    />
+                  }
+                  label={t("settings.downloadCdnShuffle")}
+                />
+                <NoMarginHelperText>{t("settings.downloadCdnShuffleDes")}</NoMarginHelperText>
+              </FormControl>
+            </SettingForm>
             <SettingForm title={t("settings.customFooterHTML")} lgWidth={5}>
               <FormControl fullWidth>
                 <DenseFilledTextField

frontend/src/redux/thunks/download.ts

@@ -137,7 +137,14 @@ export function backendBatchDownload(files: FileResponse[]): AppThunk {
 // signature stays valid through the CDN. Cancel falls back to direct.
 export function pickDownloadRoute(url: string): AppThunk<Promise<string>> {
   return async (dispatch, getState) => {
-    const routes = getState().siteConfig.basic.config.download_cdn_routes;
+    const config = getState().siteConfig.basic.config;
+    // With server-side shuffling the URL already points at a random
+    // endpoint; the manual picker would only confuse (#173).
+    if (config.download_cdn_shuffle) {
+      return url;
+    }
+
+    const routes = config.download_cdn_routes;
     if (!routes || routes.length === 0) {
       return url;
     }

inventory/setting.go

@@ -572,6 +572,7 @@ var DefaultSettings = map[string]string{
 	"qq_connect_register_enabled":                "1",
 	"upload_dedup_scope":                         "owner",
 	"download_cdn_routes":                        "",
+	"download_cdn_shuffle":                       "0",
 	"email_filter_mode":                          "0",
 	"email_filter_list":                          "",
 	"email_disable_subaddress":                   "0",

pkg/filemanager/manager/entitysource/entitysource.go

@@ -612,7 +612,13 @@ func (f *entitySource) Url(ctx context.Context, opts ...EntitySourceOption) (*En
 	// 4. The entity is encrypted and internal proxy not disabled by option
 	handlerCapability := f.handler.Capabilities()
 	if f.ShouldInternalProxy() {
+		// Download URLs may be distributed across the site URL and
+		// configured CDN routes (#173); preview/thumb URLs stay on the
+		// resolved site URL to avoid cross-origin viewer breakage.
 		siteUrl := f.settings.SiteURL(ctx)
+		if f.o.IsDownload {
+			siteUrl = f.settings.DownloadURLBase(ctx)
+		}
 		base := routes.MasterFileContentUrl(
 			siteUrl,
 			hashid.EncodeEntityID(f.hasher, f.e.ID()),

pkg/setting/provider.go

@@ -5,6 +5,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"math/rand/v2"
 	"net/url"
 	"sort"
 	"strconv"
@@ -260,6 +261,15 @@ type (
 		// DownloadCDNRoutes returns the configured alternative download
 		// endpoints users can pick from (e.g. CDN mirrors of the site).
 		DownloadCDNRoutes(ctx context.Context) []CDNRoute
+		// DownloadCDNShuffle returns true if generated download URLs should be
+		// distributed randomly across the site URL and all CDN routes.
+		DownloadCDNShuffle(ctx context.Context) bool
+		// DownloadURLBase returns the base URL used when generating file
+		// download URLs. With `download_cdn_shuffle` enabled and CDN routes
+		// configured, an endpoint is picked uniformly at random from the
+		// primary site URL plus all routes; otherwise it falls back to
+		// SiteURL. A context pinned by UseFirstSiteUrl always returns SiteURL.
+		DownloadURLBase(ctx context.Context) *url.URL
 		// AuditLogEnabled returns true if the given audit event type is
 		// recorded. An empty/unset list records everything.
 		AuditLogEnabled(ctx context.Context, eventType int) bool
@@ -1064,6 +1074,32 @@ func (s *settingProvider) DownloadCDNRoutes(ctx context.Context) []CDNRoute {
 	return routes
 }
 
+func (s *settingProvider) DownloadCDNShuffle(ctx context.Context) bool {
+	return s.getBoolean(ctx, "download_cdn_shuffle", false)
+}
+
+func (s *settingProvider) DownloadURLBase(ctx context.Context) *url.URL {
+	if _, pinned := ctx.Value(UseFirstSiteUrlCtxKey{}).(bool); pinned {
+		return s.SiteURL(ctx)
+	}
+	if !s.DownloadCDNShuffle(ctx) {
+		return s.SiteURL(ctx)
+	}
+	routes := s.DownloadCDNRoutes(ctx)
+	if len(routes) == 0 {
+		return s.SiteURL(ctx)
+	}
+
+	pool := make([]*url.URL, 0, len(routes)+1)
+	pool = append(pool, s.SiteURL(ctx))
+	for _, r := range routes {
+		if u, err := url.Parse(r.URL); err == nil && u.Scheme != "" && u.Host != "" {
+			pool = append(pool, u)
+		}
+	}
+	return pool[rand.IntN(len(pool))]
+}
+
 func (s *settingProvider) ShareDefaults(ctx context.Context) *ShareDefaults {
 	level := types.ShareLinksInProfileLevel(s.getString(ctx, "default_share_links_in_profile", ""))
 	switch level {

pkg/setting/provider_test.go

@@ -35,3 +35,52 @@ func TestDownloadCDNRoutes(t *testing.T) {
 		{Name: "cdn2", URL: "https://cdn2.example.com/base"},
 	}, routes)
 }
+
+func TestDownloadURLBase(t *testing.T) {
+	ctx := context.Background()
+
+	// Shuffle disabled: always the resolved site URL.
+	p := NewProvider(stubAdapter{
+		"siteURL":             "https://a.example.com,https://b.example.com",
+		"download_cdn_routes": "cdn1=https://cdn1.example.com",
+	})
+	require.Equal(t, "https://a.example.com", p.DownloadURLBase(ctx).String())
+
+	// Shuffle enabled without routes: still the site URL.
+	p = NewProvider(stubAdapter{
+		"siteURL":              "https://a.example.com",
+		"download_cdn_shuffle": "1",
+	})
+	require.Equal(t, "https://a.example.com", p.DownloadURLBase(ctx).String())
+
+	// Shuffle enabled: every draw lands on site URL or a configured route,
+	// and all endpoints are reached over enough draws.
+	p = NewProvider(stubAdapter{
+		"siteURL":              "https://a.example.com",
+		"download_cdn_shuffle": "1",
+		"download_cdn_routes":  "cdn1=https://cdn1.example.com\ncdn2=https://cdn2.example.com",
+	})
+	seen := map[string]bool{}
+	for i := 0; i < 300; i++ {
+		seen[p.DownloadURLBase(ctx).String()] = true
+	}
+	require.Equal(t, map[string]bool{
+		"https://a.example.com":    true,
+		"https://cdn1.example.com": true,
+		"https://cdn2.example.com": true,
+	}, seen)
+
+	// UseFirstSiteUrl pins the primary site URL even with shuffle on.
+	pinned := context.WithValue(ctx, UseFirstSiteUrlCtxKey{}, true)
+	require.Equal(t, "https://a.example.com", p.DownloadURLBase(pinned).String())
+
+	// Invalid route entries never leak into the pool.
+	p = NewProvider(stubAdapter{
+		"siteURL":              "https://a.example.com",
+		"download_cdn_shuffle": "1",
+		"download_cdn_routes":  "bad=ftp://x.example.com",
+	})
+	for i := 0; i < 50; i++ {
+		require.Equal(t, "https://a.example.com", p.DownloadURLBase(ctx).String())
+	}
+}

service/basic/site.go

@@ -58,6 +58,11 @@ type SiteConfig struct {
 	// can offer a download-route picker (#2987).
 	DownloadCDNRoutes []setting.CDNRoute `json:"download_cdn_routes,omitempty"`
 
+	// DownloadCDNShuffle tells clients generated download URLs are already
+	// spread across the site URL and all CDN routes server-side (#173), so
+	// the manual route picker can be skipped.
+	DownloadCDNShuffle bool `json:"download_cdn_shuffle,omitempty"`
+
 	// AbuseCaptcha controls whether the report-abuse dialog shows captcha.
 	AbuseCaptcha bool `json:"abuse_captcha,omitempty"`
 
@@ -260,6 +265,7 @@ func (s *GetSettingService) GetSiteConfig(c *gin.Context) (*SiteConfig, error) {
 		ShareDefaultPrivate:        shareDefaults.PrivateByDefault,
 		DefaultShareLinksInProfile: string(shareDefaults.LinksInProfile),
 		DownloadCDNRoutes:          settings.DownloadCDNRoutes(c),
+		DownloadCDNShuffle:         settings.DownloadCDNShuffle(c),
 		AbuseCaptcha:               settings.AbuseCaptchaEnabled(c),
 		UploadDedup:                settings.DBFS(c).DedupScope != "off",
 		TaskNodes:                  taskNodes,

service/explorer/file.go

@@ -463,7 +463,7 @@ func (s *FileURLService) GetArchiveDownloadSession(c *gin.Context) (*FileURLResp
 		return nil, serializer.NewError(serializer.CodeInternalSetting, "failed to create archive download session", err)
 	}
 
-	base := settings.SiteURL(c)
+	base := settings.DownloadURLBase(c)
 	downloadUrl := routes.MasterArchiveDownloadUrl(base, sessionId)
 	finalUrl, err := auth.SignURI(c, dep.GeneralAuth(), downloadUrl.String(), &expire)
 	if err != nil {