From 0a768b0f923390de103299c47d91fc9ac099f78d Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Sun, 20 Sep 2026 09:29:11 +0200 Subject: [PATCH] feat: shuffle download URLs across site URL and CDN routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fork #173: admins running multiple download endpoints want generated download links spread across them so external download managers hit different hosts per link, instead of every URL pointing at the site URL. - setting.Provider.DownloadURLBase: uniform random pick from the site URL plus all valid download_cdn_routes when download_cdn_shuffle is on; falls back to SiteURL when off/unconfigured and honors UseFirstSiteUrl pins. - entitysource: internal-proxy entity URLs use the shuffled base only for explicit downloads — preview/thumb URLs keep the resolved site URL to avoid cross-origin viewer breakage. - Archive download sessions pick from the same pool; redirect-type direct links shuffle at resolve time via the entity URL path. - Site config exposes download_cdn_shuffle; the web client skips the manual route picker when the server already distributes. - Admin Site Information gets the toggle; en-US + zh-CN strings. Authored By: TDvorak Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ROADMAP.md | 1 + frontend/public/locales/en-US/dashboard.json | 2 + frontend/public/locales/zh-CN/dashboard.json | 2 + frontend/src/api/site.ts | 1 + .../src/component/Admin/Settings/Settings.tsx | 1 + .../SiteInformation/SiteInformation.tsx | 18 +++++++ frontend/src/redux/thunks/download.ts | 9 +++- inventory/setting.go | 1 + .../manager/entitysource/entitysource.go | 6 +++ pkg/setting/provider.go | 36 ++++++++++++++ pkg/setting/provider_test.go | 49 +++++++++++++++++++ service/basic/site.go | 6 +++ service/explorer/file.go | 2 +- 13 files changed, 132 insertions(+), 2 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 2d38be23..ba326a16 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -207,6 +207,7 @@ Order = user-visible value first; each ships with backend + UI + tests. - #3454 (PG FK on upload) is **Pro-only** — `audit_logs` doesn't exist in this codebase. When B.5 adds our own audit log: insert the audit row in the same tx *after* the file row, never before. - [x] `desloppify` pass — 73 review items dispositioned (46 fixed, 27 honestly skipped), strict score 77.1 (was 18.9); scorecard lives in README. `security-reviewer` pass done incrementally per batch (OAuth secrets, SSRF, process exec, path safety) - [x] Tag management page (upstream #2962) — owner-scoped `tag:` metadata stats/rename/recolor/delete in `inventory.FileClient`, `GET/PATCH/DELETE /file/tag` routes, Settings → Tags tab with merge-on-rename semantics +- [x] Download URL shuffling (#173) — `download_cdn_shuffle` distributes generated download URLs randomly across the site URL + `download_cdn_routes` endpoints (`setting.DownloadURLBase`, honors `UseFirstSiteUrl`); covers entity downloads, archive sessions, and redirect-type direct links; manual route picker hidden client-side while active ## 6. Phase D — desktop, all platforms diff --git a/frontend/public/locales/en-US/dashboard.json b/frontend/public/locales/en-US/dashboard.json index fcc8dcf0..c7e75217 100644 --- a/frontend/public/locales/en-US/dashboard.json +++ b/frontend/public/locales/en-US/dashboard.json @@ -854,6 +854,8 @@ "ssoRegisterEnabledDes": "Automatically create a local account when a user signs in via SSO for the first time. The sign-up email filter below also applies.", "downloadCdnRoutes": "Download CDN routes", "downloadCdnRoutesDes": "Alternative download endpoints offered to users, one per line in name=url format (e.g. Line 1=https://cdn1.example.com). Routes must proxy the full request path and query back to this site so signed URLs stay valid; CORS headers are required for in-browser downloads.", + "downloadCdnShuffle": "Shuffle download routes", + "downloadCdnShuffleDes": "Randomly distribute each generated download link across the site URL and all configured CDN routes, so batch exports to download managers hit different endpoints. The manual route picker is hidden while enabled.", "ssoAutoRedirect": "Auto redirect to SSO", "ssoAutoRedirectDes": "Skip the login form and send visitors straight to the identity provider. Append <0>?nosso=1 to the login URL to reach the password form (e.g. for admin recovery).", "ssoCallbackUrl": "Callback URL", diff --git a/frontend/public/locales/zh-CN/dashboard.json b/frontend/public/locales/zh-CN/dashboard.json index c3e958bf..31ea9b1e 100644 --- a/frontend/public/locales/zh-CN/dashboard.json +++ b/frontend/public/locales/zh-CN/dashboard.json @@ -854,6 +854,8 @@ "ssoRegisterEnabledDes": "用户首次通过 SSO 登录时自动创建本地账号。下方的注册邮箱过滤规则同样适用。", "downloadCdnRoutes": "下载 CDN 线路", "downloadCdnRoutesDes": "供用户选择的备用下载端点，每行一条，格式为 名称=URL（例如 线路1=https://cdn1.example.com）。线路需将完整的请求路径与查询串代理回本站以保证签名有效；浏览器内下载需要线路配置 CORS 头。", + "downloadCdnShuffle": "随机分发下载线路", + "downloadCdnShuffleDes": "启用后，每条生成的下载链接将随机分发到本站地址或任一已配置的 CDN 线路，批量导出到下载器时不同链接会命中不同端点。启用期间将隐藏手动选择线路。", "ssoAutoRedirect": "自动跳转至 SSO", "ssoAutoRedirectDes": "跳过登录表单，直接跳转至身份提供商。在登录地址后附加 <0>?nosso=1 可进入密码登录表单（例如管理员账户恢复）。", "ssoCallbackUrl": "回调地址", diff --git a/frontend/src/api/site.ts b/frontend/src/api/site.ts index 023a6216..fdc2238c 100644 --- a/frontend/src/api/site.ts +++ b/frontend/src/api/site.ts @@ -34,6 +34,7 @@ export interface SiteConfig { sso_auto_redirect?: boolean; qq_connect_enabled?: boolean; download_cdn_routes?: { name: string; url: string }[]; + download_cdn_shuffle?: boolean; abuse_captcha?: boolean; upload_dedup?: boolean; allow_select_node?: boolean; diff --git a/frontend/src/component/Admin/Settings/Settings.tsx b/frontend/src/component/Admin/Settings/Settings.tsx index f6389bb9..7cd858f5 100644 --- a/frontend/src/component/Admin/Settings/Settings.tsx +++ b/frontend/src/component/Admin/Settings/Settings.tsx @@ -165,6 +165,7 @@ const Settings = () => { "siteDes", "siteURL", "download_cdn_routes", + "download_cdn_shuffle", "siteScript", "pwa_small_icon", "pwa_medium_icon", diff --git a/frontend/src/component/Admin/Settings/SiteInformation/SiteInformation.tsx b/frontend/src/component/Admin/Settings/SiteInformation/SiteInformation.tsx index e26b20c6..0e82135c 100644 --- a/frontend/src/component/Admin/Settings/SiteInformation/SiteInformation.tsx +++ b/frontend/src/component/Admin/Settings/SiteInformation/SiteInformation.tsx @@ -65,6 +65,24 @@ const SiteInformation = () => { {t("settings.downloadCdnRoutesDes")} + + + + setSettings({ + download_cdn_shuffle: e.target.checked ? "1" : "0", + }) + } + /> + } + label={t("settings.downloadCdnShuffle")} + /> + {t("settings.downloadCdnShuffleDes")} + + > { return async (dispatch, getState) => { - const routes = getState().siteConfig.basic.config.download_cdn_routes; + const config = getState().siteConfig.basic.config; + // With server-side shuffling the URL already points at a random + // endpoint; the manual picker would only confuse (#173). + if (config.download_cdn_shuffle) { + return url; + } + + const routes = config.download_cdn_routes; if (!routes || routes.length === 0) { return url; } diff --git a/inventory/setting.go b/inventory/setting.go index 6c0a4dc2..18e1f004 100644 --- a/inventory/setting.go +++ b/inventory/setting.go @@ -572,6 +572,7 @@ var DefaultSettings = map[string]string{ "qq_connect_register_enabled": "1", "upload_dedup_scope": "owner", "download_cdn_routes": "", + "download_cdn_shuffle": "0", "email_filter_mode": "0", "email_filter_list": "", "email_disable_subaddress": "0", diff --git a/pkg/filemanager/manager/entitysource/entitysource.go b/pkg/filemanager/manager/entitysource/entitysource.go index e00b2acf..4b9c9eb4 100644 --- a/pkg/filemanager/manager/entitysource/entitysource.go +++ b/pkg/filemanager/manager/entitysource/entitysource.go @@ -612,7 +612,13 @@ func (f *entitySource) Url(ctx context.Context, opts ...EntitySourceOption) (*En // 4. The entity is encrypted and internal proxy not disabled by option handlerCapability := f.handler.Capabilities() if f.ShouldInternalProxy() { + // Download URLs may be distributed across the site URL and + // configured CDN routes (#173); preview/thumb URLs stay on the + // resolved site URL to avoid cross-origin viewer breakage. siteUrl := f.settings.SiteURL(ctx) + if f.o.IsDownload { + siteUrl = f.settings.DownloadURLBase(ctx) + } base := routes.MasterFileContentUrl( siteUrl, hashid.EncodeEntityID(f.hasher, f.e.ID()), diff --git a/pkg/setting/provider.go b/pkg/setting/provider.go index 0d1a53a6..9c8a9f1c 100644 --- a/pkg/setting/provider.go +++ b/pkg/setting/provider.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "math/rand/v2" "net/url" "sort" "strconv" @@ -260,6 +261,15 @@ type ( // DownloadCDNRoutes returns the configured alternative download // endpoints users can pick from (e.g. CDN mirrors of the site). DownloadCDNRoutes(ctx context.Context) []CDNRoute + // DownloadCDNShuffle returns true if generated download URLs should be + // distributed randomly across the site URL and all CDN routes. + DownloadCDNShuffle(ctx context.Context) bool + // DownloadURLBase returns the base URL used when generating file + // download URLs. With `download_cdn_shuffle` enabled and CDN routes + // configured, an endpoint is picked uniformly at random from the + // primary site URL plus all routes; otherwise it falls back to + // SiteURL. A context pinned by UseFirstSiteUrl always returns SiteURL. + DownloadURLBase(ctx context.Context) *url.URL // AuditLogEnabled returns true if the given audit event type is // recorded. An empty/unset list records everything. AuditLogEnabled(ctx context.Context, eventType int) bool @@ -1064,6 +1074,32 @@ func (s *settingProvider) DownloadCDNRoutes(ctx context.Context) []CDNRoute { return routes } +func (s *settingProvider) DownloadCDNShuffle(ctx context.Context) bool { + return s.getBoolean(ctx, "download_cdn_shuffle", false) +} + +func (s *settingProvider) DownloadURLBase(ctx context.Context) *url.URL { + if _, pinned := ctx.Value(UseFirstSiteUrlCtxKey{}).(bool); pinned { + return s.SiteURL(ctx) + } + if !s.DownloadCDNShuffle(ctx) { + return s.SiteURL(ctx) + } + routes := s.DownloadCDNRoutes(ctx) + if len(routes) == 0 { + return s.SiteURL(ctx) + } + + pool := make([]*url.URL, 0, len(routes)+1) + pool = append(pool, s.SiteURL(ctx)) + for _, r := range routes { + if u, err := url.Parse(r.URL); err == nil && u.Scheme != "" && u.Host != "" { + pool = append(pool, u) + } + } + return pool[rand.IntN(len(pool))] +} + func (s *settingProvider) ShareDefaults(ctx context.Context) *ShareDefaults { level := types.ShareLinksInProfileLevel(s.getString(ctx, "default_share_links_in_profile", "")) switch level { diff --git a/pkg/setting/provider_test.go b/pkg/setting/provider_test.go index 9645b4bf..98c64fc6 100644 --- a/pkg/setting/provider_test.go +++ b/pkg/setting/provider_test.go @@ -35,3 +35,52 @@ func TestDownloadCDNRoutes(t *testing.T) { {Name: "cdn2", URL: "https://cdn2.example.com/base"}, }, routes) } + +func TestDownloadURLBase(t *testing.T) { + ctx := context.Background() + + // Shuffle disabled: always the resolved site URL. + p := NewProvider(stubAdapter{ + "siteURL": "https://a.example.com,https://b.example.com", + "download_cdn_routes": "cdn1=https://cdn1.example.com", + }) + require.Equal(t, "https://a.example.com", p.DownloadURLBase(ctx).String()) + + // Shuffle enabled without routes: still the site URL. + p = NewProvider(stubAdapter{ + "siteURL": "https://a.example.com", + "download_cdn_shuffle": "1", + }) + require.Equal(t, "https://a.example.com", p.DownloadURLBase(ctx).String()) + + // Shuffle enabled: every draw lands on site URL or a configured route, + // and all endpoints are reached over enough draws. + p = NewProvider(stubAdapter{ + "siteURL": "https://a.example.com", + "download_cdn_shuffle": "1", + "download_cdn_routes": "cdn1=https://cdn1.example.com\ncdn2=https://cdn2.example.com", + }) + seen := map[string]bool{} + for i := 0; i < 300; i++ { + seen[p.DownloadURLBase(ctx).String()] = true + } + require.Equal(t, map[string]bool{ + "https://a.example.com": true, + "https://cdn1.example.com": true, + "https://cdn2.example.com": true, + }, seen) + + // UseFirstSiteUrl pins the primary site URL even with shuffle on. + pinned := context.WithValue(ctx, UseFirstSiteUrlCtxKey{}, true) + require.Equal(t, "https://a.example.com", p.DownloadURLBase(pinned).String()) + + // Invalid route entries never leak into the pool. + p = NewProvider(stubAdapter{ + "siteURL": "https://a.example.com", + "download_cdn_shuffle": "1", + "download_cdn_routes": "bad=ftp://x.example.com", + }) + for i := 0; i < 50; i++ { + require.Equal(t, "https://a.example.com", p.DownloadURLBase(ctx).String()) + } +} diff --git a/service/basic/site.go b/service/basic/site.go index 74f84285..1415eb86 100644 --- a/service/basic/site.go +++ b/service/basic/site.go @@ -58,6 +58,11 @@ type SiteConfig struct { // can offer a download-route picker (#2987). DownloadCDNRoutes []setting.CDNRoute `json:"download_cdn_routes,omitempty"` + // DownloadCDNShuffle tells clients generated download URLs are already + // spread across the site URL and all CDN routes server-side (#173), so + // the manual route picker can be skipped. + DownloadCDNShuffle bool `json:"download_cdn_shuffle,omitempty"` + // AbuseCaptcha controls whether the report-abuse dialog shows captcha. AbuseCaptcha bool `json:"abuse_captcha,omitempty"` @@ -260,6 +265,7 @@ func (s *GetSettingService) GetSiteConfig(c *gin.Context) (*SiteConfig, error) { ShareDefaultPrivate: shareDefaults.PrivateByDefault, DefaultShareLinksInProfile: string(shareDefaults.LinksInProfile), DownloadCDNRoutes: settings.DownloadCDNRoutes(c), + DownloadCDNShuffle: settings.DownloadCDNShuffle(c), AbuseCaptcha: settings.AbuseCaptchaEnabled(c), UploadDedup: settings.DBFS(c).DedupScope != "off", TaskNodes: taskNodes, diff --git a/service/explorer/file.go b/service/explorer/file.go index c558b0b0..215363f2 100644 --- a/service/explorer/file.go +++ b/service/explorer/file.go @@ -463,7 +463,7 @@ func (s *FileURLService) GetArchiveDownloadSession(c *gin.Context) (*FileURLResp return nil, serializer.NewError(serializer.CodeInternalSetting, "failed to create archive download session", err) } - base := settings.SiteURL(c) + base := settings.DownloadURLBase(c) downloadUrl := routes.MasterArchiveDownloadUrl(base, sessionId) finalUrl, err := auth.SignURI(c, dep.GeneralAuth(), downloadUrl.String(), &expire) if err != nil {