diff --git a/ROADMAP.md b/ROADMAP.md index 8a777585..833afd0f 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -219,6 +219,7 @@ Order = user-visible value first; each ships with backend + UI + tests. - [x] Public share directory (upstream #2729 items 4+5) — `share.listed_publicly` opt-in column gated by new `GroupPermissionSharePublicList` group bit; rejected on password-protected shares at the service layer and normalized off at creation; `GET /share/listed` anonymous endpoint (rate-limited 60/min/IP, cursor pagination) listing only non-expired passwordless listed shares with case-insensitive name search across anchor + covered files; `listed_publicly` owner-visible in share responses; `/discover` page (anonymous-visible nav item + sign-in link), admin group Share section switch, en+zh locales - [x] Decompression-bomb guards (meta #2 item 10) — `DecompressSize` now bounds cumulative extracted output (its documented "total file size" intent), not just compressed input: `checkExtractGuards` aborts at the limit and at a 100k-entry cap (`maxExtractEntries`, bounds dir-creation bombs), each entry stream wrapped in `cappedFile` so understated size headers cannot overrun; slave path receives the limit via `SlaveExtractArchiveTaskState.ExtractLimit`; all failures carry `queue.CriticalErr` (no retry of the same bomb); resume-safe via cursor-skip size accounting - [x] Storage policy total capacity (upstream #2178 item 1) — `PolicySetting.MaxTotalSize` caps cumulative entity bytes per policy; enforced in `PrepareUpload`, batch upload validation, and `copyFiles` (baseline usage + per-batch accumulation since tx writes are invisible to the usage query); canonical `ErrInsufficientCapacity` preserved so `errors.Is` quota handling still matches; admin policy editor gains a Max total capacity SizeInput, en+zh locales +- [x] Storage policy overflow chain (upstream #2178 item 4) — `PolicySetting.OverflowPolicyID` links a fallback policy; `overflowChain` walks hops with cycle guard + hop cap, skipping suspended members and resolving load-balance members to weighted children; `PrepareUpload` spills to the first member with headroom for the file size so name/size/extension rules apply to the landing policy; `PreValidateUpload` checks aggregate chain headroom since batches may split across members; admin policy editor gains an Overflow policy select, en+zh locales ## 6. Phase D — desktop, all platforms diff --git a/frontend/public/locales/en-US/dashboard.json b/frontend/public/locales/en-US/dashboard.json index ad61bdc1..56d4a403 100644 --- a/frontend/public/locales/en-US/dashboard.json +++ b/frontend/public/locales/en-US/dashboard.json @@ -1060,6 +1060,9 @@ "maxSizeOfSingleFileDes": "Enter 0 to disable the limit.", "maxTotalSize": "Max total capacity", "maxTotalSizeDes": "Maximum total bytes stored under this policy. Uploads and copies that would exceed it are rejected. 0 means unlimited.", + "overflowPolicy": "Overflow policy", + "overflowPolicyDes": "When this policy is out of capacity, new uploads spill into the selected policy. Chains of policies are supported.", + "overflowNone": "None", "enterFileExt": "Separated by semi-colon commas, leave blank to allow all file extensions.", "extList": "File extension restrictions", "noLimit": "No limit", diff --git a/frontend/public/locales/zh-CN/dashboard.json b/frontend/public/locales/zh-CN/dashboard.json index f0b47b9f..09d60f4b 100644 --- a/frontend/public/locales/zh-CN/dashboard.json +++ b/frontend/public/locales/zh-CN/dashboard.json @@ -1060,6 +1060,9 @@ "maxSizeOfSingleFileDes": "单个文件的最大大小，输入限制为 0 时表示不限制单文件大小。", "maxTotalSize": "存储策略总容量", "maxTotalSizeDes": "此存储策略下可存储的文件总大小上限，超出后上传和复制将被拒绝。输入 0 表示不限制。", + "overflowPolicy": "溢出存储策略", + "overflowPolicyDes": "当此存储策略容量用尽时，新上传将自动切换到所选存储策略。支持多级链式溢出。", + "overflowNone": "无", "enterFileExt": "留空表示不限制文件扩展名，多个请以半角逗号 , 隔开。", "extList": "文件扩展名限制", "noLimit": "无限制", diff --git a/frontend/src/api/dashboard.ts b/frontend/src/api/dashboard.ts index 4e5fe409..948ea051 100644 --- a/frontend/src/api/dashboard.ts +++ b/frontend/src/api/dashboard.ts @@ -254,6 +254,7 @@ export interface PolicySetting { thumb_support_all_exts?: boolean; thumb_max_size?: number; max_total_size?: number; + overflow_policy_id?: number; relay?: boolean; pre_allocate?: boolean; media_meta_exts?: string[]; diff --git a/frontend/src/component/Admin/StoragePolicy/EditStoragePolicy/FormSections/StorageAndUploadSection.tsx b/frontend/src/component/Admin/StoragePolicy/EditStoragePolicy/FormSections/StorageAndUploadSection.tsx index e72526f5..5092dc06 100644 --- a/frontend/src/component/Admin/StoragePolicy/EditStoragePolicy/FormSections/StorageAndUploadSection.tsx +++ b/frontend/src/component/Admin/StoragePolicy/EditStoragePolicy/FormSections/StorageAndUploadSection.tsx @@ -5,15 +5,19 @@ import { InputAdornment, Link, MenuItem, + SelectChangeEvent, Switch, Typography, } from "@mui/material"; import { useCallback, useContext, useEffect, useMemo, useRef, useState } from "react"; import { Trans, useTranslation } from "react-i18next"; +import { getStoragePolicyList } from "../../../../../api/api"; import { StoragePolicy } from "../../../../../api/dashboard"; import { PolicyType } from "../../../../../api/explorer"; +import { useAppDispatch } from "../../../../../redux/hooks"; import SizeInput, { StyleOutlinedSelect } from "../../../../Common/SizeInput"; -import { DenseFilledTextField } from "../../../../Common/StyledComponents"; +import { DenseFilledTextField, DenseSelect } from "../../../../Common/StyledComponents"; +import { SquareMenuItem } from "../../../../FileManager/ContextMenu/ContextMenu"; import SettingForm from "../../../../Pages/Setting/SettingForm"; import MagicVarDialog from "../../../Common/MagicVarDialog"; import { NoMarginHelperText, SettingSection, SettingSectionContent } from "../../../Settings/Settings"; @@ -25,8 +29,18 @@ import { fileMagicVars, pathMagicVars } from "./magicVars"; const StorageAndUploadSection = () => { const { t } = useTranslation("dashboard"); const { values, setPolicy, formRef } = useContext(StoragePolicySettingContext); + const dispatch = useAppDispatch(); const [magicVarDialogOpen, setMagicVarDialogOpen] = useState(false); const [dialogType, setDialogType] = useState<"path" | "file">("path"); + const [overflowCandidates, setOverflowCandidates] = useState([]); + + useEffect(() => { + dispatch(getStoragePolicyList({ page: 1, page_size: 1000, order_by: "id", order_direction: "asc" })).then( + (res) => { + setOverflowCandidates(res.policies.filter((p) => p.id !== values.id)); + }, + ); + }, [values.id]); const fileNameInputRef = useRef(null); @@ -120,6 +134,17 @@ const StorageAndUploadSection = () => { [setPolicy], ); + const onOverflowChange = useCallback( + (e: SelectChangeEvent) => { + const id = e.target.value as number; + setPolicy((p: StoragePolicy) => ({ + ...p, + settings: { ...p.settings, overflow_policy_id: id === 0 ? undefined : id }, + })); + }, + [setPolicy], + ); + const fileExts = useMemo(() => { return values.settings?.file_type?.join() ?? ""; }, [values.settings?.file_type]); @@ -294,6 +319,22 @@ const StorageAndUploadSection = () => { {t("policy.maxTotalSizeDes")} + + + + {t("policy.overflowNone")} + {overflowCandidates.map((p) => ( + + {p.name} + + ))} + + {t("policy.overflowPolicyDes")} + + b -> a, both full. Resolution must terminate and return + // the last reachable member so the caller still fails validation. + b := mkPolicy(t, client, "b", 10, 0) + a := mkPolicy(t, client, "a", 10, b.ID) + client.StoragePolicy.UpdateOne(b).SetSettings(&types.PolicySetting{MaxTotalSize: 10, OverflowPolicyID: a.ID}).ExecX(ctx) + seedEntity(t, client, u, a, 10) + seedEntity(t, client, u, b, 10) + + require.Equal(t, b.ID, f.resolveOverflowPolicy(ctx, a, 1).ID) + require.ErrorIs(t, f.validatePolicyCapacity(ctx, 1, f.resolveOverflowPolicy(ctx, a, 1)), + fs.ErrInsufficientCapacity) + + // Suspended members are skipped over to the next hop. + suspended := client.StoragePolicy.Create().SetName("sus").SetType("local"). + SetStatus(storagepolicy.StatusSuspended). + SetSettings(&types.PolicySetting{}).SaveX(ctx) + tail := mkPolicy(t, client, "tail", 0, 0) + client.StoragePolicy.UpdateOne(suspended).SetSettings(&types.PolicySetting{OverflowPolicyID: tail.ID}).ExecX(ctx) + entry := mkPolicy(t, client, "entry", 5, suspended.ID) + seedEntity(t, client, u, entry, 5) + + require.Equal(t, tail.ID, f.resolveOverflowPolicy(ctx, entry, 1).ID) +} + +func TestOverflowChainHeadroom(t *testing.T) { + client := enttest.Open(t, "sqlite3", "file:"+t.Name()+"?mode=memory&cache=shared") + t.Cleanup(func() { require.NoError(t, client.Close()) }) + ctx := context.Background() + f := overflowDBFS(t, client) + u := mkUser(t, client) + + // Any uncapped member makes the chain unbounded. + open := mkPolicy(t, client, "open", 0, 0) + capped := mkPolicy(t, client, "capped", 100, open.ID) + seedEntity(t, client, u, capped, 60) + headroom, err := f.chainHeadroom(ctx, capped) + require.NoError(t, err) + require.Equal(t, int64(math.MaxInt64), headroom) + + // Fully capped chain sums per-member headroom. + p2 := mkPolicy(t, client, "p2", 50, 0) + p1 := mkPolicy(t, client, "p1", 100, p2.ID) + seedEntity(t, client, u, p1, 30) + seedEntity(t, client, u, p2, 10) + headroom, err = f.chainHeadroom(ctx, p1) + require.NoError(t, err) + require.Equal(t, int64(70+40), headroom) +} diff --git a/pkg/filemanager/fs/dbfs/upload.go b/pkg/filemanager/fs/dbfs/upload.go index e2081aa2..b39f4100 100644 --- a/pkg/filemanager/fs/dbfs/upload.go +++ b/pkg/filemanager/fs/dbfs/upload.go @@ -64,13 +64,19 @@ func (f *DBFS) PreValidateUpload(ctx context.Context, dst *fs.URI, files ...fs.P } } - // Validate available capacity + // Validate available capacity — a batch may spill across the policy's + // overflow chain, so check aggregate headroom rather than one member. if err := f.validateUserCapacity(ctx, total, dstFile.Owner()); err != nil { return err } - if err := f.validatePolicyCapacity(ctx, total, policy); err != nil { + headroom, err := f.chainHeadroom(ctx, policy) + if err != nil { return err } + if total > headroom { + f.l.Warning("storage policy %q overflow chain lacks headroom (%d > %d)", policy.Name, total, headroom) + return fs.ErrInsufficientCapacity + } return nil } @@ -147,6 +153,13 @@ func (f *DBFS) PrepareUpload(ctx context.Context, req *fs.UploadRequest, opts .. return nil, serializer.NewError(serializer.CodePolicyNotAllowed, "Storage policy is suspended", nil) } + // When the preferred policy has no headroom, spill into its overflow + // chain before any rule validation so constraints apply to the policy + // the entity will actually land on. + if policy.Settings.OverflowPolicyID != 0 { + policy = f.resolveOverflowPolicy(ctx, policy, req.Props.Size) + } + // Encryption setting var ( encryptMetadata *types.EncryptMetadata diff --git a/pkg/filemanager/fs/dbfs/validator.go b/pkg/filemanager/fs/dbfs/validator.go index ac906351..37751b4c 100644 --- a/pkg/filemanager/fs/dbfs/validator.go +++ b/pkg/filemanager/fs/dbfs/validator.go @@ -3,10 +3,13 @@ package dbfs import ( "context" "fmt" + "math" "regexp" "strings" "github.com/cloudreve/Cloudreve/v4/ent" + "github.com/cloudreve/Cloudreve/v4/ent/storagepolicy" + "github.com/cloudreve/Cloudreve/v4/inventory" "github.com/cloudreve/Cloudreve/v4/inventory/types" "github.com/cloudreve/Cloudreve/v4/pkg/activity" "github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs" @@ -148,3 +151,77 @@ func (f *DBFS) validatePolicyCapacityRaw(size int64, policy *ent.StoragePolicy, } return nil } + +// maxOverflowHops bounds the overflow-chain walk so misconfigured cycles or +// long chains cannot spin the upload path. +const maxOverflowHops = 16 + +// overflowChain returns the ordered concrete policies an upload may spill +// into: `policy` itself first, then each `overflow_policy_id` hop. Suspended +// members are skipped; load-balance members resolve to a weighted child. The +// walk stops at a missing hop, a cycle, or maxOverflowHops. +func (f *DBFS) overflowChain(ctx context.Context, policy *ent.StoragePolicy) []*ent.StoragePolicy { + sc, _ := inventory.InheritTx(ctx, f.storagePolicyClient) + seen := map[int]bool{policy.ID: true} + chain := make([]*ent.StoragePolicy, 0, 4) + cur := policy + for hops := 0; cur != nil && hops < maxOverflowHops; hops++ { + if cur.Type == types.PolicyTypeLoadBalance { + if child, err := sc.ResolveLoadBalance(ctx, cur); err == nil && + child.Status == storagepolicy.StatusActive && !seen[child.ID] { + seen[child.ID] = true + chain = append(chain, child) + } + } else if cur.Status == storagepolicy.StatusActive { + chain = append(chain, cur) + } + + nextID := cur.Settings.OverflowPolicyID + if nextID == 0 || seen[nextID] { + break + } + seen[nextID] = true + next, err := sc.GetPolicyByID(ctx, nextID) + if err != nil { + break + } + cur = next + } + return chain +} + +// resolveOverflowPolicy picks the first chain member with headroom for +// `size` more bytes. When every member is full it returns the last one, so +// the caller's own capacity check still reports the canonical error. +func (f *DBFS) resolveOverflowPolicy(ctx context.Context, policy *ent.StoragePolicy, size int64) *ent.StoragePolicy { + chain := f.overflowChain(ctx, policy) + for _, p := range chain { + if err := f.validatePolicyCapacity(ctx, size, p); err == nil { + if p.ID != policy.ID { + f.l.Info("storage policy %q full, upload overflows to %q", policy.Name, p.Name) + } + return p + } + } + if len(chain) == 0 { + return policy + } + return chain[len(chain)-1] +} + +// chainHeadroom returns the aggregate bytes still storable across the +// policy's overflow chain; math.MaxInt64 when any member is uncapped. +func (f *DBFS) chainHeadroom(ctx context.Context, policy *ent.StoragePolicy) (int64, error) { + total := int64(0) + for _, p := range f.overflowChain(ctx, policy) { + if p.Settings.MaxTotalSize <= 0 { + return math.MaxInt64, nil + } + _, used, err := f.fileClient.CountEntityByStoragePolicyID(ctx, p.ID) + if err != nil { + return 0, fmt.Errorf("failed to get storage policy usage: %w", err) + } + total += max(p.Settings.MaxTotalSize-int64(used), 0) + } + return total, nil +}