Merge pull request #221 from Dvorinka/feat/policy-overflow

feat(storage): overflow chain — uploads spill to next policy when full

ROADMAP.md

@@ -219,6 +219,7 @@ Order = user-visible value first; each ships with backend + UI + tests.
 - [x] Public share directory (upstream #2729 items 4+5) — `share.listed_publicly` opt-in column gated by new `GroupPermissionSharePublicList` group bit; rejected on password-protected shares at the service layer and normalized off at creation; `GET /share/listed` anonymous endpoint (rate-limited 60/min/IP, cursor pagination) listing only non-expired passwordless listed shares with case-insensitive name search across anchor + covered files; `listed_publicly` owner-visible in share responses; `/discover` page (anonymous-visible nav item + sign-in link), admin group Share section switch, en+zh locales
 - [x] Decompression-bomb guards (meta #2 item 10) — `DecompressSize` now bounds cumulative extracted output (its documented "total file size" intent), not just compressed input: `checkExtractGuards` aborts at the limit and at a 100k-entry cap (`maxExtractEntries`, bounds dir-creation bombs), each entry stream wrapped in `cappedFile` so understated size headers cannot overrun; slave path receives the limit via `SlaveExtractArchiveTaskState.ExtractLimit`; all failures carry `queue.CriticalErr` (no retry of the same bomb); resume-safe via cursor-skip size accounting
 - [x] Storage policy total capacity (upstream #2178 item 1) — `PolicySetting.MaxTotalSize` caps cumulative entity bytes per policy; enforced in `PrepareUpload`, batch upload validation, and `copyFiles` (baseline usage + per-batch accumulation since tx writes are invisible to the usage query); canonical `ErrInsufficientCapacity` preserved so `errors.Is` quota handling still matches; admin policy editor gains a Max total capacity SizeInput, en+zh locales
+- [x] Storage policy overflow chain (upstream #2178 item 4) — `PolicySetting.OverflowPolicyID` links a fallback policy; `overflowChain` walks hops with cycle guard + hop cap, skipping suspended members and resolving load-balance members to weighted children; `PrepareUpload` spills to the first member with headroom for the file size so name/size/extension rules apply to the landing policy; `PreValidateUpload` checks aggregate chain headroom since batches may split across members; admin policy editor gains an Overflow policy select, en+zh locales
 
 ## 6. Phase D — desktop, all platforms
 

frontend/public/locales/en-US/dashboard.json

@@ -1060,6 +1060,9 @@
     "maxSizeOfSingleFileDes": "Enter 0 to disable the limit.",
     "maxTotalSize": "Max total capacity",
     "maxTotalSizeDes": "Maximum total bytes stored under this policy. Uploads and copies that would exceed it are rejected. 0 means unlimited.",
+    "overflowPolicy": "Overflow policy",
+    "overflowPolicyDes": "When this policy is out of capacity, new uploads spill into the selected policy. Chains of policies are supported.",
+    "overflowNone": "None",
     "enterFileExt": "Separated by semi-colon commas, leave blank to allow all file extensions.",
     "extList": "File extension restrictions",
     "noLimit": "No limit",

frontend/public/locales/zh-CN/dashboard.json

@@ -1060,6 +1060,9 @@
     "maxSizeOfSingleFileDes": "单个文件的最大大小，输入限制为 0 时表示不限制单文件大小。",
     "maxTotalSize": "存储策略总容量",
     "maxTotalSizeDes": "此存储策略下可存储的文件总大小上限，超出后上传和复制将被拒绝。输入 0 表示不限制。",
+    "overflowPolicy": "溢出存储策略",
+    "overflowPolicyDes": "当此存储策略容量用尽时，新上传将自动切换到所选存储策略。支持多级链式溢出。",
+    "overflowNone": "无",
     "enterFileExt": "留空表示不限制文件扩展名，多个请以半角逗号 , 隔开。",
     "extList": "文件扩展名限制",
     "noLimit": "无限制",

frontend/src/api/dashboard.ts

@@ -254,6 +254,7 @@ export interface PolicySetting {
   thumb_support_all_exts?: boolean;
   thumb_max_size?: number;
   max_total_size?: number;
+  overflow_policy_id?: number;
   relay?: boolean;
   pre_allocate?: boolean;
   media_meta_exts?: string[];

frontend/src/component/Admin/StoragePolicy/EditStoragePolicy/FormSections/StorageAndUploadSection.tsx

@@ -5,15 +5,19 @@ import {
   InputAdornment,
   Link,
   MenuItem,
+  SelectChangeEvent,
   Switch,
   Typography,
 } from "@mui/material";
 import { useCallback, useContext, useEffect, useMemo, useRef, useState } from "react";
 import { Trans, useTranslation } from "react-i18next";
+import { getStoragePolicyList } from "../../../../../api/api";
 import { StoragePolicy } from "../../../../../api/dashboard";
 import { PolicyType } from "../../../../../api/explorer";
+import { useAppDispatch } from "../../../../../redux/hooks";
 import SizeInput, { StyleOutlinedSelect } from "../../../../Common/SizeInput";
-import { DenseFilledTextField } from "../../../../Common/StyledComponents";
+import { DenseFilledTextField, DenseSelect } from "../../../../Common/StyledComponents";
+import { SquareMenuItem } from "../../../../FileManager/ContextMenu/ContextMenu";
 import SettingForm from "../../../../Pages/Setting/SettingForm";
 import MagicVarDialog from "../../../Common/MagicVarDialog";
 import { NoMarginHelperText, SettingSection, SettingSectionContent } from "../../../Settings/Settings";
@@ -25,8 +29,18 @@ import { fileMagicVars, pathMagicVars } from "./magicVars";
 const StorageAndUploadSection = () => {
   const { t } = useTranslation("dashboard");
   const { values, setPolicy, formRef } = useContext(StoragePolicySettingContext);
+  const dispatch = useAppDispatch();
   const [magicVarDialogOpen, setMagicVarDialogOpen] = useState(false);
   const [dialogType, setDialogType] = useState<"path" | "file">("path");
+  const [overflowCandidates, setOverflowCandidates] = useState<StoragePolicy[]>([]);
+
+  useEffect(() => {
+    dispatch(getStoragePolicyList({ page: 1, page_size: 1000, order_by: "id", order_direction: "asc" })).then(
+      (res) => {
+        setOverflowCandidates(res.policies.filter((p) => p.id !== values.id));
+      },
+    );
+  }, [values.id]);
 
   const fileNameInputRef = useRef<HTMLInputElement>(null);
 
@@ -120,6 +134,17 @@ const StorageAndUploadSection = () => {
     [setPolicy],
   );
 
+  const onOverflowChange = useCallback(
+    (e: SelectChangeEvent<unknown>) => {
+      const id = e.target.value as number;
+      setPolicy((p: StoragePolicy) => ({
+        ...p,
+        settings: { ...p.settings, overflow_policy_id: id === 0 ? undefined : id },
+      }));
+    },
+    [setPolicy],
+  );
+
   const fileExts = useMemo(() => {
     return values.settings?.file_type?.join() ?? "";
   }, [values.settings?.file_type]);
@@ -294,6 +319,22 @@ const StorageAndUploadSection = () => {
             <NoMarginHelperText>{t("policy.maxTotalSizeDes")}</NoMarginHelperText>
           </FormControl>
         </SettingForm>
+        <SettingForm title={t("policy.overflowPolicy")} lgWidth={5}>
+          <FormControl fullWidth>
+            <DenseSelect
+              value={values.settings?.overflow_policy_id ?? 0}
+              onChange={onOverflowChange}
+            >
+              <SquareMenuItem value={0}>{t("policy.overflowNone")}</SquareMenuItem>
+              {overflowCandidates.map((p) => (
+                <SquareMenuItem key={p.id} value={p.id}>
+                  {p.name}
+                </SquareMenuItem>
+              ))}
+            </DenseSelect>
+            <NoMarginHelperText>{t("policy.overflowPolicyDes")}</NoMarginHelperText>
+          </FormControl>
+        </SettingForm>
         <SettingForm title={t("policy.extList")} lgWidth={5}>
           <FormControl fullWidth>
             <DenseFilledTextField

inventory/types/types.go

@@ -127,6 +127,10 @@ type (
 		// MaxTotalSize caps the total bytes of entities stored under this
 		// policy. Checked at upload/copy validation; 0 means unlimited.
 		MaxTotalSize int64 `json:"max_total_size,omitempty"`
+		// OverflowPolicyID links the next policy in an overflow chain: when
+		// this policy has no headroom for an upload, the upload spills into
+		// the linked policy. Chains terminate at 0; cycles are guarded.
+		OverflowPolicyID int `json:"overflow_policy_id,omitempty"`
 		// Whether to upload file through server's relay.
 		Relay bool `json:"relay,omitempty"`
 		// Whether to pre allocate space for file before upload in physical disk.

pkg/filemanager/fs/dbfs/overflow_test.go

@@ -0,0 +1,130 @@
+package dbfs
+
+import (
+	"context"
+	"math"
+	"testing"
+
+	"github.com/cloudreve/Cloudreve/v4/ent"
+	"github.com/cloudreve/Cloudreve/v4/ent/enttest"
+	"github.com/cloudreve/Cloudreve/v4/ent/storagepolicy"
+	"github.com/cloudreve/Cloudreve/v4/inventory"
+	"github.com/cloudreve/Cloudreve/v4/inventory/types"
+	"github.com/cloudreve/Cloudreve/v4/pkg/boolset"
+	"github.com/cloudreve/Cloudreve/v4/pkg/conf"
+	"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
+	"github.com/cloudreve/Cloudreve/v4/pkg/hashid"
+	"github.com/cloudreve/Cloudreve/v4/pkg/logging"
+	"github.com/stretchr/testify/require"
+)
+
+func overflowDBFS(t *testing.T, client *ent.Client) *DBFS {
+	t.Helper()
+	hasher, err := hashid.New("overflow-test-salt")
+	require.NoError(t, err)
+	return &DBFS{
+		fileClient:          inventory.NewFileClient(client, conf.SQLiteDB, hasher),
+		storagePolicyClient: inventory.NewStoragePolicyClient(client, nil),
+		hasher:              hasher,
+		l:                   logging.NewConsoleLogger(logging.LevelError),
+	}
+}
+
+func mkPolicy(t *testing.T, client *ent.Client, name string, cap int64, overflow int) *ent.StoragePolicy {
+	t.Helper()
+	return client.StoragePolicy.Create().SetName(name).SetType("local").
+		SetStatus(storagepolicy.StatusActive).
+		SetSettings(&types.PolicySetting{MaxTotalSize: cap, OverflowPolicyID: overflow}).
+		SaveX(context.Background())
+}
+
+func mkUser(t *testing.T, client *ent.Client) *ent.User {
+	t.Helper()
+	g := client.Group.Create().SetName("g").SetPermissions(&boolset.BooleanSet{}).SaveX(context.Background())
+	return client.User.Create().SetEmail("o@example.com").SetNick("o").
+		SetGroup(g).SaveX(context.Background())
+}
+
+func seedEntity(t *testing.T, client *ent.Client, u *ent.User, p *ent.StoragePolicy, size int64) {
+	t.Helper()
+	client.Entity.Create().SetType(int(types.EntityTypeVersion)).
+		SetSource("cloudreve/data/" + p.Name + "/" + t.Name()).SetSize(size).
+		SetReferenceCount(1).SetCreatedBy(u.ID).
+		SetStoragePolicyEntities(p.ID).SaveX(context.Background())
+}
+
+func TestOverflowChainResolution(t *testing.T) {
+	client := enttest.Open(t, "sqlite3", "file:"+t.Name()+"?mode=memory&cache=shared")
+	t.Cleanup(func() { require.NoError(t, client.Close()) })
+	ctx := context.Background()
+	f := overflowDBFS(t, client)
+	u := mkUser(t, client)
+
+	open := mkPolicy(t, client, "open", 0, 0)
+	mid := mkPolicy(t, client, "mid", 300, open.ID)
+	head := mkPolicy(t, client, "head", 1000, mid.ID)
+	seedEntity(t, client, u, head, 900) // head: 100 bytes headroom
+
+	// Fits in the preferred policy — no spill.
+	require.Equal(t, head.ID, f.resolveOverflowPolicy(ctx, head, 100).ID)
+	// Head full for this size — spills to mid.
+	require.Equal(t, mid.ID, f.resolveOverflowPolicy(ctx, head, 200).ID)
+	// Head and mid both too small — lands on the uncapped tail.
+	require.Equal(t, open.ID, f.resolveOverflowPolicy(ctx, head, 400).ID)
+}
+
+func TestOverflowChainExhaustedAndCycles(t *testing.T) {
+	client := enttest.Open(t, "sqlite3", "file:"+t.Name()+"?mode=memory&cache=shared")
+	t.Cleanup(func() { require.NoError(t, client.Close()) })
+	ctx := context.Background()
+	f := overflowDBFS(t, client)
+	u := mkUser(t, client)
+
+	// Cycle: a -> b -> a, both full. Resolution must terminate and return
+	// the last reachable member so the caller still fails validation.
+	b := mkPolicy(t, client, "b", 10, 0)
+	a := mkPolicy(t, client, "a", 10, b.ID)
+	client.StoragePolicy.UpdateOne(b).SetSettings(&types.PolicySetting{MaxTotalSize: 10, OverflowPolicyID: a.ID}).ExecX(ctx)
+	seedEntity(t, client, u, a, 10)
+	seedEntity(t, client, u, b, 10)
+
+	require.Equal(t, b.ID, f.resolveOverflowPolicy(ctx, a, 1).ID)
+	require.ErrorIs(t, f.validatePolicyCapacity(ctx, 1, f.resolveOverflowPolicy(ctx, a, 1)),
+		fs.ErrInsufficientCapacity)
+
+	// Suspended members are skipped over to the next hop.
+	suspended := client.StoragePolicy.Create().SetName("sus").SetType("local").
+		SetStatus(storagepolicy.StatusSuspended).
+		SetSettings(&types.PolicySetting{}).SaveX(ctx)
+	tail := mkPolicy(t, client, "tail", 0, 0)
+	client.StoragePolicy.UpdateOne(suspended).SetSettings(&types.PolicySetting{OverflowPolicyID: tail.ID}).ExecX(ctx)
+	entry := mkPolicy(t, client, "entry", 5, suspended.ID)
+	seedEntity(t, client, u, entry, 5)
+
+	require.Equal(t, tail.ID, f.resolveOverflowPolicy(ctx, entry, 1).ID)
+}
+
+func TestOverflowChainHeadroom(t *testing.T) {
+	client := enttest.Open(t, "sqlite3", "file:"+t.Name()+"?mode=memory&cache=shared")
+	t.Cleanup(func() { require.NoError(t, client.Close()) })
+	ctx := context.Background()
+	f := overflowDBFS(t, client)
+	u := mkUser(t, client)
+
+	// Any uncapped member makes the chain unbounded.
+	open := mkPolicy(t, client, "open", 0, 0)
+	capped := mkPolicy(t, client, "capped", 100, open.ID)
+	seedEntity(t, client, u, capped, 60)
+	headroom, err := f.chainHeadroom(ctx, capped)
+	require.NoError(t, err)
+	require.Equal(t, int64(math.MaxInt64), headroom)
+
+	// Fully capped chain sums per-member headroom.
+	p2 := mkPolicy(t, client, "p2", 50, 0)
+	p1 := mkPolicy(t, client, "p1", 100, p2.ID)
+	seedEntity(t, client, u, p1, 30)
+	seedEntity(t, client, u, p2, 10)
+	headroom, err = f.chainHeadroom(ctx, p1)
+	require.NoError(t, err)
+	require.Equal(t, int64(70+40), headroom)
+}

pkg/filemanager/fs/dbfs/upload.go

@@ -64,13 +64,19 @@ func (f *DBFS) PreValidateUpload(ctx context.Context, dst *fs.URI, files ...fs.P
 		}
 	}
 
-	// Validate available capacity
+	// Validate available capacity — a batch may spill across the policy's
+	// overflow chain, so check aggregate headroom rather than one member.
 	if err := f.validateUserCapacity(ctx, total, dstFile.Owner()); err != nil {
 		return err
 	}
-	if err := f.validatePolicyCapacity(ctx, total, policy); err != nil {
+	headroom, err := f.chainHeadroom(ctx, policy)
+	if err != nil {
 		return err
 	}
+	if total > headroom {
+		f.l.Warning("storage policy %q overflow chain lacks headroom (%d > %d)", policy.Name, total, headroom)
+		return fs.ErrInsufficientCapacity
+	}
 
 	return nil
 }
@@ -147,6 +153,13 @@ func (f *DBFS) PrepareUpload(ctx context.Context, req *fs.UploadRequest, opts ..
 		return nil, serializer.NewError(serializer.CodePolicyNotAllowed, "Storage policy is suspended", nil)
 	}
 
+	// When the preferred policy has no headroom, spill into its overflow
+	// chain before any rule validation so constraints apply to the policy
+	// the entity will actually land on.
+	if policy.Settings.OverflowPolicyID != 0 {
+		policy = f.resolveOverflowPolicy(ctx, policy, req.Props.Size)
+	}
+
 	// Encryption setting
 	var (
 		encryptMetadata *types.EncryptMetadata

pkg/filemanager/fs/dbfs/validator.go

@@ -3,10 +3,13 @@ package dbfs
 import (
 	"context"
 	"fmt"
+	"math"
 	"regexp"
 	"strings"
 
 	"github.com/cloudreve/Cloudreve/v4/ent"
+	"github.com/cloudreve/Cloudreve/v4/ent/storagepolicy"
+	"github.com/cloudreve/Cloudreve/v4/inventory"
 	"github.com/cloudreve/Cloudreve/v4/inventory/types"
 	"github.com/cloudreve/Cloudreve/v4/pkg/activity"
 	"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
@@ -148,3 +151,77 @@ func (f *DBFS) validatePolicyCapacityRaw(size int64, policy *ent.StoragePolicy,
 	}
 	return nil
 }
+
+// maxOverflowHops bounds the overflow-chain walk so misconfigured cycles or
+// long chains cannot spin the upload path.
+const maxOverflowHops = 16
+
+// overflowChain returns the ordered concrete policies an upload may spill
+// into: `policy` itself first, then each `overflow_policy_id` hop. Suspended
+// members are skipped; load-balance members resolve to a weighted child. The
+// walk stops at a missing hop, a cycle, or maxOverflowHops.
+func (f *DBFS) overflowChain(ctx context.Context, policy *ent.StoragePolicy) []*ent.StoragePolicy {
+	sc, _ := inventory.InheritTx(ctx, f.storagePolicyClient)
+	seen := map[int]bool{policy.ID: true}
+	chain := make([]*ent.StoragePolicy, 0, 4)
+	cur := policy
+	for hops := 0; cur != nil && hops < maxOverflowHops; hops++ {
+		if cur.Type == types.PolicyTypeLoadBalance {
+			if child, err := sc.ResolveLoadBalance(ctx, cur); err == nil &&
+				child.Status == storagepolicy.StatusActive && !seen[child.ID] {
+				seen[child.ID] = true
+				chain = append(chain, child)
+			}
+		} else if cur.Status == storagepolicy.StatusActive {
+			chain = append(chain, cur)
+		}
+
+		nextID := cur.Settings.OverflowPolicyID
+		if nextID == 0 || seen[nextID] {
+			break
+		}
+		seen[nextID] = true
+		next, err := sc.GetPolicyByID(ctx, nextID)
+		if err != nil {
+			break
+		}
+		cur = next
+	}
+	return chain
+}
+
+// resolveOverflowPolicy picks the first chain member with headroom for
+// `size` more bytes. When every member is full it returns the last one, so
+// the caller's own capacity check still reports the canonical error.
+func (f *DBFS) resolveOverflowPolicy(ctx context.Context, policy *ent.StoragePolicy, size int64) *ent.StoragePolicy {
+	chain := f.overflowChain(ctx, policy)
+	for _, p := range chain {
+		if err := f.validatePolicyCapacity(ctx, size, p); err == nil {
+			if p.ID != policy.ID {
+				f.l.Info("storage policy %q full, upload overflows to %q", policy.Name, p.Name)
+			}
+			return p
+		}
+	}
+	if len(chain) == 0 {
+		return policy
+	}
+	return chain[len(chain)-1]
+}
+
+// chainHeadroom returns the aggregate bytes still storable across the
+// policy's overflow chain; math.MaxInt64 when any member is uncapped.
+func (f *DBFS) chainHeadroom(ctx context.Context, policy *ent.StoragePolicy) (int64, error) {
+	total := int64(0)
+	for _, p := range f.overflowChain(ctx, policy) {
+		if p.Settings.MaxTotalSize <= 0 {
+			return math.MaxInt64, nil
+		}
+		_, used, err := f.fileClient.CountEntityByStoragePolicyID(ctx, p.ID)
+		if err != nil {
+			return 0, fmt.Errorf("failed to get storage policy usage: %w", err)
+		}
+		total += max(p.Settings.MaxTotalSize-int64(used), 0)
+	}
+	return total, nil
+}