From 6ef462f41240aed0cb350d9ccbc7080df0240e55 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Sun, 20 Sep 2026 12:47:30 +0200 Subject: [PATCH] Cap total stored bytes per storage policy Upstream #2178 item 1: DecompressSize-style bound for policy capacity. PolicySetting.MaxTotalSize caps cumulative entity bytes under a policy; PrepareUpload, batch upload validation, and copyFiles all enforce it. copyFiles tracks a baseline usage + locally accumulated batches because entities written inside its transaction are invisible to the usage query. The canonical ErrInsufficientCapacity is returned unwrapped so upstream errors.Is quota handling still matches; the policy detail is logged server-side. Admin policy editor gains a Max total capacity SizeInput. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ROADMAP.md | 1 + frontend/public/locales/en-US/dashboard.json | 2 ++ frontend/public/locales/zh-CN/dashboard.json | 2 ++ frontend/src/api/dashboard.ts | 1 + .../FormSections/StorageAndUploadSection.tsx | 16 +++++++++++ inventory/types/types.go | 3 +++ pkg/filemanager/fs/dbfs/manage.go | 21 +++++++++++++++ pkg/filemanager/fs/dbfs/upload.go | 6 +++++ pkg/filemanager/fs/dbfs/validator.go | 27 +++++++++++++++++++ pkg/filemanager/fs/dbfs/validator_test.go | 18 +++++++++++++ 10 files changed, 97 insertions(+) diff --git a/ROADMAP.md b/ROADMAP.md index f9539449..759dc6b3 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -217,6 +217,7 @@ Order = user-visible value first; each ships with backend + UI + tests. - [x] Share `hide_readme` option (upstream #2729 item 6) — `ShareProps.HideReadMe` (only meaningful with `ShowReadMe`); share navigator filters `README.md`/`README.txt` (case-insensitive) from listings while direct-path resolution stays open for the readme viewer; `detectReadMe` URI fallback now probes unconditionally; owner-only `hide_readme` on share responses; Share dialog nested checkbox - [x] 2FA recovery codes (upstream #2729 item 3) — `user.two_factor_backup_codes` sensitive JSON of `salt:sha256` digests; `PUT /user/setting/2fa/backup` regenerates 10 one-time codes behind a valid TOTP (rate-limited 5/h); `Verify2FA` falls back to single-use code consumption on TOTP failure; codes invalidated on secret rotation/disable; login phase gains a recovery-code input mode; security settings show remaining count + regenerate dialog - [x] Public share directory (upstream #2729 items 4+5) — `share.listed_publicly` opt-in column gated by new `GroupPermissionSharePublicList` group bit; rejected on password-protected shares at the service layer and normalized off at creation; `GET /share/listed` anonymous endpoint (rate-limited 60/min/IP, cursor pagination) listing only non-expired passwordless listed shares with case-insensitive name search across anchor + covered files; `listed_publicly` owner-visible in share responses; `/discover` page (anonymous-visible nav item + sign-in link), admin group Share section switch, en+zh locales +- [x] Storage policy total capacity (upstream #2178 item 1) — `PolicySetting.MaxTotalSize` caps cumulative entity bytes per policy; enforced in `PrepareUpload`, batch upload validation, and `copyFiles` (baseline usage + per-batch accumulation since tx writes are invisible to the usage query); canonical `ErrInsufficientCapacity` preserved so `errors.Is` quota handling still matches; admin policy editor gains a Max total capacity SizeInput, en+zh locales ## 6. Phase D — desktop, all platforms diff --git a/frontend/public/locales/en-US/dashboard.json b/frontend/public/locales/en-US/dashboard.json index 5eb9c1bc..ad61bdc1 100644 --- a/frontend/public/locales/en-US/dashboard.json +++ b/frontend/public/locales/en-US/dashboard.json @@ -1058,6 +1058,8 @@ "selectAStorageProvider": "Select a storage provider", "maxSizeOfSingleFile": "Max single file size", "maxSizeOfSingleFileDes": "Enter 0 to disable the limit.", + "maxTotalSize": "Max total capacity", + "maxTotalSizeDes": "Maximum total bytes stored under this policy. Uploads and copies that would exceed it are rejected. 0 means unlimited.", "enterFileExt": "Separated by semi-colon commas, leave blank to allow all file extensions.", "extList": "File extension restrictions", "noLimit": "No limit", diff --git a/frontend/public/locales/zh-CN/dashboard.json b/frontend/public/locales/zh-CN/dashboard.json index 6a4bc7ec..f0b47b9f 100644 --- a/frontend/public/locales/zh-CN/dashboard.json +++ b/frontend/public/locales/zh-CN/dashboard.json @@ -1058,6 +1058,8 @@ "selectAStorageProvider": "选择存储方式", "maxSizeOfSingleFile": "文件大小限制", "maxSizeOfSingleFileDes": "单个文件的最大大小，输入限制为 0 时表示不限制单文件大小。", + "maxTotalSize": "存储策略总容量", + "maxTotalSizeDes": "此存储策略下可存储的文件总大小上限，超出后上传和复制将被拒绝。输入 0 表示不限制。", "enterFileExt": "留空表示不限制文件扩展名，多个请以半角逗号 , 隔开。", "extList": "文件扩展名限制", "noLimit": "无限制", diff --git a/frontend/src/api/dashboard.ts b/frontend/src/api/dashboard.ts index eb311e64..4e5fe409 100644 --- a/frontend/src/api/dashboard.ts +++ b/frontend/src/api/dashboard.ts @@ -253,6 +253,7 @@ export interface PolicySetting { thumb_exts?: string[]; thumb_support_all_exts?: boolean; thumb_max_size?: number; + max_total_size?: number; relay?: boolean; pre_allocate?: boolean; media_meta_exts?: string[]; diff --git a/frontend/src/component/Admin/StoragePolicy/EditStoragePolicy/FormSections/StorageAndUploadSection.tsx b/frontend/src/component/Admin/StoragePolicy/EditStoragePolicy/FormSections/StorageAndUploadSection.tsx index bcbc0e81..e72526f5 100644 --- a/frontend/src/component/Admin/StoragePolicy/EditStoragePolicy/FormSections/StorageAndUploadSection.tsx +++ b/frontend/src/component/Admin/StoragePolicy/EditStoragePolicy/FormSections/StorageAndUploadSection.tsx @@ -110,6 +110,16 @@ const StorageAndUploadSection = () => { [setPolicy], ); + const onMaxTotalSizeChange = useCallback( + (e: number) => { + setPolicy((p: StoragePolicy) => ({ + ...p, + settings: { ...p.settings, max_total_size: e === 0 ? undefined : e }, + })); + }, + [setPolicy], + ); + const fileExts = useMemo(() => { return values.settings?.file_type?.join() ?? ""; }, [values.settings?.file_type]); @@ -278,6 +288,12 @@ const StorageAndUploadSection = () => { {t("policy.maxSizeOfSingleFileDes")} + + + + {t("policy.maxTotalSizeDes")} + + 0 { + var used int + _, used, err = f.fileClient.CountEntityByStoragePolicyID(ctx, dstPolicy.ID) + if err != nil { + return nil, nil, nil, fmt.Errorf("copy files: failed to get storage policy usage: %w", err) + } + dstPolicyUsed = int64(used) + } + dstAncestors := lo.Map(destination.AncestorsChain(), func(item *File, index int) *ent.File { return item.Model }) @@ -991,6 +1008,10 @@ func (f *DBFS) copyFiles(ctx context.Context, targets map[Navigator][]*File, des if err := f.validateUserCapacityRaw(ctx, sizeTotal, capacity); err != nil { return fs.ErrInsufficientCapacity } + if err := f.validatePolicyCapacityRaw(sizeTotal, dstPolicy, dstPolicyUsed); err != nil { + return err + } + dstPolicyUsed += sizeTotal limit -= len(targets) newDstMap, diff, err := fc.Copy(ctx, &inventory.CopyParameter{ diff --git a/pkg/filemanager/fs/dbfs/upload.go b/pkg/filemanager/fs/dbfs/upload.go index 22ae21d1..e2081aa2 100644 --- a/pkg/filemanager/fs/dbfs/upload.go +++ b/pkg/filemanager/fs/dbfs/upload.go @@ -68,6 +68,9 @@ func (f *DBFS) PreValidateUpload(ctx context.Context, dst *fs.URI, files ...fs.P if err := f.validateUserCapacity(ctx, total, dstFile.Owner()); err != nil { return err } + if err := f.validatePolicyCapacity(ctx, total, policy); err != nil { + return err + } return nil } @@ -162,6 +165,9 @@ func (f *DBFS) PrepareUpload(ctx context.Context, req *fs.UploadRequest, opts .. if err := validateNewFile(req.Props.Uri.Name(), req.Props.Size, policy); err != nil { return nil, err } + if err := f.validatePolicyCapacity(ctx, req.Props.Size, policy); err != nil { + return nil, err + } owner := ancestor.Owner() capacity, err := f.Capacity(ctx, owner) diff --git a/pkg/filemanager/fs/dbfs/validator.go b/pkg/filemanager/fs/dbfs/validator.go index dbe9582e..ac906351 100644 --- a/pkg/filemanager/fs/dbfs/validator.go +++ b/pkg/filemanager/fs/dbfs/validator.go @@ -121,3 +121,30 @@ func (f *DBFS) validateUserCapacityRaw(ctx context.Context, size int64, capacity } return nil } + +// validatePolicyCapacity checks that storing `size` more bytes under `policy` +// stays within the policy's MaxTotalSize cap, fetching current usage first. +func (f *DBFS) validatePolicyCapacity(ctx context.Context, size int64, policy *ent.StoragePolicy) error { + if policy.Settings.MaxTotalSize <= 0 { + return nil + } + + _, used, err := f.fileClient.CountEntityByStoragePolicyID(ctx, policy.ID) + if err != nil { + return fmt.Errorf("failed to get storage policy usage: %w", err) + } + return f.validatePolicyCapacityRaw(size, policy, int64(used)) +} + +// validatePolicyCapacityRaw validates the policy capacity against a +// caller-supplied usage figure — needed when new entities are being written +// inside a transaction the usage query cannot see yet. The canonical +// ErrInsufficientCapacity is returned unwrapped so upstream errors.Is +// checks still match; the policy detail is logged server-side. +func (f *DBFS) validatePolicyCapacityRaw(size int64, policy *ent.StoragePolicy, used int64) error { + if policyCap := policy.Settings.MaxTotalSize; policyCap > 0 && used+size > policyCap { + f.l.Warning("storage policy %q is full (%d + %d > %d)", policy.Name, used, size, policyCap) + return fs.ErrInsufficientCapacity + } + return nil +} diff --git a/pkg/filemanager/fs/dbfs/validator_test.go b/pkg/filemanager/fs/dbfs/validator_test.go index 3b381335..fd96076e 100644 --- a/pkg/filemanager/fs/dbfs/validator_test.go +++ b/pkg/filemanager/fs/dbfs/validator_test.go @@ -5,6 +5,8 @@ import ( "github.com/cloudreve/Cloudreve/v4/ent" "github.com/cloudreve/Cloudreve/v4/inventory/types" + "github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs" + "github.com/cloudreve/Cloudreve/v4/pkg/logging" "github.com/stretchr/testify/require" ) @@ -25,3 +27,19 @@ func TestValidateFileNameNativeChars(t *testing.T) { require.NoError(t, validateFileName("normal file.txt", nil)) } + +func TestValidatePolicyCapacityRaw(t *testing.T) { + f := &DBFS{l: logging.NewConsoleLogger(logging.LevelError)} + capped := &ent.StoragePolicy{Name: "s3", Settings: &types.PolicySetting{MaxTotalSize: 1000}} + open := &ent.StoragePolicy{Name: "local", Settings: &types.PolicySetting{}} + + // Zero cap disables the check. + require.NoError(t, f.validatePolicyCapacityRaw(1<<62, open, 1<<62)) + // Within budget. + require.NoError(t, f.validatePolicyCapacityRaw(400, capped, 500)) + // Exactly at the cap is the last allowed byte. + require.NoError(t, f.validatePolicyCapacityRaw(500, capped, 500)) + // Over the cap. + require.ErrorIs(t, f.validatePolicyCapacityRaw(501, capped, 500), fs.ErrInsufficientCapacity) + require.ErrorIs(t, f.validatePolicyCapacityRaw(1, capped, 1000), fs.ErrInsufficientCapacity) +}