Cap total stored bytes per storage policy

Upstream #2178 item 1: DecompressSize-style bound for policy capacity.
PolicySetting.MaxTotalSize caps cumulative entity bytes under a policy;
PrepareUpload, batch upload validation, and copyFiles all enforce it.
copyFiles tracks a baseline usage + locally accumulated batches because
entities written inside its transaction are invisible to the usage
query. The canonical ErrInsufficientCapacity is returned unwrapped so
upstream errors.Is quota handling still matches; the policy detail is
logged server-side. Admin policy editor gains a Max total capacity
SizeInput.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
pull/3589/head
Tomas Dvorak 2 weeks ago
parent ce6b33267e
commit 6ef462f412

@ -217,6 +217,7 @@ Order = user-visible value first; each ships with backend + UI + tests.
- [x] Share `hide_readme` option (upstream #2729 item 6) — `ShareProps.HideReadMe` (only meaningful with `ShowReadMe`); share navigator filters `README.md`/`README.txt` (case-insensitive) from listings while direct-path resolution stays open for the readme viewer; `detectReadMe` URI fallback now probes unconditionally; owner-only `hide_readme` on share responses; Share dialog nested checkbox - [x] Share `hide_readme` option (upstream #2729 item 6) — `ShareProps.HideReadMe` (only meaningful with `ShowReadMe`); share navigator filters `README.md`/`README.txt` (case-insensitive) from listings while direct-path resolution stays open for the readme viewer; `detectReadMe` URI fallback now probes unconditionally; owner-only `hide_readme` on share responses; Share dialog nested checkbox
- [x] 2FA recovery codes (upstream #2729 item 3) — `user.two_factor_backup_codes` sensitive JSON of `salt:sha256` digests; `PUT /user/setting/2fa/backup` regenerates 10 one-time codes behind a valid TOTP (rate-limited 5/h); `Verify2FA` falls back to single-use code consumption on TOTP failure; codes invalidated on secret rotation/disable; login phase gains a recovery-code input mode; security settings show remaining count + regenerate dialog - [x] 2FA recovery codes (upstream #2729 item 3) — `user.two_factor_backup_codes` sensitive JSON of `salt:sha256` digests; `PUT /user/setting/2fa/backup` regenerates 10 one-time codes behind a valid TOTP (rate-limited 5/h); `Verify2FA` falls back to single-use code consumption on TOTP failure; codes invalidated on secret rotation/disable; login phase gains a recovery-code input mode; security settings show remaining count + regenerate dialog
- [x] Public share directory (upstream #2729 items 4+5) — `share.listed_publicly` opt-in column gated by new `GroupPermissionSharePublicList` group bit; rejected on password-protected shares at the service layer and normalized off at creation; `GET /share/listed` anonymous endpoint (rate-limited 60/min/IP, cursor pagination) listing only non-expired passwordless listed shares with case-insensitive name search across anchor + covered files; `listed_publicly` owner-visible in share responses; `/discover` page (anonymous-visible nav item + sign-in link), admin group Share section switch, en+zh locales - [x] Public share directory (upstream #2729 items 4+5) — `share.listed_publicly` opt-in column gated by new `GroupPermissionSharePublicList` group bit; rejected on password-protected shares at the service layer and normalized off at creation; `GET /share/listed` anonymous endpoint (rate-limited 60/min/IP, cursor pagination) listing only non-expired passwordless listed shares with case-insensitive name search across anchor + covered files; `listed_publicly` owner-visible in share responses; `/discover` page (anonymous-visible nav item + sign-in link), admin group Share section switch, en+zh locales
- [x] Storage policy total capacity (upstream #2178 item 1) — `PolicySetting.MaxTotalSize` caps cumulative entity bytes per policy; enforced in `PrepareUpload`, batch upload validation, and `copyFiles` (baseline usage + per-batch accumulation since tx writes are invisible to the usage query); canonical `ErrInsufficientCapacity` preserved so `errors.Is` quota handling still matches; admin policy editor gains a Max total capacity SizeInput, en+zh locales
## 6. Phase D — desktop, all platforms ## 6. Phase D — desktop, all platforms

@ -1058,6 +1058,8 @@
"selectAStorageProvider": "Select a storage provider", "selectAStorageProvider": "Select a storage provider",
"maxSizeOfSingleFile": "Max single file size", "maxSizeOfSingleFile": "Max single file size",
"maxSizeOfSingleFileDes": "Enter 0 to disable the limit.", "maxSizeOfSingleFileDes": "Enter 0 to disable the limit.",
"maxTotalSize": "Max total capacity",
"maxTotalSizeDes": "Maximum total bytes stored under this policy. Uploads and copies that would exceed it are rejected. 0 means unlimited.",
"enterFileExt": "Separated by semi-colon commas, leave blank to allow all file extensions.", "enterFileExt": "Separated by semi-colon commas, leave blank to allow all file extensions.",
"extList": "File extension restrictions", "extList": "File extension restrictions",
"noLimit": "No limit", "noLimit": "No limit",

@ -1058,6 +1058,8 @@
"selectAStorageProvider": "选择存储方式", "selectAStorageProvider": "选择存储方式",
"maxSizeOfSingleFile": "文件大小限制", "maxSizeOfSingleFile": "文件大小限制",
"maxSizeOfSingleFileDes": "单个文件的最大大小,输入限制为 0 时表示不限制单文件大小。", "maxSizeOfSingleFileDes": "单个文件的最大大小,输入限制为 0 时表示不限制单文件大小。",
"maxTotalSize": "存储策略总容量",
"maxTotalSizeDes": "此存储策略下可存储的文件总大小上限,超出后上传和复制将被拒绝。输入 0 表示不限制。",
"enterFileExt": "留空表示不限制文件扩展名,多个请以半角逗号 , 隔开。", "enterFileExt": "留空表示不限制文件扩展名,多个请以半角逗号 , 隔开。",
"extList": "文件扩展名限制", "extList": "文件扩展名限制",
"noLimit": "无限制", "noLimit": "无限制",

@ -253,6 +253,7 @@ export interface PolicySetting {
thumb_exts?: string[]; thumb_exts?: string[];
thumb_support_all_exts?: boolean; thumb_support_all_exts?: boolean;
thumb_max_size?: number; thumb_max_size?: number;
max_total_size?: number;
relay?: boolean; relay?: boolean;
pre_allocate?: boolean; pre_allocate?: boolean;
media_meta_exts?: string[]; media_meta_exts?: string[];

@ -110,6 +110,16 @@ const StorageAndUploadSection = () => {
[setPolicy], [setPolicy],
); );
const onMaxTotalSizeChange = useCallback(
(e: number) => {
setPolicy((p: StoragePolicy) => ({
...p,
settings: { ...p.settings, max_total_size: e === 0 ? undefined : e },
}));
},
[setPolicy],
);
const fileExts = useMemo(() => { const fileExts = useMemo(() => {
return values.settings?.file_type?.join() ?? ""; return values.settings?.file_type?.join() ?? "";
}, [values.settings?.file_type]); }, [values.settings?.file_type]);
@ -278,6 +288,12 @@ const StorageAndUploadSection = () => {
<NoMarginHelperText>{t("policy.maxSizeOfSingleFileDes")}</NoMarginHelperText> <NoMarginHelperText>{t("policy.maxSizeOfSingleFileDes")}</NoMarginHelperText>
</FormControl> </FormControl>
</SettingForm> </SettingForm>
<SettingForm title={t("policy.maxTotalSize")} lgWidth={5}>
<FormControl fullWidth>
<SizeInput variant={"outlined"} value={values.settings?.max_total_size ?? 0} onChange={onMaxTotalSizeChange} />
<NoMarginHelperText>{t("policy.maxTotalSizeDes")}</NoMarginHelperText>
</FormControl>
</SettingForm>
<SettingForm title={t("policy.extList")} lgWidth={5}> <SettingForm title={t("policy.extList")} lgWidth={5}>
<FormControl fullWidth> <FormControl fullWidth>
<DenseFilledTextField <DenseFilledTextField

@ -124,6 +124,9 @@ type (
ThumbSupportAllExts bool `json:"thumb_support_all_exts,omitempty"` ThumbSupportAllExts bool `json:"thumb_support_all_exts,omitempty"`
// ThumbMaxSize indicates the maximum allowed size of a thumbnail. 0 indicates that no limit is set. // ThumbMaxSize indicates the maximum allowed size of a thumbnail. 0 indicates that no limit is set.
ThumbMaxSize int64 `json:"thumb_max_size,omitempty"` ThumbMaxSize int64 `json:"thumb_max_size,omitempty"`
// MaxTotalSize caps the total bytes of entities stored under this
// policy. Checked at upload/copy validation; 0 means unlimited.
MaxTotalSize int64 `json:"max_total_size,omitempty"`
// Whether to upload file through server's relay. // Whether to upload file through server's relay.
Relay bool `json:"relay,omitempty"` Relay bool `json:"relay,omitempty"`
// Whether to pre allocate space for file before upload in physical disk. // Whether to pre allocate space for file before upload in physical disk.

@ -967,6 +967,23 @@ func (f *DBFS) copyFiles(ctx context.Context, targets map[Navigator][]*File, des
return nil, nil, nil, fmt.Errorf("copy files: failed to destination owner capacity: %w", err) return nil, nil, nil, fmt.Errorf("copy files: failed to destination owner capacity: %w", err)
} }
// Baseline usage of the destination policy: new entities are written in
// this transaction, invisible to the usage query, so copied batches are
// accumulated locally.
dstPolicy, err := f.getPreferredPolicy(ctx, destination)
if err != nil {
return nil, nil, nil, fmt.Errorf("copy files: failed to get destination storage policy: %w", err)
}
var dstPolicyUsed int64
if dstPolicy.Settings.MaxTotalSize > 0 {
var used int
_, used, err = f.fileClient.CountEntityByStoragePolicyID(ctx, dstPolicy.ID)
if err != nil {
return nil, nil, nil, fmt.Errorf("copy files: failed to get storage policy usage: %w", err)
}
dstPolicyUsed = int64(used)
}
dstAncestors := lo.Map(destination.AncestorsChain(), func(item *File, index int) *ent.File { dstAncestors := lo.Map(destination.AncestorsChain(), func(item *File, index int) *ent.File {
return item.Model return item.Model
}) })
@ -991,6 +1008,10 @@ func (f *DBFS) copyFiles(ctx context.Context, targets map[Navigator][]*File, des
if err := f.validateUserCapacityRaw(ctx, sizeTotal, capacity); err != nil { if err := f.validateUserCapacityRaw(ctx, sizeTotal, capacity); err != nil {
return fs.ErrInsufficientCapacity return fs.ErrInsufficientCapacity
} }
if err := f.validatePolicyCapacityRaw(sizeTotal, dstPolicy, dstPolicyUsed); err != nil {
return err
}
dstPolicyUsed += sizeTotal
limit -= len(targets) limit -= len(targets)
newDstMap, diff, err := fc.Copy(ctx, &inventory.CopyParameter{ newDstMap, diff, err := fc.Copy(ctx, &inventory.CopyParameter{

@ -68,6 +68,9 @@ func (f *DBFS) PreValidateUpload(ctx context.Context, dst *fs.URI, files ...fs.P
if err := f.validateUserCapacity(ctx, total, dstFile.Owner()); err != nil { if err := f.validateUserCapacity(ctx, total, dstFile.Owner()); err != nil {
return err return err
} }
if err := f.validatePolicyCapacity(ctx, total, policy); err != nil {
return err
}
return nil return nil
} }
@ -162,6 +165,9 @@ func (f *DBFS) PrepareUpload(ctx context.Context, req *fs.UploadRequest, opts ..
if err := validateNewFile(req.Props.Uri.Name(), req.Props.Size, policy); err != nil { if err := validateNewFile(req.Props.Uri.Name(), req.Props.Size, policy); err != nil {
return nil, err return nil, err
} }
if err := f.validatePolicyCapacity(ctx, req.Props.Size, policy); err != nil {
return nil, err
}
owner := ancestor.Owner() owner := ancestor.Owner()
capacity, err := f.Capacity(ctx, owner) capacity, err := f.Capacity(ctx, owner)

@ -121,3 +121,30 @@ func (f *DBFS) validateUserCapacityRaw(ctx context.Context, size int64, capacity
} }
return nil return nil
} }
// validatePolicyCapacity checks that storing `size` more bytes under `policy`
// stays within the policy's MaxTotalSize cap, fetching current usage first.
func (f *DBFS) validatePolicyCapacity(ctx context.Context, size int64, policy *ent.StoragePolicy) error {
if policy.Settings.MaxTotalSize <= 0 {
return nil
}
_, used, err := f.fileClient.CountEntityByStoragePolicyID(ctx, policy.ID)
if err != nil {
return fmt.Errorf("failed to get storage policy usage: %w", err)
}
return f.validatePolicyCapacityRaw(size, policy, int64(used))
}
// validatePolicyCapacityRaw validates the policy capacity against a
// caller-supplied usage figure — needed when new entities are being written
// inside a transaction the usage query cannot see yet. The canonical
// ErrInsufficientCapacity is returned unwrapped so upstream errors.Is
// checks still match; the policy detail is logged server-side.
func (f *DBFS) validatePolicyCapacityRaw(size int64, policy *ent.StoragePolicy, used int64) error {
if policyCap := policy.Settings.MaxTotalSize; policyCap > 0 && used+size > policyCap {
f.l.Warning("storage policy %q is full (%d + %d > %d)", policy.Name, used, size, policyCap)
return fs.ErrInsufficientCapacity
}
return nil
}

@ -5,6 +5,8 @@ import (
"github.com/cloudreve/Cloudreve/v4/ent" "github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/inventory/types" "github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -25,3 +27,19 @@ func TestValidateFileNameNativeChars(t *testing.T) {
require.NoError(t, validateFileName("normal file.txt", nil)) require.NoError(t, validateFileName("normal file.txt", nil))
} }
func TestValidatePolicyCapacityRaw(t *testing.T) {
f := &DBFS{l: logging.NewConsoleLogger(logging.LevelError)}
capped := &ent.StoragePolicy{Name: "s3", Settings: &types.PolicySetting{MaxTotalSize: 1000}}
open := &ent.StoragePolicy{Name: "local", Settings: &types.PolicySetting{}}
// Zero cap disables the check.
require.NoError(t, f.validatePolicyCapacityRaw(1<<62, open, 1<<62))
// Within budget.
require.NoError(t, f.validatePolicyCapacityRaw(400, capped, 500))
// Exactly at the cap is the last allowed byte.
require.NoError(t, f.validatePolicyCapacityRaw(500, capped, 500))
// Over the cap.
require.ErrorIs(t, f.validatePolicyCapacityRaw(501, capped, 500), fs.ErrInsufficientCapacity)
require.ErrorIs(t, f.validatePolicyCapacityRaw(1, capped, 1000), fs.ErrInsufficientCapacity)
}

Loading…
Cancel
Save