diff --git a/ROADMAP.md b/ROADMAP.md index 5337fce8..1a92a81d 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -220,6 +220,7 @@ Order = user-visible value first; each ships with backend + UI + tests. - [x] Decompression-bomb guards (meta #2 item 10) — `DecompressSize` now bounds cumulative extracted output (its documented "total file size" intent), not just compressed input: `checkExtractGuards` aborts at the limit and at a 100k-entry cap (`maxExtractEntries`, bounds dir-creation bombs), each entry stream wrapped in `cappedFile` so understated size headers cannot overrun; slave path receives the limit via `SlaveExtractArchiveTaskState.ExtractLimit`; all failures carry `queue.CriticalErr` (no retry of the same bomb); resume-safe via cursor-skip size accounting - [x] Storage policy total capacity (upstream #2178 item 1) — `PolicySetting.MaxTotalSize` caps cumulative entity bytes per policy; enforced in `PrepareUpload`, batch upload validation, and `copyFiles` (baseline usage + per-batch accumulation since tx writes are invisible to the usage query); canonical `ErrInsufficientCapacity` preserved so `errors.Is` quota handling still matches; admin policy editor gains a Max total capacity SizeInput, en+zh locales - [x] Storage policy overflow chain (upstream #2178 item 4) — `PolicySetting.OverflowPolicyID` links a fallback policy; `overflowChain` walks hops with cycle guard + hop cap, skipping suspended members and resolving load-balance members to weighted children; `PrepareUpload` spills to the first member with headroom for the file size so name/size/extension rules apply to the landing policy; `PreValidateUpload` checks aggregate chain headroom since batches may split across members; admin policy editor gains an Overflow policy select, en+zh locales +- [x] Thumbnail generation controls (upstream #2178 items 7+8) — `PolicySetting.ThumbForceProxy` skips the backend's native thumbnail API even when supported, implying the local proxy pipeline; `PolicySetting.ThumbStoragePolicyID` redirects generated thumb entities to a designated policy (honored via `PreferredStoragePolicy` for thumbnail uploads in `PrepareUpload`); Thumbnails section gains both controls, en+zh locales ## 6. Phase D — desktop, all platforms diff --git a/frontend/public/locales/en-US/dashboard.json b/frontend/public/locales/en-US/dashboard.json index 56d4a403..0a58c0cf 100644 --- a/frontend/public/locales/en-US/dashboard.json +++ b/frontend/public/locales/en-US/dashboard.json @@ -977,6 +977,11 @@ "nativeMediaMetaExtUpyun": "The image processing service supports <0>extracting image EXIF. No manual configuration is required, just add the extensions you want to process above.", "thumbProxy": "Proxy thumbnail generation", "thumbProxyDes": "Enable this feature to generate thumbnails for files that do not meet the native thumbnail conditions. Cloudreve will try to generate thumbnails and upload them to the storage side. Please configure the thumbnail generator in <0>Media processing.", + "thumbForceProxy": "Force proxy thumbnail generation", + "thumbForceProxyDes": "Always generate thumbnails with the local proxy pipeline, even when the storage backend offers a native thumbnail API. Implies proxy thumbnail generation.", + "thumbStoragePolicy": "Thumbnail storage policy", + "thumbStoragePolicySource": "Same as file", + "thumbStoragePolicyDes": "Generated thumbnails are stored under the selected policy instead of the file's own storage policy.", "nativeThumbnailMaxSize": "Max size of native thumbnails", "nativeThumbnailMaxSizeDes": "Enter 0 to disable the size limit, files larger than this size will not use native thumbnails.", "nativeThumbNailsSupportAllExts": "Enable for all file extension", diff --git a/frontend/public/locales/zh-CN/dashboard.json b/frontend/public/locales/zh-CN/dashboard.json index 09d60f4b..37633f55 100644 --- a/frontend/public/locales/zh-CN/dashboard.json +++ b/frontend/public/locales/zh-CN/dashboard.json @@ -977,6 +977,11 @@ "nativeMediaMetaExtUpyun": "图片处理服务支持 <0>提取图片 EXIF。无需手动配置，在上面加上你想要处理的图片的扩展名即可。", "thumbProxy": "代理生成缩略图", "thumbProxyDes": "开启后，对于不符合原生缩略图条件的文件，Cloudreve 会尝试为其生成缩略图文件，并上传到存储端。请在 <0>媒体处理 中配置 Cloudreve 缩略图生成器。", + "thumbForceProxy": "强制代理生成缩略图", + "thumbForceProxyDes": "即使存储端支持原生缩略图 API，也始终使用本地代理流水线生成缩略图。启用后等同于开启代理生成缩略图。", + "thumbStoragePolicy": "缩略图存储策略", + "thumbStoragePolicySource": "与文件相同", + "thumbStoragePolicyDes": "生成的缩略图将保存到所选存储策略，而非文件自身的存储策略。", "nativeThumbnailMaxSize": "使用原生缩略图的最大文件大小", "nativeThumbnailMaxSizeDes": "填写 0 表示不限制，超出此大小的文件将不会使用原生缩略图。", "nativeThumbNailsSupportAllExts": "对所有文件扩展名使用", diff --git a/frontend/src/api/dashboard.ts b/frontend/src/api/dashboard.ts index 948ea051..783d3b78 100644 --- a/frontend/src/api/dashboard.ts +++ b/frontend/src/api/dashboard.ts @@ -260,6 +260,8 @@ export interface PolicySetting { media_meta_exts?: string[]; media_meta_generator_proxy?: boolean; thumb_generator_proxy?: boolean; + thumb_force_proxy?: boolean; + thumb_storage_policy_id?: number; native_media_processing?: boolean; s3_delete_batch_size?: number; stream_saver?: boolean; diff --git a/frontend/src/component/Admin/StoragePolicy/EditStoragePolicy/FormSections/ThumbnailsSection.tsx b/frontend/src/component/Admin/StoragePolicy/EditStoragePolicy/FormSections/ThumbnailsSection.tsx index 13d8ca3b..a17ab526 100644 --- a/frontend/src/component/Admin/StoragePolicy/EditStoragePolicy/FormSections/ThumbnailsSection.tsx +++ b/frontend/src/component/Admin/StoragePolicy/EditStoragePolicy/FormSections/ThumbnailsSection.tsx @@ -1,11 +1,14 @@ -import { Checkbox, FormControl, FormControlLabel, Link, Switch, Typography } from "@mui/material"; -import { useCallback, useContext, useMemo } from "react"; +import { Checkbox, FormControl, FormControlLabel, Link, SelectChangeEvent, Switch, Typography } from "@mui/material"; +import { useCallback, useContext, useEffect, useMemo, useState } from "react"; import { Trans, useTranslation } from "react-i18next"; import { Link as RouterLink } from "react-router-dom"; +import { getStoragePolicyList } from "../../../../../api/api"; import { StoragePolicy } from "../../../../../api/dashboard"; import { PolicyType } from "../../../../../api/explorer"; +import { useAppDispatch } from "../../../../../redux/hooks"; import SizeInput from "../../../../Common/SizeInput"; -import { DenseFilledTextField } from "../../../../Common/StyledComponents"; +import { DenseFilledTextField, DenseSelect } from "../../../../Common/StyledComponents"; +import { SquareMenuItem } from "../../../../FileManager/ContextMenu/ContextMenu"; import SettingForm from "../../../../Pages/Setting/SettingForm"; import { NoMarginHelperText, SettingSection, SettingSectionContent } from "../../../Settings/Settings"; import { PolicyPropsMap } from "../../StoragePolicySetting"; @@ -14,6 +17,16 @@ import { StoragePolicySettingContext } from "../StoragePolicySettingWrapper"; const ThumbnailsSection = () => { const { t } = useTranslation("dashboard"); const { values, setPolicy } = useContext(StoragePolicySettingContext); + const dispatch = useAppDispatch(); + const [thumbPolicyCandidates, setThumbPolicyCandidates] = useState([]); + + useEffect(() => { + dispatch(getStoragePolicyList({ page: 1, page_size: 1000, order_by: "id", order_direction: "asc" })).then( + (res) => { + setThumbPolicyCandidates(res.policies.filter((p) => p.id !== values.id)); + }, + ); + }, [values.id]); const policyProps = useMemo(() => { return PolicyPropsMap[values.type]; @@ -66,9 +79,28 @@ const ThumbnailsSection = () => { [setPolicy], ); - if (values.type === PolicyType.local) { - return null; - } + const onThumbForceProxyChange = useCallback( + (e: React.ChangeEvent) => { + setPolicy((p: StoragePolicy) => ({ + ...p, + settings: { ...p.settings, thumb_force_proxy: e.target.checked ? true : undefined }, + })); + }, + [setPolicy], + ); + + const onThumbPolicyChange = useCallback( + (e: SelectChangeEvent) => { + const id = e.target.value as number; + setPolicy((p: StoragePolicy) => ({ + ...p, + settings: { ...p.settings, thumb_storage_policy_id: id === 0 ? undefined : id }, + })); + }, + [setPolicy], + ); + + const isLocal = values.type === PolicyType.local; return ( @@ -76,7 +108,7 @@ const ThumbnailsSection = () => { {t("settings.thumbnails")} - {!noNativeThumbnail && ( + {!isLocal && !noNativeThumbnail && ( <> @@ -129,21 +161,52 @@ const ThumbnailsSection = () => { )} - - - - } - label={t("policy.thumbProxy")} - /> - - ]} + {!isLocal && ( + + + + } + label={t("policy.thumbProxy")} /> - + + ]} + /> + + + + )} + {!isLocal && !noNativeThumbnail && ( + + + + } + label={t("policy.thumbForceProxy")} + /> + {t("policy.thumbForceProxyDes")} + + + )} + + + + {t("policy.thumbStoragePolicySource")} + {thumbPolicyCandidates.map((p) => ( + + {p.name} + + ))} + + {t("policy.thumbStoragePolicyDes")} diff --git a/inventory/types/types.go b/inventory/types/types.go index 50c175e2..6e6d69df 100644 --- a/inventory/types/types.go +++ b/inventory/types/types.go @@ -141,6 +141,14 @@ type ( MediaMetaGeneratorProxy bool `json:"media_meta_generator_proxy,omitempty"` // ThumbGeneratorProxy whether to use local proxy to generate thumbnail. ThumbGeneratorProxy bool `json:"thumb_generator_proxy,omitempty"` + // ThumbForceProxy forces thumbnail generation through the local proxy + // pipeline even when the storage backend offers a native thumbnail API. + // Implies ThumbGeneratorProxy behavior. + ThumbForceProxy bool `json:"thumb_force_proxy,omitempty"` + // ThumbStoragePolicyID overrides where generated thumbnail entities are + // stored — they ignore the source file's policy and land here. 0 keeps + // thumbnails on the source policy. + ThumbStoragePolicyID int `json:"thumb_storage_policy_id,omitempty"` // NativeMediaProcessing whether to use native media processing API from storage provider. NativeMediaProcessing bool `json:"native_media_processing"` // S3DeleteBatchSize the number of objects to delete in each batch. diff --git a/pkg/filemanager/fs/dbfs/thumb_policy_test.go b/pkg/filemanager/fs/dbfs/thumb_policy_test.go new file mode 100644 index 00000000..f4705985 --- /dev/null +++ b/pkg/filemanager/fs/dbfs/thumb_policy_test.go @@ -0,0 +1,64 @@ +package dbfs + +import ( + "context" + "testing" + "time" + + "github.com/cloudreve/Cloudreve/v4/ent/enttest" + "github.com/cloudreve/Cloudreve/v4/ent/storagepolicy" + "github.com/cloudreve/Cloudreve/v4/inventory/types" + "github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs" + "github.com/cloudreve/Cloudreve/v4/pkg/hashid" + "github.com/gofrs/uuid" + "github.com/stretchr/testify/require" +) + +func thumbUploadReq(t *testing.T, f *DBFS, u int, name string, size int64, preferredPolicy int) *fs.UploadRequest { + t.Helper() + uri, err := fs.NewUriFromString(fs.NewMyUri(hashid.EncodeUserID(f.hasher, u)) + "/" + name) + require.NoError(t, err) + thumbType := types.EntityTypeThumbnail + return &fs.UploadRequest{ + Props: &fs.UploadProps{ + Uri: uri, + Size: size, + UploadSessionID: uuid.Must(uuid.NewV4()).String(), + ExpireAt: time.Now().Add(time.Hour), + EntityType: &thumbType, + PreferredStoragePolicy: preferredPolicy, + }, + } +} + +func TestPrepareUploadThumbPolicyOverride(t *testing.T) { + client := enttest.Open(t, "sqlite3", "file:"+t.Name()+"?mode=memory&cache=shared") + t.Cleanup(func() { require.NoError(t, client.Close()) }) + ctx := context.Background() + + u, src, f := dedupUploadFixture(t, client, "off", 1<<40) + + dst := client.StoragePolicy.Create().SetName("thumbs").SetType("local"). + SetStatus(storagepolicy.StatusActive).SetSettings(&types.PolicySetting{}).SaveX(ctx) + + // Create the backing file so a thumbnail entity can attach; completing + // releases the upload-session lock the thumb request would hit. + s, err := f.PrepareUpload(ctx, uploadReq(t, f.hasher, u, "img.png", 1024, "")) + require.NoError(t, err) + require.Equal(t, src.ID, s.Policy.ID) + _, err = f.CompleteUpload(ctx, s) + require.NoError(t, err) + + // Thumbnail upload with a designated policy resolves to it, not the + // file's own policy. + s, err = f.PrepareUpload(ctx, thumbUploadReq(t, f, u.ID, "img.png", 64, dst.ID)) + require.NoError(t, err) + require.Equal(t, dst.ID, s.Policy.ID) + _, err = f.CompleteUpload(ctx, s) + require.NoError(t, err) + + // Without a designation the thumb follows the file's policy. + s, err = f.PrepareUpload(ctx, thumbUploadReq(t, f, u.ID, "img.png", 64, 0)) + require.NoError(t, err) + require.Equal(t, src.ID, s.Policy.ID) +} diff --git a/pkg/filemanager/fs/dbfs/upload.go b/pkg/filemanager/fs/dbfs/upload.go index b39f4100..10ff5694 100644 --- a/pkg/filemanager/fs/dbfs/upload.go +++ b/pkg/filemanager/fs/dbfs/upload.go @@ -140,10 +140,13 @@ func (f *DBFS) PrepareUpload(ctx context.Context, req *fs.UploadRequest, opts .. var ( policy *ent.StoragePolicy ) - if req.ImportFrom == nil { - policy, err = f.getPreferredPolicy(ctx, ancestor) - } else { + isThumb := req.Props.EntityType != nil && *req.Props.EntityType == types.EntityTypeThumbnail + if req.ImportFrom != nil || (isThumb && req.Props.PreferredStoragePolicy != 0) { + // Imports and server-designated thumbnail targets name their policy + // explicitly instead of inheriting the ancestor's preference. policy, err = f.storagePolicyClient.GetPolicyByID(ctx, req.Props.PreferredStoragePolicy) + } else { + policy, err = f.getPreferredPolicy(ctx, ancestor) } if err != nil { return nil, err diff --git a/pkg/filemanager/fs/dbfs/upload_dedup_test.go b/pkg/filemanager/fs/dbfs/upload_dedup_test.go index 17cbca1c..96619943 100644 --- a/pkg/filemanager/fs/dbfs/upload_dedup_test.go +++ b/pkg/filemanager/fs/dbfs/upload_dedup_test.go @@ -33,6 +33,10 @@ func (p dedupSettingProvider) DBFS(context.Context) *setting.DBFS { return &setting.DBFS{DedupScope: p.scope, MaxPageSize: 200, MaxRecursiveSearchedFolder: 100} } +func (p dedupSettingProvider) ThumbEntitySuffix(context.Context) string { + return "{blob_path}/{blob_name}._thumb" +} + func (p dedupSettingProvider) AuditLogEnabled(context.Context, int) bool { return true } diff --git a/pkg/filemanager/manager/thumbnail.go b/pkg/filemanager/manager/thumbnail.go index eb6c1c2a..ad578415 100644 --- a/pkg/filemanager/manager/thumbnail.go +++ b/pkg/filemanager/manager/thumbnail.go @@ -10,6 +10,7 @@ import ( "time" "github.com/cloudreve/Cloudreve/v4/ent" + "github.com/cloudreve/Cloudreve/v4/ent/storagepolicy" "github.com/cloudreve/Cloudreve/v4/ent/task" "github.com/cloudreve/Cloudreve/v4/inventory/types" "github.com/cloudreve/Cloudreve/v4/pkg/activity" @@ -59,15 +60,18 @@ func (m *manager) Thumbnail(ctx context.Context, uri *fs.URI) (entitysource.Enti } // 2. Thumb entity not exist, try native policy generator - _, handler, err := m.getEntityPolicyDriver(ctx, latest, nil) + policy, handler, err := m.getEntityPolicyDriver(ctx, latest, nil) if err != nil { return nil, fmt.Errorf("failed to get entity policy driver: %w", err) } capabilities := handler.Capabilities() + // thumb_force_proxy skips the native generator even when it supports + // this file — the local pipeline is mandatory. + forceProxy := policy.Settings != nil && policy.Settings.ThumbForceProxy // Check if file extension and size is supported by native policy generator. - if capabilities.ThumbSupportAllExts || util.IsInExtensionList(capabilities.ThumbSupportedExts, file.DisplayName()) && + if !forceProxy && (capabilities.ThumbSupportAllExts || util.IsInExtensionList(capabilities.ThumbSupportedExts, file.DisplayName()) && (capabilities.ThumbMaxSize == 0 || latest.Size() <= capabilities.ThumbMaxSize) && - !latest.Encrypted() { + !latest.Encrypted()) { thumbSource, err := m.GetEntitySource(ctx, 0, fs.WithEntity(latest), fs.WithUseThumb(true)) if err != nil { return nil, fmt.Errorf("failed to get latest entity source: %w", err) @@ -75,7 +79,7 @@ func (m *manager) Thumbnail(ctx context.Context, uri *fs.URI) (entitysource.Enti thumbSource.Apply(entitysource.WithDisplayName(file.DisplayName())) return thumbSource, nil - } else if capabilities.ThumbProxy { + } else if capabilities.ThumbProxy || forceProxy { if err := m.fs.CheckCapability(ctx, uri, dbfs.WithRequiredCapabilities(dbfs.NavigatorCapabilityGenerateThumb)); err != nil { // Current FS does not support generate new thumb. @@ -204,6 +208,18 @@ func (m *manager) generateThumb(ctx context.Context, uri *fs.URI, ext string, es Seeker: thumbFile, } + // The source policy may designate a dedicated thumbnail storage + // policy — the entity then ignores the file's own policy. + if srcPolicy, err := m.policyClient.GetPolicyByID(ctx, es.Entity().PolicyID()); err == nil && + srcPolicy.Settings != nil && srcPolicy.Settings.ThumbStoragePolicyID != 0 { + if dst, err := m.policyClient.GetPolicyByID(ctx, srcPolicy.Settings.ThumbStoragePolicyID); err == nil && + dst.Status == storagepolicy.StatusActive { + req.Props.PreferredStoragePolicy = dst.ID + } else { + m.l.Warning("designated thumbnail policy %d unavailable, storing with source policy", srcPolicy.Settings.ThumbStoragePolicyID) + } + } + // Generating thumb can be triggered by users with read-only permission. We can bypass update permission check. ctx = dbfs.WithBypassOwnerCheck(ctx)