feat(admin): per-user blob relocation scope

RelocateEntityService gains a third mutually-exclusive scope,
src_user_id, alongside entity_ids and src_policy_id. The new
NewRelocateUserTask selects entities via created_by and reuses the
existing cursor-resumable transfer path. The admin user editor gains
a "Relocate files" button opening the relocate dialog prefilled with
the user scope.

Generated with Devin
pull/3589/head
Tomas Dvorak 2 weeks ago
parent f23f5efc44
commit 291709ceda

@ -221,6 +221,7 @@ Order = user-visible value first; each ships with backend + UI + tests.
- [x] Storage policy total capacity (upstream #2178 item 1) — `PolicySetting.MaxTotalSize` caps cumulative entity bytes per policy; enforced in `PrepareUpload`, batch upload validation, and `copyFiles` (baseline usage + per-batch accumulation since tx writes are invisible to the usage query); canonical `ErrInsufficientCapacity` preserved so `errors.Is` quota handling still matches; admin policy editor gains a Max total capacity SizeInput, en+zh locales - [x] Storage policy total capacity (upstream #2178 item 1) — `PolicySetting.MaxTotalSize` caps cumulative entity bytes per policy; enforced in `PrepareUpload`, batch upload validation, and `copyFiles` (baseline usage + per-batch accumulation since tx writes are invisible to the usage query); canonical `ErrInsufficientCapacity` preserved so `errors.Is` quota handling still matches; admin policy editor gains a Max total capacity SizeInput, en+zh locales
- [x] Storage policy overflow chain (upstream #2178 item 4) — `PolicySetting.OverflowPolicyID` links a fallback policy; `overflowChain` walks hops with cycle guard + hop cap, skipping suspended members and resolving load-balance members to weighted children; `PrepareUpload` spills to the first member with headroom for the file size so name/size/extension rules apply to the landing policy; `PreValidateUpload` checks aggregate chain headroom since batches may split across members; admin policy editor gains an Overflow policy select, en+zh locales - [x] Storage policy overflow chain (upstream #2178 item 4) — `PolicySetting.OverflowPolicyID` links a fallback policy; `overflowChain` walks hops with cycle guard + hop cap, skipping suspended members and resolving load-balance members to weighted children; `PrepareUpload` spills to the first member with headroom for the file size so name/size/extension rules apply to the landing policy; `PreValidateUpload` checks aggregate chain headroom since batches may split across members; admin policy editor gains an Overflow policy select, en+zh locales
- [x] Thumbnail generation controls (upstream #2178 items 7+8) — `PolicySetting.ThumbForceProxy` skips the backend's native thumbnail API even when supported, implying the local proxy pipeline; `PolicySetting.ThumbStoragePolicyID` redirects generated thumb entities to a designated policy (honored via `PreferredStoragePolicy` for thumbnail uploads in `PrepareUpload`); Thumbnails section gains both controls, en+zh locales - [x] Thumbnail generation controls (upstream #2178 items 7+8) — `PolicySetting.ThumbForceProxy` skips the backend's native thumbnail API even when supported, implying the local proxy pipeline; `PolicySetting.ThumbStoragePolicyID` redirects generated thumb entities to a designated policy (honored via `PreferredStoragePolicy` for thumbnail uploads in `PrepareUpload`); Thumbnails section gains both controls, en+zh locales
- [x] Per-user blob relocation (upstream #2729/misc) — `RelocateEntityService` gains a third scope `src_user_id` (mutually exclusive with `entity_ids`/`src_policy_id`); `NewRelocateUserTask` selects entities by `created_by` with the same cursor-resumable transfer path; admin user editor gains a "Relocate files" button opening the relocate dialog prefilled with the user scope; en+zh locales
## 6. Phase D — desktop, all platforms ## 6. Phase D — desktop, all platforms

@ -1429,6 +1429,7 @@
"groupExpired": "Group expired date", "groupExpired": "Group expired date",
"groupExpiredDes": "ISO8601 format group expired date, leave blank means the group is permanent.", "groupExpiredDes": "ISO8601 format group expired date, leave blank means the group is permanent.",
"openUserFiles": "Open user files", "openUserFiles": "Open user files",
"relocateFiles": "Relocate files",
"id": "ID", "id": "ID",
"idValue": "{{id}} ({{hash_id}})", "idValue": "{{id}} ({{hash_id}})",
"avatar": "Profile picture", "avatar": "Profile picture",
@ -1587,6 +1588,7 @@
"relocateTitle": "Relocate Blobs", "relocateTitle": "Relocate Blobs",
"relocateEntitiesDes": "Move the data of {{num}} selected Blobs to another storage policy. The task runs in the background and can be resumed.", "relocateEntitiesDes": "Move the data of {{num}} selected Blobs to another storage policy. The task runs in the background and can be resumed.",
"relocatePolicyDes": "Move all Blobs stored on \"{{name}}\" to another storage policy. The task runs in the background and can be resumed.", "relocatePolicyDes": "Move all Blobs stored on \"{{name}}\" to another storage policy. The task runs in the background and can be resumed.",
"relocateUserDes": "Move all Blobs owned by \"{{name}}\" to another storage policy. The task runs in the background and can be resumed.",
"relocatePolicy": "Migrate all Blobs on {{name}} to another policy", "relocatePolicy": "Migrate all Blobs on {{name}} to another policy",
"relocateStart": "Start relocation", "relocateStart": "Start relocation",
"relocateSubmitted": "Relocation task submitted" "relocateSubmitted": "Relocation task submitted"

@ -1429,6 +1429,7 @@
"groupExpired": "用户组过期日期", "groupExpired": "用户组过期日期",
"groupExpiredDes": "ISO8601 格式的用户组到期日期,留空表示当前用户组永久有效。", "groupExpiredDes": "ISO8601 格式的用户组到期日期,留空表示当前用户组永久有效。",
"openUserFiles": "打开用户文件", "openUserFiles": "打开用户文件",
"relocateFiles": "迁移文件存储策略",
"id": "ID", "id": "ID",
"idValue": "{{id}} ({{hash_id}})", "idValue": "{{id}} ({{hash_id}})",
"avatar": "头像", "avatar": "头像",
@ -1587,6 +1588,7 @@
"relocateTitle": "迁移 Blob", "relocateTitle": "迁移 Blob",
"relocateEntitiesDes": "将选中的 {{num}} 个 Blob 的数据迁移到另一个存储策略。任务将在后台运行,可断点续传。", "relocateEntitiesDes": "将选中的 {{num}} 个 Blob 的数据迁移到另一个存储策略。任务将在后台运行,可断点续传。",
"relocatePolicyDes": "将存储在“{{name}}”上的所有 Blob 迁移到另一个存储策略。任务将在后台运行,可断点续传。", "relocatePolicyDes": "将存储在“{{name}}”上的所有 Blob 迁移到另一个存储策略。任务将在后台运行,可断点续传。",
"relocateUserDes": "将“{{name}}”名下的所有 Blob 迁移到另一个存储策略。任务将在后台运行,可断点续传。",
"relocatePolicy": "将 {{name}} 上的所有 Blob 迁移到其他策略", "relocatePolicy": "将 {{name}} 上的所有 Blob 迁移到其他策略",
"relocateStart": "开始迁移", "relocateStart": "开始迁移",
"relocateSubmitted": "迁移任务已提交" "relocateSubmitted": "迁移任务已提交"

@ -189,5 +189,6 @@ export interface BlobAuditWorkflowService {
export interface RelocateEntityService { export interface RelocateEntityService {
entity_ids?: number[]; entity_ids?: number[];
src_policy_id?: number; src_policy_id?: number;
src_user_id?: number;
dst_policy_id: number; dst_policy_id: number;
} }

@ -13,9 +13,11 @@ export interface RelocateDialogProps {
entityIDs?: number[]; entityIDs?: number[];
srcPolicyID?: number; srcPolicyID?: number;
srcPolicyName?: string; srcPolicyName?: string;
srcUserID?: number;
srcUserName?: string;
} }
const RelocateDialog = ({ open, onClose, entityIDs, srcPolicyID, srcPolicyName }: RelocateDialogProps) => { const RelocateDialog = ({ open, onClose, entityIDs, srcPolicyID, srcPolicyName, srcUserID, srcUserName }: RelocateDialogProps) => {
const { t } = useTranslation("dashboard"); const { t } = useTranslation("dashboard");
const dispatch = useAppDispatch(); const dispatch = useAppDispatch();
const { enqueueSnackbar } = useSnackbar(); const { enqueueSnackbar } = useSnackbar();
@ -37,6 +39,7 @@ const RelocateDialog = ({ open, onClose, entityIDs, srcPolicyID, srcPolicyName }
relocateEntities({ relocateEntities({
entity_ids: entityIDs, entity_ids: entityIDs,
src_policy_id: srcPolicyID, src_policy_id: srcPolicyID,
src_user_id: srcUserID,
dst_policy_id: dstPolicyID, dst_policy_id: dstPolicyID,
}), }),
) )
@ -56,7 +59,9 @@ const RelocateDialog = ({ open, onClose, entityIDs, srcPolicyID, srcPolicyName }
<DialogContentText sx={{ mb: 2 }}> <DialogContentText sx={{ mb: 2 }}>
{srcPolicyID {srcPolicyID
? t("entity.relocatePolicyDes", { name: srcPolicyName ?? "" }) ? t("entity.relocatePolicyDes", { name: srcPolicyName ?? "" })
: t("entity.relocateEntitiesDes", { num: entityIDs?.length ?? 0 })} : srcUserID
? t("entity.relocateUserDes", { name: srcUserName ?? "" })
: t("entity.relocateEntitiesDes", { num: entityIDs?.length ?? 0 })}
</DialogContentText> </DialogContentText>
<SinglePolicySelectionInput value={dstPolicyID} onChange={setDstPolicyID} /> <SinglePolicySelectionInput value={dstPolicyID} onChange={setDstPolicyID} />
</DialogContent> </DialogContent>

@ -14,7 +14,7 @@ import {
useTheme, useTheme,
} from "@mui/material"; } from "@mui/material";
import { useSnackbar } from "notistack"; import { useSnackbar } from "notistack";
import { useCallback, useContext } from "react"; import { useCallback, useContext, useState } from "react";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
import { useNavigate } from "react-router-dom"; import { useNavigate } from "react-router-dom";
import { sendCalibrateUserStorage } from "../../../../api/api"; import { sendCalibrateUserStorage } from "../../../../api/api";
@ -27,6 +27,7 @@ import { SquareMenuItem } from "../../../FileManager/ContextMenu/ContextMenu";
import Delete from "../../../Icons/Delete"; import Delete from "../../../Icons/Delete";
import SettingForm from "../../../Pages/Setting/SettingForm"; import SettingForm from "../../../Pages/Setting/SettingForm";
import { CapacityBar } from "../../../Pages/Setting/StorageSetting"; import { CapacityBar } from "../../../Pages/Setting/StorageSetting";
import RelocateDialog from "../../Entity/RelocateDialog";
import GroupSelectionInput from "../../Common/GroupSelectionInput"; import GroupSelectionInput from "../../Common/GroupSelectionInput";
import { NoMarginHelperText } from "../../Settings/Settings"; import { NoMarginHelperText } from "../../Settings/Settings";
import { UserDialogContext } from "./UserDialog"; import { UserDialogContext } from "./UserDialog";
@ -39,6 +40,7 @@ const UserForm = ({ reload, setLoading }: { reload: () => void; setLoading: (loa
const { t } = useTranslation("dashboard"); const { t } = useTranslation("dashboard");
const navigate = useNavigate(); const navigate = useNavigate();
const { formRef, values, setUser } = useContext(UserDialogContext); const { formRef, values, setUser } = useContext(UserDialogContext);
const [relocateOpen, setRelocateOpen] = useState(false);
const removeAvatar = useCallback(() => { const removeAvatar = useCallback(() => {
setUser((prev) => ({ ...prev, avatar: undefined })); setUser((prev) => ({ ...prev, avatar: undefined }));
@ -179,6 +181,15 @@ const UserForm = ({ reload, setLoading }: { reload: () => void; setLoading: (loa
> >
{t("user.openUserFiles")} {t("user.openUserFiles")}
</SecondaryButton> </SecondaryButton>
<SecondaryButton sx={{ mt: 1 }} onClick={() => setRelocateOpen(true)} variant="contained">
{t("user.relocateFiles")}
</SecondaryButton>
<RelocateDialog
open={relocateOpen}
onClose={() => setRelocateOpen(false)}
srcUserID={values.id}
srcUserName={values.email}
/>
</Box> </Box>
</Stack> </Stack>
<Divider orientation="vertical" flexItem /> <Divider orientation="vertical" flexItem />

@ -41,6 +41,9 @@ type (
// SrcPolicyID relocates every entity still on this policy instead of an // SrcPolicyID relocates every entity still on this policy instead of an
// explicit list. // explicit list.
SrcPolicyID int `json:"src_policy_id,omitempty"` SrcPolicyID int `json:"src_policy_id,omitempty"`
// SrcUserID relocates every entity created by this user instead of an
// explicit list.
SrcUserID int `json:"src_user_id,omitempty"`
DstPolicyID int `json:"dst_policy_id"` DstPolicyID int `json:"dst_policy_id"`
// Cursor is the ID of the last successfully processed entity. // Cursor is the ID of the last successfully processed entity.
Cursor int `json:"cursor,omitempty"` Cursor int `json:"cursor,omitempty"`
@ -91,6 +94,16 @@ func NewRelocatePolicyTask(ctx context.Context, srcPolicyID, dstPolicyID int) (q
}) })
} }
// NewRelocateUserTask creates a RelocateTask moving every entity created by
// srcUserID to dstPolicyID.
func NewRelocateUserTask(ctx context.Context, srcUserID, dstPolicyID int) (queue.Task, error) {
return newRelocateTask(ctx, &RelocateTaskState{
SrcUserID: srcUserID,
DstPolicyID: dstPolicyID,
NodeState: NodeState{},
})
}
func newRelocateTask(ctx context.Context, state *RelocateTaskState) (queue.Task, error) { func newRelocateTask(ctx context.Context, state *RelocateTaskState) (queue.Task, error) {
stateBytes, err := json.Marshal(state) stateBytes, err := json.Marshal(state)
if err != nil { if err != nil {
@ -230,6 +243,19 @@ func (m *RelocateTask) pendingEntities(ctx context.Context, dep dependency.Dep)
All(ctx) All(ctx)
} }
if m.state.SrcUserID != 0 {
return dep.DBClient().Entity.Query().
Where(
entity.CreatedBy(m.state.SrcUserID),
entity.IDGT(m.state.Cursor),
).
Order(ent.Asc(entity.FieldID)).
Limit(relocatePageSize).
WithFile().
WithUser().
All(ctx)
}
if m.state.SrcPolicyID == 0 { if m.state.SrcPolicyID == 0 {
return nil, nil return nil, nil
} }
@ -251,6 +277,8 @@ func (m *RelocateTask) progressTotals(ctx context.Context, dep dependency.Dep) (
var q *ent.EntityQuery var q *ent.EntityQuery
if len(m.state.EntityIDs) > 0 { if len(m.state.EntityIDs) > 0 {
q = dep.DBClient().Entity.Query().Where(entity.IDIn(m.state.EntityIDs...)) q = dep.DBClient().Entity.Query().Where(entity.IDIn(m.state.EntityIDs...))
} else if m.state.SrcUserID != 0 {
q = dep.DBClient().Entity.Query().Where(entity.CreatedBy(m.state.SrcUserID))
} else if m.state.SrcPolicyID != 0 { } else if m.state.SrcPolicyID != 0 {
q = dep.DBClient().Entity.Query().Where(entity.StoragePolicyEntities(m.state.SrcPolicyID)) q = dep.DBClient().Entity.Query().Where(entity.StoragePolicyEntities(m.state.SrcPolicyID))
} else { } else {

@ -142,6 +142,45 @@ func TestRelocatePendingPolicyScope(t *testing.T) {
_ = moved _ = moved
} }
func TestRelocatePendingUserScope(t *testing.T) {
client := enttest.Open(t, "sqlite3", "file:"+t.Name()+"?mode=memory&cache=shared")
t.Cleanup(func() { require.NoError(t, client.Close()) })
ctx, dep := newRelocateTestDep(t, client)
src := client.StoragePolicy.Create().SetName("src").SetType("local").
SetStatus(storagepolicy.StatusActive).SaveX(ctx)
dst := client.StoragePolicy.Create().SetName("dst").SetType("local").
SetStatus(storagepolicy.StatusActive).SaveX(ctx)
group := client.Group.Create().SetName("g").SetPermissions(&boolset.BooleanSet{}).SaveX(ctx)
u7 := client.User.Create().SetEmail("a@example.com").SetNick("a").SetGroup(group).SaveX(ctx)
u9 := client.User.Create().SetEmail("b@example.com").SetNick("b").SetGroup(group).SaveX(ctx)
mine := client.Entity.Create().SetType(0).SetSource("a").SetSize(1).
SetStoragePolicyEntities(src.ID).SetCreatedBy(u7.ID).SaveX(ctx)
mine2 := client.Entity.Create().SetType(0).SetSource("b").SetSize(1).
SetStoragePolicyEntities(src.ID).SetCreatedBy(u7.ID).SaveX(ctx)
client.Entity.Create().SetType(0).SetSource("theirs").SetSize(1).
SetStoragePolicyEntities(src.ID).SetCreatedBy(u9.ID).SaveX(ctx)
m := relocateTaskWithState(t, &RelocateTaskState{SrcUserID: u7.ID, DstPolicyID: dst.ID})
m.state = &RelocateTaskState{SrcUserID: u7.ID, DstPolicyID: dst.ID}
pending, err := m.pendingEntities(ctx, dep)
require.NoError(t, err)
require.Len(t, pending, 2)
m.state.Cursor = mine.ID
pending, err = m.pendingEntities(ctx, dep)
require.NoError(t, err)
require.Len(t, pending, 1)
require.Equal(t, mine2.ID, pending[0].ID)
count, size := m.progressTotals(ctx, dep)
require.Equal(t, int64(2), count)
require.Equal(t, int64(2), size)
}
func TestRelocateProgressTotals(t *testing.T) { func TestRelocateProgressTotals(t *testing.T) {
client := enttest.Open(t, "sqlite3", "file:"+t.Name()+"?mode=memory&cache=shared") client := enttest.Open(t, "sqlite3", "file:"+t.Name()+"?mode=memory&cache=shared")
t.Cleanup(func() { require.NoError(t, client.Close()) }) t.Cleanup(func() { require.NoError(t, client.Close()) })

@ -542,11 +542,12 @@ func (s *BatchEntityService) Delete(c *gin.Context) error {
type ( type (
// RelocateEntityService moves blobs between storage policies. Exactly one // RelocateEntityService moves blobs between storage policies. Exactly one
// scope is required: explicit entity IDs, or a source policy whose entities // scope is required: explicit entity IDs, a source policy whose entities
// are all migrated. // are all migrated, or a source user whose entities are all migrated.
RelocateEntityService struct { RelocateEntityService struct {
EntityIDs []int `json:"entity_ids"` EntityIDs []int `json:"entity_ids"`
SrcPolicyID int `json:"src_policy_id"` SrcPolicyID int `json:"src_policy_id"`
SrcUserID int `json:"src_user_id"`
DstPolicyID int `json:"dst_policy_id" binding:"required"` DstPolicyID int `json:"dst_policy_id" binding:"required"`
} }
RelocateEntityParamCtx struct{} RelocateEntityParamCtx struct{}
@ -556,11 +557,17 @@ func (s *RelocateEntityService) Relocate(c *gin.Context) (*RelocateTaskResponse,
dep := dependency.FromContext(c) dep := dependency.FromContext(c)
hasher := dep.HashIDEncoder() hasher := dep.HashIDEncoder()
if len(s.EntityIDs) == 0 && s.SrcPolicyID == 0 { scopes := 0
return nil, serializer.NewError(serializer.CodeParamErr, "either entity_ids or src_policy_id is required", nil) for _, set := range []bool{len(s.EntityIDs) > 0, s.SrcPolicyID != 0, s.SrcUserID != 0} {
if set {
scopes++
}
}
if scopes == 0 {
return nil, serializer.NewError(serializer.CodeParamErr, "one of entity_ids, src_policy_id or src_user_id is required", nil)
} }
if len(s.EntityIDs) > 0 && s.SrcPolicyID != 0 { if scopes > 1 {
return nil, serializer.NewError(serializer.CodeParamErr, "entity_ids and src_policy_id are mutually exclusive", nil) return nil, serializer.NewError(serializer.CodeParamErr, "entity_ids, src_policy_id and src_user_id are mutually exclusive", nil)
} }
if s.SrcPolicyID == s.DstPolicyID && s.SrcPolicyID != 0 { if s.SrcPolicyID == s.DstPolicyID && s.SrcPolicyID != 0 {
return nil, serializer.NewError(serializer.CodeParamErr, "source and destination policies are identical", nil) return nil, serializer.NewError(serializer.CodeParamErr, "source and destination policies are identical", nil)
@ -572,13 +579,20 @@ func (s *RelocateEntityService) Relocate(c *gin.Context) (*RelocateTaskResponse,
} }
var t queue.Task var t queue.Task
if s.SrcPolicyID != 0 { switch {
case s.SrcPolicyID != 0:
srcPolicy, err := dep.StoragePolicyClient().GetPolicyByID(c, s.SrcPolicyID) srcPolicy, err := dep.StoragePolicyClient().GetPolicyByID(c, s.SrcPolicyID)
if err != nil || srcPolicy == nil { if err != nil || srcPolicy == nil {
return nil, serializer.NewError(serializer.CodeParamErr, "source policy does not exist", err) return nil, serializer.NewError(serializer.CodeParamErr, "source policy does not exist", err)
} }
t, err = workflows.NewRelocatePolicyTask(c, s.SrcPolicyID, s.DstPolicyID) t, err = workflows.NewRelocatePolicyTask(c, s.SrcPolicyID, s.DstPolicyID)
} else { case s.SrcUserID != 0:
srcUser, err := dep.UserClient().GetByID(c, s.SrcUserID)
if err != nil || srcUser == nil {
return nil, serializer.NewError(serializer.CodeParamErr, "source user does not exist", err)
}
t, err = workflows.NewRelocateUserTask(c, s.SrcUserID, s.DstPolicyID)
default:
t, err = workflows.NewRelocateTask(c, s.EntityIDs, s.DstPolicyID) t, err = workflows.NewRelocateTask(c, s.EntityIDs, s.DstPolicyID)
} }
if err != nil { if err != nil {

Loading…
Cancel
Save