From 712a3a5bb23b70d7f7325a04b80cc5f20a8f9d9a Mon Sep 17 00:00:00 2001 From: NGPixel Date: Tue, 22 Sep 2026 13:42:41 -0400 Subject: [PATCH] fix: truncated export for v3 --- server/core/backup.js | 46 +++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/server/core/backup.js b/server/core/backup.js index d8c347b62..438f52307 100644 --- a/server/core/backup.js +++ b/server/core/backup.js @@ -67,12 +67,15 @@ async function * batched (fetch, batchSize, onBatch) { let offset = 0 while (true) { const rows = await fetch(offset, batchSize) + // Stop only on an empty batch, and advance by what was actually consumed. + // Treating a short batch as the end would silently truncate a stream the + // moment a query returned fewer rows than it was asked for. + if (rows.length < 1) { break } for (const row of rows) { yield row } if (onBatch) { onBatch(rows.length) } - if (rows.length < batchSize) { break } - offset += batchSize + offset += rows.length } } @@ -209,12 +212,23 @@ module.exports = { status.progress = Math.min(100, ((doneSteps + Math.min(1, fraction)) / steps.length) * 100) } // Track a streamed step of `total` records, batch by batch. - const batchTracker = total => { + const batchTracker = (name, total) => { let done = 0 - return rows => { + const track = rows => { done += rows stepPartial(total > 0 ? done / total : 1) } + // The manifest's count is written before the stream and is what the + // importer uses as its denominator, so a stream that does not hold + // exactly that many records is worth saying out loud rather than + // handing over a quietly wrong package. Rows added or removed while a + // long export runs can account for a small difference. + track.verify = () => { + if (done !== total) { + WIKI.logger.warn(`Backup: ${name} stream holds ${done} records, but the manifest counted ${total}.`) + } + } + return track } // ----------------------------------------- @@ -332,17 +346,18 @@ module.exports = { // password or re-enrol an authenticator after the migration. if (has('users')) { WIKI.logger.info(`Backup: writing ${counts.users} users...`) - const onBatch = batchTracker(counts.users) + const onBatch = batchTracker('users', counts.users) await zip.addStream('streams/users.ndjson', ndjson(batched(async (offset, limit) => { const users = await WIKI.models.users.query() .orderBy('id').offset(offset).limit(limit) - .withGraphJoined({ groups: true }) + .withGraphFetched({ groups: true }) .modifyGraph('groups', builder => builder.select('groups.id', 'groups.name')) return users.map(usr => ({ ..._.omit(usr, ['groups']), groups: usr.groups.map(g => g.id) })) }, BATCH_SIZE.users, onBatch))) + onBatch.verify() stepDone() } @@ -397,13 +412,14 @@ module.exports = { // ----------------------------------------- if (has('pages')) { WIKI.logger.info(`Backup: writing ${counts.tree} folders...`) - const onBatch = batchTracker(counts.tree) + const onBatch = batchTracker('tree', counts.tree) await zip.addStream(`${sitePath}/tree.ndjson`, ndjson(batched(async (offset, limit) => { return WIKI.models.knex('pageTree') .select('id', 'path', 'depth', 'title', 'isPrivate', 'privateNS', 'parent', 'localeCode') .where('isFolder', true) .orderBy('id').offset(offset).limit(limit) }, BATCH_SIZE.tree, onBatch))) + onBatch.verify() stepDone() // ----------------------------------------- @@ -414,17 +430,18 @@ module.exports = { // stored render says nothing about which pipeline produced it. 3.x // produces the HTML itself. WIKI.logger.info(`Backup: writing ${counts.pages} pages...`) - const onPageBatch = batchTracker(counts.pages) + const onPageBatch = batchTracker('pages', counts.pages) await zip.addStream(`${sitePath}/pages.ndjson`, ndjson(batched(async (offset, limit) => { const pages = await WIKI.models.pages.query() .orderBy('id').offset(offset).limit(limit) - .withGraphJoined({ tags: true }) + .withGraphFetched({ tags: true }) .modifyGraph('tags', builder => builder.select('tags.tag', 'tags.title')) return Promise.all(pages.map(page => this.spillContent({ ..._.omit(page, ['render', 'toc']), tags: page.tags.map(t => t.tag) }, tmpPath, blobs))) }, BATCH_SIZE.pages, onPageBatch))) + onPageBatch.verify() stepDone() } @@ -433,17 +450,18 @@ module.exports = { // ----------------------------------------- if (has('history')) { WIKI.logger.info(`Backup: writing ${counts.history} page history entries...`) - const onBatch = batchTracker(counts.history) + const onBatch = batchTracker('history', counts.history) await zip.addStream(`${sitePath}/page-history.ndjson`, ndjson(batched(async (offset, limit) => { const versions = await WIKI.models.pageHistory.query() .orderBy('id').offset(offset).limit(limit) - .withGraphJoined({ tags: true }) + .withGraphFetched({ tags: true }) .modifyGraph('tags', builder => builder.select('tags.tag', 'tags.title')) return Promise.all(versions.map(version => this.spillContent({ ...version, tags: version.tags.map(t => t.tag) }, tmpPath, blobs))) }, BATCH_SIZE.history, onBatch))) + onBatch.verify() stepDone() } @@ -452,10 +470,11 @@ module.exports = { // ----------------------------------------- if (has('comments')) { WIKI.logger.info(`Backup: writing ${counts.comments} comments...`) - const onBatch = batchTracker(counts.comments) + const onBatch = batchTracker('comments', counts.comments) await zip.addStream(`${sitePath}/comments.ndjson`, ndjson(batched(async (offset, limit) => { return WIKI.models.comments.query().orderBy('id').offset(offset).limit(limit) }, BATCH_SIZE.comments, onBatch))) + onBatch.verify() stepDone() } @@ -466,7 +485,7 @@ module.exports = { if (has('assets')) { WIKI.logger.info(`Backup: writing ${counts.assets} assets...`) const assetFolders = await WIKI.models.assetFolders.getAllPaths() - const onBatch = batchTracker(counts.assets) + const onBatch = batchTracker('assets', counts.assets) await zip.addStream(`${sitePath}/assets.ndjson`, ndjson(batched(async (offset, limit) => { const assets = await WIKI.models.knex .select('assets.*', 'assetData.data') @@ -486,6 +505,7 @@ module.exports = { } }) }, BATCH_SIZE.assets, onBatch))) + onBatch.verify() stepDone() }