mirror of https://github.com/requarks/wiki
NGPixel
1 week ago
parent
1161ddc6e6
commit
3b28ac18e5
@ -0,0 +1,312 @@
|
||||
<template lang='pug'>
|
||||
v-card
|
||||
v-toolbar(flat, color='primary', dark, dense)
|
||||
.subtitle-1 Export for Wiki.js 3.x
|
||||
v-card-text
|
||||
.text-center
|
||||
img.animated.fadeInUp.wait-p1s(src='/_assets/svg/icon-big-parcel.svg')
|
||||
.body-2 Export to a .wkbackup migration package
|
||||
v-divider.my-4
|
||||
.body-2 What do you want to include?
|
||||
v-checkbox(
|
||||
v-for='choice of entityChoices'
|
||||
:class='choice.requires ? `ml-6` : ``'
|
||||
:key='choice.key'
|
||||
:label='choice.label'
|
||||
:value='choice.key'
|
||||
:disabled='choice.requires && !entities.includes(choice.requires)'
|
||||
color='deep-orange darken-2'
|
||||
hide-details
|
||||
v-model='entities'
|
||||
)
|
||||
template(v-slot:label)
|
||||
div
|
||||
strong.deep-orange--text.text--darken-2 {{choice.label}}
|
||||
.text-caption {{choice.hint}}
|
||||
|
||||
v-alert.mt-7(color='blue-grey darken-2', outlined, icon='mdi-information-outline', prominent)
|
||||
.body-2 The package is written to the #[strong backup] folder of your Wiki.js data path, and offered as a download once it's ready.
|
||||
.body-2 Import it from Wiki.js 3.x using #[strong Administration → Utilities → Import from Wiki.js 2.x]. The file is read by your browser, so it's never uploaded anywhere.
|
||||
|
||||
v-alert.mt-3(color='deep-orange', outlined, icon='mdi-alert', prominent)
|
||||
.body-2 The package carries hashed user passwords, 2FA secrets and site configuration keys, and it is neither encrypted nor signed. Treat it as you would the database itself.
|
||||
.body-2 Carrying those credentials is what lets everyone keep their password and their authenticator app after the migration.
|
||||
|
||||
v-card-chin
|
||||
v-btn.px-3(depressed, color='deep-orange darken-2', :disabled='entities.length < 1', @click='startBackup').ml-0
|
||||
v-icon(left, color='white') mdi-package-variant-closed
|
||||
span.white--text Create Package
|
||||
v-dialog(
|
||||
v-model='isLoading'
|
||||
persistent
|
||||
max-width='350'
|
||||
)
|
||||
v-card(color='deep-orange darken-2', dark)
|
||||
v-card-text.pa-10.text-center
|
||||
self-building-square-spinner.animated.fadeIn(
|
||||
:animation-duration='4500'
|
||||
:size='40'
|
||||
color='#FFF'
|
||||
style='margin: 0 auto;'
|
||||
)
|
||||
.mt-5.body-1.white--text Creating package...
|
||||
.caption Please wait, this may take a while
|
||||
v-progress-linear.mt-5(
|
||||
color='white'
|
||||
:value='progress'
|
||||
stream
|
||||
rounded
|
||||
:buffer-value='0'
|
||||
)
|
||||
v-dialog(
|
||||
v-model='isSuccess'
|
||||
persistent
|
||||
max-width='450'
|
||||
)
|
||||
v-card(color='green darken-2', dark)
|
||||
v-card-text.pa-10.text-center
|
||||
v-icon(size='60') mdi-check-circle-outline
|
||||
.mt-5.body-1.white--text Package created
|
||||
.mt-2.caption.white--text {{filename}}
|
||||
.caption(v-if='fileSize') {{prettyFileSize}}
|
||||
.mt-3.caption.text-truncate(v-if='filePath', :title='filePath') {{filePath}}
|
||||
v-card-actions.green.darken-1
|
||||
v-spacer
|
||||
v-btn.px-5(
|
||||
color='white'
|
||||
outlined
|
||||
@click='downloadBackup'
|
||||
)
|
||||
v-icon(left) mdi-download
|
||||
span Download
|
||||
v-btn.px-5(
|
||||
color='white'
|
||||
text
|
||||
@click='isSuccess = false'
|
||||
) Close
|
||||
v-spacer
|
||||
v-dialog(
|
||||
v-model='isFailed'
|
||||
persistent
|
||||
max-width='800'
|
||||
)
|
||||
v-card(color='red darken-2', dark)
|
||||
v-toolbar(color='red darken-2', dense)
|
||||
v-icon mdi-alert
|
||||
.body-2.pl-3 Package creation failed
|
||||
v-spacer
|
||||
v-btn.px-5(
|
||||
color='white'
|
||||
text
|
||||
@click='isFailed = false'
|
||||
) Close
|
||||
v-card-text.pa-5.red.darken-4.white--text
|
||||
span {{errorMessage}}
|
||||
</template>
|
||||
|
||||
<script>
|
||||
import { SelfBuildingSquareSpinner } from 'epic-spinners'
|
||||
|
||||
import gql from 'graphql-tag'
|
||||
import _get from 'lodash/get'
|
||||
|
||||
export default {
|
||||
components: {
|
||||
SelfBuildingSquareSpinner
|
||||
},
|
||||
data() {
|
||||
return {
|
||||
entities: ['assets', 'navigation', 'pages', 'history', 'comments', 'settings', 'groups', 'users'],
|
||||
isLoading: false,
|
||||
isSuccess: false,
|
||||
isFailed: false,
|
||||
errorMessage: '',
|
||||
filename: '',
|
||||
filePath: '',
|
||||
fileSize: 0,
|
||||
progress: 0
|
||||
}
|
||||
},
|
||||
computed: {
|
||||
entityChoices () {
|
||||
return [
|
||||
{
|
||||
key: 'assets',
|
||||
label: 'Assets',
|
||||
hint: 'Media files such as images, documents, etc.'
|
||||
},
|
||||
{
|
||||
key: 'navigation',
|
||||
label: 'Navigation',
|
||||
hint: 'Sidebar links when using Static or Custom Navigation.'
|
||||
},
|
||||
{
|
||||
key: 'pages',
|
||||
label: 'Pages',
|
||||
hint: 'Page content, tags and related metadata. Rendered HTML is not included, as Wiki.js 3.x renders pages itself.'
|
||||
},
|
||||
// -> Neither of these makes sense without the pages they belong to.
|
||||
{
|
||||
key: 'history',
|
||||
label: 'Pages History',
|
||||
hint: 'All previous versions of pages and their related metadata.',
|
||||
requires: 'pages'
|
||||
},
|
||||
{
|
||||
key: 'comments',
|
||||
label: 'Comments',
|
||||
hint: 'Comments made using the default comment module only.',
|
||||
requires: 'pages'
|
||||
},
|
||||
{
|
||||
key: 'settings',
|
||||
label: 'Settings',
|
||||
hint: 'Site configuration and modules settings.'
|
||||
},
|
||||
{
|
||||
key: 'groups',
|
||||
label: 'User Groups',
|
||||
hint: 'Group permissions and page rules.'
|
||||
},
|
||||
{
|
||||
key: 'users',
|
||||
label: 'Users',
|
||||
hint: 'Users metadata, group memberships, password hashes and 2FA secrets.'
|
||||
}
|
||||
]
|
||||
},
|
||||
prettyFileSize () {
|
||||
const units = ['bytes', 'KB', 'MB', 'GB', 'TB']
|
||||
let size = this.fileSize
|
||||
let unit = 0
|
||||
while (size >= 1024 && unit < units.length - 1) {
|
||||
size /= 1024
|
||||
unit++
|
||||
}
|
||||
return `${unit === 0 ? size : size.toFixed(2)} ${units[unit]}`
|
||||
}
|
||||
},
|
||||
watch: {
|
||||
entities (newEntities) {
|
||||
const orphaned = this.entityChoices
|
||||
.filter(choice => choice.requires && !newEntities.includes(choice.requires))
|
||||
.map(choice => choice.key)
|
||||
if (orphaned.some(key => newEntities.includes(key))) {
|
||||
this.entities = newEntities.filter(key => !orphaned.includes(key))
|
||||
}
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
downloadBackup () {
|
||||
window.location.assign(`/_backup/${this.filename}`)
|
||||
},
|
||||
async checkProgress () {
|
||||
try {
|
||||
const respStatus = await this.$apollo.query({
|
||||
query: gql`
|
||||
{
|
||||
system {
|
||||
backupStatus {
|
||||
status
|
||||
progress
|
||||
message
|
||||
startedAt
|
||||
filename
|
||||
filePath
|
||||
fileSize
|
||||
}
|
||||
}
|
||||
}
|
||||
`,
|
||||
fetchPolicy: 'network-only'
|
||||
})
|
||||
const respStatusObj = _get(respStatus, 'data.system.backupStatus', {})
|
||||
if (!respStatusObj) {
|
||||
throw new Error('An unexpected error occured.')
|
||||
} else {
|
||||
switch (respStatusObj.status) {
|
||||
case 'error': {
|
||||
throw new Error(respStatusObj.message || 'An unexpected error occured.')
|
||||
}
|
||||
case 'running': {
|
||||
this.progress = respStatusObj.progress || 0
|
||||
window.requestAnimationFrame(() => {
|
||||
setTimeout(() => {
|
||||
this.checkProgress()
|
||||
}, 5000)
|
||||
})
|
||||
break
|
||||
}
|
||||
case 'success': {
|
||||
this.filename = respStatusObj.filename
|
||||
this.filePath = respStatusObj.filePath || ''
|
||||
this.fileSize = respStatusObj.fileSize || 0
|
||||
this.isLoading = false
|
||||
this.isSuccess = true
|
||||
break
|
||||
}
|
||||
default: {
|
||||
throw new Error('Invalid backup status.')
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
this.errorMessage = err.message
|
||||
this.isLoading = false
|
||||
this.isFailed = true
|
||||
}
|
||||
},
|
||||
async startBackup () {
|
||||
this.isFailed = false
|
||||
this.isSuccess = false
|
||||
this.isLoading = true
|
||||
this.progress = 0
|
||||
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
// -> Initiate backup
|
||||
const respBackup = await this.$apollo.mutate({
|
||||
mutation: gql`
|
||||
mutation (
|
||||
$entities: [String]!
|
||||
) {
|
||||
system {
|
||||
createBackup (
|
||||
entities: $entities
|
||||
) {
|
||||
responseResult {
|
||||
succeeded
|
||||
message
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`,
|
||||
variables: {
|
||||
entities: this.entities
|
||||
}
|
||||
})
|
||||
|
||||
const respBackupObj = _get(respBackup, 'data.system.createBackup', {})
|
||||
if (!_get(respBackupObj, 'responseResult.succeeded', false)) {
|
||||
this.errorMessage = _get(respBackupObj, 'responseResult.message', 'An unexpected error occurred')
|
||||
this.isLoading = false
|
||||
this.isFailed = true
|
||||
return
|
||||
}
|
||||
|
||||
// -> Check for progress
|
||||
this.checkProgress()
|
||||
} catch (err) {
|
||||
this.$store.commit('pushGraphError', err)
|
||||
this.isLoading = false
|
||||
}
|
||||
}, 1500)
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style lang='scss'>
|
||||
|
||||
</style>
|
||||
@ -0,0 +1,614 @@
|
||||
const _ = require('lodash')
|
||||
const fs = require('fs-extra')
|
||||
const path = require('path')
|
||||
const crypto = require('crypto')
|
||||
const { Readable } = require('node:stream')
|
||||
const { ZipWriter } = require('../helpers/zip')
|
||||
|
||||
/* global WIKI */
|
||||
|
||||
/**
|
||||
* .wkbackup package writer, for migrating a 2.x instance to 3.x.
|
||||
*
|
||||
* The package is a ZIP holding a manifest, a set of NDJSON record streams and
|
||||
* one content-addressed entry per distinct file. See dev/specs/wkbackup.md in
|
||||
* the 3.x repository for the format itself.
|
||||
*
|
||||
* Records are written in 2.x's own shape, with 2.x's integer ids left intact:
|
||||
* `source.kind: wikijs2` tells the 3.x importer to run the translation, and the
|
||||
* importer derives every UUID from those source ids. This exporter judges
|
||||
* nothing it cannot judge from here, and reports what it knows will not carry
|
||||
* over in `manifest.warnings`.
|
||||
*/
|
||||
|
||||
const FORMAT_VERSION = 1
|
||||
const EXPORTER_VERSION = '1.0.0'
|
||||
|
||||
// 2.x has no sites, so the single site it does have is named `default`.
|
||||
const SITE_ID = 'default'
|
||||
|
||||
// A content or render field over this size spills to a blob, so that no single
|
||||
// NDJSON line forces a multi-megabyte string through the reader's parser.
|
||||
const BLOB_SPILL_THRESHOLD = 1024 * 1024
|
||||
|
||||
const BATCH_SIZE = {
|
||||
assets: 25,
|
||||
comments: 50,
|
||||
history: 10,
|
||||
pages: 10,
|
||||
tree: 100,
|
||||
users: 50
|
||||
}
|
||||
|
||||
/**
|
||||
* Permissions Wiki.js 3.x knows about, as listed by its group editor.
|
||||
*
|
||||
* A 2.x permission missing from this set has no 3.x equivalent. It is carried
|
||||
* in the package as-is and reported in `manifest.warnings`, never remapped: an
|
||||
* unrecognised permission string silently never matches, so a wrong guess would
|
||||
* hide controls with no error anywhere.
|
||||
*/
|
||||
const WIKI3_PERMISSIONS = [
|
||||
'access:admin', 'delete:pages', 'manage:assets', 'manage:comments',
|
||||
'manage:groups', 'manage:navigation', 'manage:pages', 'manage:scim',
|
||||
'manage:sites', 'manage:storage', 'manage:system', 'manage:theme',
|
||||
'manage:users', 'manage:webhooks', 'read:assets', 'read:audit',
|
||||
'read:comments', 'read:groups', 'read:history', 'read:metrics',
|
||||
'read:pages', 'read:source', 'read:users', 'read:webhooks',
|
||||
'review:pages', 'write:assets', 'write:comments', 'write:groups',
|
||||
'write:pages', 'write:scripts', 'write:styles', 'write:tags',
|
||||
'write:users'
|
||||
]
|
||||
|
||||
/**
|
||||
* Yield every row of a table in batches, so no table is ever fully resident.
|
||||
*/
|
||||
async function * batched (fetch, batchSize, onBatch) {
|
||||
let offset = 0
|
||||
while (true) {
|
||||
const rows = await fetch(offset, batchSize)
|
||||
for (const row of rows) {
|
||||
yield row
|
||||
}
|
||||
if (onBatch) { onBatch(rows.length) }
|
||||
if (rows.length < batchSize) { break }
|
||||
offset += batchSize
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn an async iterable of records into a stream of NDJSON lines.
|
||||
*/
|
||||
function ndjson (records) {
|
||||
return Readable.from((async function * () {
|
||||
for await (const record of records) {
|
||||
yield Buffer.from(JSON.stringify(record) + '\n')
|
||||
}
|
||||
})())
|
||||
}
|
||||
|
||||
function sha256 (buf) {
|
||||
return crypto.createHash('sha256').update(buf).digest('hex')
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
status: {
|
||||
status: 'notrunning',
|
||||
progress: 0,
|
||||
message: '',
|
||||
startedAt: null,
|
||||
filename: null,
|
||||
filePath: null,
|
||||
fileSize: null
|
||||
},
|
||||
|
||||
/**
|
||||
* Directory holding generated packages. The operator downloads the file from
|
||||
* here; nothing ever expires it, because no server holds a package for the
|
||||
* import's sake.
|
||||
*/
|
||||
get outputDir () {
|
||||
return path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, 'backup')
|
||||
},
|
||||
|
||||
/**
|
||||
* Resolve a package filename to a path inside the backup directory, or null
|
||||
* if it names anything else.
|
||||
*/
|
||||
resolveFile (filename) {
|
||||
if (!/^[A-Za-z0-9._-]+\.wkbackup$/.test(filename)) {
|
||||
return null
|
||||
}
|
||||
const target = path.resolve(this.outputDir, filename)
|
||||
if (path.dirname(target) !== this.outputDir) {
|
||||
return null
|
||||
}
|
||||
return target
|
||||
},
|
||||
|
||||
/**
|
||||
* Generate a .wkbackup package.
|
||||
*
|
||||
* @param {Object} opts
|
||||
* @param {string[]} opts.entities Entities to include.
|
||||
*/
|
||||
async create (opts) {
|
||||
const status = this.status
|
||||
const startedAt = new Date()
|
||||
|
||||
status.status = 'running'
|
||||
status.progress = 0
|
||||
status.message = ''
|
||||
status.startedAt = startedAt
|
||||
status.filename = null
|
||||
status.filePath = null
|
||||
status.fileSize = null
|
||||
|
||||
const entities = opts.entities
|
||||
const has = entity => entities.includes(entity)
|
||||
|
||||
// Millisecond precision, so two runs never name the same file and abort
|
||||
// the earlier package's cleanup over a previous one.
|
||||
const filename = `wikijs-${startedAt.toISOString().replace(/[-:.]/g, '')}.wkbackup`
|
||||
const filePath = path.join(this.outputDir, filename)
|
||||
const tmpPath = path.join(this.outputDir, `.tmp-${startedAt.getTime()}`)
|
||||
|
||||
WIKI.logger.info(`Backup started to ${filePath}`)
|
||||
WIKI.logger.info(`Entities to include: ${entities.join(', ')}`)
|
||||
|
||||
const zip = new ZipWriter(filePath, { tmpPath })
|
||||
|
||||
// Blobs referenced by the streams, written last. Content addressing makes
|
||||
// "have I already written this one?" answerable without any bookkeeping.
|
||||
const blobs = new Map()
|
||||
|
||||
try {
|
||||
await fs.ensureDir(this.outputDir)
|
||||
await zip.open()
|
||||
|
||||
// -----------------------------------------
|
||||
// COUNTS
|
||||
// -----------------------------------------
|
||||
// Counting first is what lets the manifest be written as the first entry
|
||||
// with real counts, while the streams are produced afterwards.
|
||||
const counts = {
|
||||
assets: has('assets') ? await this.countOf(WIKI.models.assets) : 0,
|
||||
comments: has('comments') ? await this.countOf(WIKI.models.comments) : 0,
|
||||
groups: has('groups') ? await this.countOf(WIKI.models.groups) : 0,
|
||||
history: has('history') ? await this.countOf(WIKI.models.pageHistory) : 0,
|
||||
locales: await this.countOf(WIKI.models.locales),
|
||||
pages: has('pages') ? await this.countOf(WIKI.models.pages) : 0,
|
||||
tree: has('pages') ? await this.countOfFolders() : 0,
|
||||
users: has('users') ? await this.countOf(WIKI.models.users) : 0
|
||||
}
|
||||
const blobBytes = has('assets') ? await this.sumAssetBytes() : 0
|
||||
|
||||
const warnings = await this.collectWarnings(entities)
|
||||
|
||||
// -> Steps, for progress reporting
|
||||
const steps = ['manifest']
|
||||
if (has('settings')) { steps.push('settings') }
|
||||
steps.push('locales')
|
||||
if (has('groups')) { steps.push('groups') }
|
||||
if (has('users')) { steps.push('users') }
|
||||
steps.push('site')
|
||||
if (has('navigation')) { steps.push('navigation') }
|
||||
if (has('pages')) { steps.push('tree', 'pages') }
|
||||
if (has('history')) { steps.push('history') }
|
||||
if (has('comments')) { steps.push('comments') }
|
||||
if (has('assets')) { steps.push('assets') }
|
||||
steps.push('blobs')
|
||||
|
||||
let doneSteps = 0
|
||||
const stepDone = () => {
|
||||
doneSteps++
|
||||
status.progress = Math.min(100, (doneSteps / steps.length) * 100)
|
||||
}
|
||||
// Report partial progress through the step currently being written.
|
||||
const stepPartial = fraction => {
|
||||
status.progress = Math.min(100, ((doneSteps + Math.min(1, fraction)) / steps.length) * 100)
|
||||
}
|
||||
// Track a streamed step of `total` records, batch by batch.
|
||||
const batchTracker = total => {
|
||||
let done = 0
|
||||
return rows => {
|
||||
done += rows
|
||||
stepPartial(total > 0 ? done / total : 1)
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------
|
||||
// MANIFEST — always the first entry
|
||||
// -----------------------------------------
|
||||
const streams = {}
|
||||
const siteStreams = {}
|
||||
|
||||
if (has('settings')) { streams.settings = { schema: 1, path: 'streams/settings.json' } }
|
||||
streams.locales = { schema: 1, path: 'streams/locales.ndjson', count: counts.locales }
|
||||
if (has('groups')) { streams.groups = { schema: 1, path: 'streams/groups.ndjson', count: counts.groups } }
|
||||
if (has('users')) { streams.users = { schema: 1, path: 'streams/users.ndjson', count: counts.users } }
|
||||
|
||||
const sitePath = `sites/${SITE_ID}`
|
||||
if (has('navigation')) { siteStreams.navigation = { schema: 1, path: `${sitePath}/navigation.json` } }
|
||||
if (has('pages')) {
|
||||
siteStreams.tree = { schema: 1, path: `${sitePath}/tree.ndjson`, count: counts.tree }
|
||||
siteStreams.pages = { schema: 1, path: `${sitePath}/pages.ndjson`, count: counts.pages }
|
||||
}
|
||||
if (has('history')) { siteStreams['page-history'] = { schema: 1, path: `${sitePath}/page-history.ndjson`, count: counts.history } }
|
||||
if (has('comments')) { siteStreams.comments = { schema: 1, path: `${sitePath}/comments.ndjson`, count: counts.comments } }
|
||||
if (has('assets')) { siteStreams.assets = { schema: 1, path: `${sitePath}/assets.ndjson`, count: counts.assets, blobBytes } }
|
||||
|
||||
const manifest = {
|
||||
format: 'wkbackup',
|
||||
formatVersion: FORMAT_VERSION,
|
||||
createdAt: startedAt.toISOString(),
|
||||
generator: {
|
||||
product: 'wiki.js',
|
||||
version: WIKI.version,
|
||||
exporter: EXPORTER_VERSION
|
||||
},
|
||||
source: {
|
||||
kind: 'wikijs2',
|
||||
instanceId: _.get(WIKI.config, 'telemetry.clientId', null),
|
||||
baseUrl: WIKI.config.host
|
||||
},
|
||||
streams,
|
||||
sites: [
|
||||
{
|
||||
id: SITE_ID,
|
||||
title: WIKI.config.title,
|
||||
hostname: WIKI.config.host,
|
||||
locales: _.union([WIKI.config.lang.code], _.get(WIKI.config, 'lang.namespaces', [])),
|
||||
streams: siteStreams
|
||||
}
|
||||
],
|
||||
options: { includes: entities },
|
||||
warnings
|
||||
}
|
||||
await zip.addBuffer('manifest.json', Buffer.from(JSON.stringify(manifest, null, 2)))
|
||||
stepDone()
|
||||
|
||||
// -----------------------------------------
|
||||
// SETTINGS
|
||||
// -----------------------------------------
|
||||
// Written whole, keyed as 2.x keys it. The exporter cannot know which
|
||||
// version will read the package, so the importer owns the mapping.
|
||||
if (has('settings')) {
|
||||
WIKI.logger.info('Backup: writing settings...')
|
||||
const settings = {
|
||||
...WIKI.config,
|
||||
modules: {
|
||||
analytics: await WIKI.models.analytics.query(),
|
||||
authentication: (await WIKI.models.authentication.query()).map(a => ({
|
||||
...a,
|
||||
domainWhitelist: _.get(a, 'domainWhitelist.v', []),
|
||||
autoEnrollGroups: _.get(a, 'autoEnrollGroups.v', [])
|
||||
})),
|
||||
commentProviders: await WIKI.models.commentProviders.query(),
|
||||
renderers: await WIKI.models.renderers.query(),
|
||||
searchEngines: await WIKI.models.searchEngines.query(),
|
||||
storage: await WIKI.models.storage.query()
|
||||
},
|
||||
apiKeys: await WIKI.models.apiKeys.query().where('isRevoked', false)
|
||||
}
|
||||
await zip.addBuffer('streams/settings.json', Buffer.from(JSON.stringify(settings, null, 2)))
|
||||
stepDone()
|
||||
}
|
||||
|
||||
// -----------------------------------------
|
||||
// LOCALES
|
||||
// -----------------------------------------
|
||||
// Codes only; the string sets themselves are 3.x's own.
|
||||
WIKI.logger.info('Backup: writing locales...')
|
||||
const primaryLocale = WIKI.config.lang.code
|
||||
const activeLocales = _.union([primaryLocale], _.get(WIKI.config, 'lang.namespaces', []))
|
||||
const locales = await WIKI.models.locales.query()
|
||||
.select('code', 'name', 'nativeName', 'isRTL', 'availability', 'createdAt', 'updatedAt')
|
||||
.orderBy('code')
|
||||
await zip.addStream('streams/locales.ndjson', ndjson(locales.map(lc => ({
|
||||
...lc,
|
||||
isPrimary: lc.code === primaryLocale,
|
||||
isActive: activeLocales.includes(lc.code)
|
||||
}))))
|
||||
stepDone()
|
||||
|
||||
// -----------------------------------------
|
||||
// GROUPS
|
||||
// -----------------------------------------
|
||||
if (has('groups')) {
|
||||
WIKI.logger.info('Backup: writing groups...')
|
||||
const groups = await WIKI.models.groups.query().orderBy('id')
|
||||
await zip.addStream('streams/groups.ndjson', ndjson(groups))
|
||||
stepDone()
|
||||
}
|
||||
|
||||
// -----------------------------------------
|
||||
// USERS
|
||||
// -----------------------------------------
|
||||
// Password hashes and TOTP secrets travel as-is: both versions use
|
||||
// bcryptjs at cost 12 and plain RFC 6238 TOTP, so nobody has to reset a
|
||||
// password or re-enrol an authenticator after the migration.
|
||||
if (has('users')) {
|
||||
WIKI.logger.info(`Backup: writing ${counts.users} users...`)
|
||||
const onBatch = batchTracker(counts.users)
|
||||
await zip.addStream('streams/users.ndjson', ndjson(batched(async (offset, limit) => {
|
||||
const users = await WIKI.models.users.query()
|
||||
.orderBy('id').offset(offset).limit(limit)
|
||||
.withGraphJoined({ groups: true })
|
||||
.modifyGraph('groups', builder => builder.select('groups.id', 'groups.name'))
|
||||
return users.map(usr => ({
|
||||
..._.omit(usr, ['groups']),
|
||||
groups: usr.groups.map(g => g.id)
|
||||
}))
|
||||
}, BATCH_SIZE.users, onBatch)))
|
||||
stepDone()
|
||||
}
|
||||
|
||||
// -----------------------------------------
|
||||
// SITE
|
||||
// -----------------------------------------
|
||||
WIKI.logger.info('Backup: writing site settings...')
|
||||
await zip.addBuffer(`${sitePath}/site.json`, Buffer.from(JSON.stringify({
|
||||
id: SITE_ID,
|
||||
title: WIKI.config.title,
|
||||
hostname: WIKI.config.host,
|
||||
company: WIKI.config.company,
|
||||
contentLicense: WIKI.config.contentLicense,
|
||||
footerOverride: WIKI.config.footerOverride,
|
||||
logoUrl: WIKI.config.logoUrl,
|
||||
pageExtensions: WIKI.config.pageExtensions,
|
||||
locales: {
|
||||
primary: primaryLocale,
|
||||
active: activeLocales,
|
||||
namespacing: _.get(WIKI.config, 'lang.namespacing', false)
|
||||
},
|
||||
theme: WIKI.config.theming,
|
||||
features: WIKI.config.features,
|
||||
security: WIKI.config.security,
|
||||
seo: WIKI.config.seo,
|
||||
editShortcuts: WIKI.config.editShortcuts,
|
||||
uploads: WIKI.config.uploads
|
||||
}, null, 2)))
|
||||
stepDone()
|
||||
|
||||
// -----------------------------------------
|
||||
// NAVIGATION
|
||||
// -----------------------------------------
|
||||
// 2.x keeps one tree per locale; the importer turns each into the
|
||||
// site-wide menu for its locale.
|
||||
if (has('navigation')) {
|
||||
WIKI.logger.info('Backup: writing navigation...')
|
||||
const navigationRaw = await WIKI.models.navigation.query()
|
||||
const navigation = navigationRaw.reduce((obj, cur) => {
|
||||
obj[cur.key] = cur.config
|
||||
return obj
|
||||
}, {})
|
||||
await zip.addBuffer(`${sitePath}/navigation.json`, Buffer.from(JSON.stringify({
|
||||
mode: _.get(WIKI.config, 'nav.mode', 'MIXED'),
|
||||
trees: navigation
|
||||
}, null, 2)))
|
||||
stepDone()
|
||||
}
|
||||
|
||||
// -----------------------------------------
|
||||
// TREE (folders)
|
||||
// -----------------------------------------
|
||||
if (has('pages')) {
|
||||
WIKI.logger.info(`Backup: writing ${counts.tree} folders...`)
|
||||
const onBatch = batchTracker(counts.tree)
|
||||
await zip.addStream(`${sitePath}/tree.ndjson`, ndjson(batched(async (offset, limit) => {
|
||||
return WIKI.models.knex('pageTree')
|
||||
.select('id', 'path', 'depth', 'title', 'isPrivate', 'privateNS', 'parent', 'localeCode')
|
||||
.where('isFolder', true)
|
||||
.orderBy('id').offset(offset).limit(limit)
|
||||
}, BATCH_SIZE.tree, onBatch)))
|
||||
stepDone()
|
||||
|
||||
// -----------------------------------------
|
||||
// PAGES
|
||||
// -----------------------------------------
|
||||
// The render is deliberately left behind: 3.x renders differently
|
||||
// enough that a 2.x render would be silently wrong under it, and a
|
||||
// stored render says nothing about which pipeline produced it. 3.x
|
||||
// produces the HTML itself.
|
||||
WIKI.logger.info(`Backup: writing ${counts.pages} pages...`)
|
||||
const onPageBatch = batchTracker(counts.pages)
|
||||
await zip.addStream(`${sitePath}/pages.ndjson`, ndjson(batched(async (offset, limit) => {
|
||||
const pages = await WIKI.models.pages.query()
|
||||
.orderBy('id').offset(offset).limit(limit)
|
||||
.withGraphJoined({ tags: true })
|
||||
.modifyGraph('tags', builder => builder.select('tags.tag', 'tags.title'))
|
||||
return Promise.all(pages.map(page => this.spillContent({
|
||||
..._.omit(page, ['render', 'toc']),
|
||||
tags: page.tags.map(t => t.tag)
|
||||
}, tmpPath, blobs)))
|
||||
}, BATCH_SIZE.pages, onPageBatch)))
|
||||
stepDone()
|
||||
}
|
||||
|
||||
// -----------------------------------------
|
||||
// PAGE HISTORY
|
||||
// -----------------------------------------
|
||||
if (has('history')) {
|
||||
WIKI.logger.info(`Backup: writing ${counts.history} page history entries...`)
|
||||
const onBatch = batchTracker(counts.history)
|
||||
await zip.addStream(`${sitePath}/page-history.ndjson`, ndjson(batched(async (offset, limit) => {
|
||||
const versions = await WIKI.models.pageHistory.query()
|
||||
.orderBy('id').offset(offset).limit(limit)
|
||||
.withGraphJoined({ tags: true })
|
||||
.modifyGraph('tags', builder => builder.select('tags.tag', 'tags.title'))
|
||||
return Promise.all(versions.map(version => this.spillContent({
|
||||
...version,
|
||||
tags: version.tags.map(t => t.tag)
|
||||
}, tmpPath, blobs)))
|
||||
}, BATCH_SIZE.history, onBatch)))
|
||||
stepDone()
|
||||
}
|
||||
|
||||
// -----------------------------------------
|
||||
// COMMENTS
|
||||
// -----------------------------------------
|
||||
if (has('comments')) {
|
||||
WIKI.logger.info(`Backup: writing ${counts.comments} comments...`)
|
||||
const onBatch = batchTracker(counts.comments)
|
||||
await zip.addStream(`${sitePath}/comments.ndjson`, ndjson(batched(async (offset, limit) => {
|
||||
return WIKI.models.comments.query().orderBy('id').offset(offset).limit(limit)
|
||||
}, BATCH_SIZE.comments, onBatch)))
|
||||
stepDone()
|
||||
}
|
||||
|
||||
// -----------------------------------------
|
||||
// ASSETS
|
||||
// -----------------------------------------
|
||||
// Metadata only; the bytes go to blobs/, keyed by their own digest.
|
||||
if (has('assets')) {
|
||||
WIKI.logger.info(`Backup: writing ${counts.assets} assets...`)
|
||||
const assetFolders = await WIKI.models.assetFolders.getAllPaths()
|
||||
const onBatch = batchTracker(counts.assets)
|
||||
await zip.addStream(`${sitePath}/assets.ndjson`, ndjson(batched(async (offset, limit) => {
|
||||
const assets = await WIKI.models.knex
|
||||
.select('assets.*', 'assetData.data')
|
||||
.from('assets')
|
||||
.join('assetData', 'assets.id', '=', 'assetData.id')
|
||||
.orderBy('assets.id').offset(offset).limit(limit)
|
||||
return assets.map(asset => {
|
||||
const digest = sha256(asset.data)
|
||||
if (!blobs.has(digest)) {
|
||||
blobs.set(digest, { kind: 'asset', id: asset.id })
|
||||
}
|
||||
return {
|
||||
..._.omit(asset, ['data']),
|
||||
// 2.x's `hash` is a digest of the path, not of the contents.
|
||||
folderPath: (asset.folderId && asset.folderId > 0) ? _.get(assetFolders, asset.folderId, '') : '',
|
||||
blob: digest
|
||||
}
|
||||
})
|
||||
}, BATCH_SIZE.assets, onBatch)))
|
||||
stepDone()
|
||||
}
|
||||
|
||||
// -----------------------------------------
|
||||
// BLOBS
|
||||
// -----------------------------------------
|
||||
// One entry per distinct file, named after its own digest, stored rather
|
||||
// than deflated: deflating a JPEG costs CPU on both ends to make it very
|
||||
// slightly larger. Oversized page content spilled here too.
|
||||
WIKI.logger.info(`Backup: writing ${blobs.size} blobs...`)
|
||||
let doneBlobs = 0
|
||||
for (const [digest, ref] of blobs) {
|
||||
const data = ref.kind === 'asset' ? _.get(
|
||||
await WIKI.models.knex('assetData').select('data').where('id', ref.id).first(),
|
||||
'data',
|
||||
null
|
||||
) : await fs.readFile(ref.path)
|
||||
if (!data) {
|
||||
WIKI.logger.warn(`Backup: blob ${digest} has no data, skipping...`)
|
||||
continue
|
||||
}
|
||||
await zip.addBuffer(`blobs/${digest}`, data, { compress: false })
|
||||
doneBlobs++
|
||||
stepPartial(doneBlobs / blobs.size)
|
||||
}
|
||||
stepDone()
|
||||
|
||||
await zip.close()
|
||||
|
||||
const { size } = await fs.stat(filePath)
|
||||
status.status = 'success'
|
||||
status.progress = 100
|
||||
status.filename = filename
|
||||
status.filePath = filePath
|
||||
status.fileSize = size
|
||||
WIKI.logger.info(`Backup completed: ${filePath} (${size} bytes)`)
|
||||
} catch (err) {
|
||||
WIKI.logger.warn(err)
|
||||
status.status = 'error'
|
||||
status.message = err.message
|
||||
await zip.abort().catch(() => {})
|
||||
} finally {
|
||||
await fs.remove(tmpPath).catch(() => {})
|
||||
}
|
||||
},
|
||||
|
||||
async countOf (model) {
|
||||
const result = await model.query().count('* as total').first()
|
||||
return parseInt(result.total)
|
||||
},
|
||||
|
||||
async countOfFolders () {
|
||||
const result = await WIKI.models.knex('pageTree').where('isFolder', true).count('* as total').first()
|
||||
return parseInt(result.total)
|
||||
},
|
||||
|
||||
async sumAssetBytes () {
|
||||
const result = await WIKI.models.knex('assets').sum('fileSize as total').first()
|
||||
return parseInt(_.get(result, 'total', 0)) || 0
|
||||
},
|
||||
|
||||
/**
|
||||
* Move an oversized `content` field out to a blob, so that no NDJSON line
|
||||
* forces a multi-megabyte string through the reader's parser.
|
||||
*/
|
||||
async spillContent (record, tmpPath, blobs) {
|
||||
if (!record.content || Buffer.byteLength(record.content, 'utf8') <= BLOB_SPILL_THRESHOLD) {
|
||||
return record
|
||||
}
|
||||
const data = Buffer.from(record.content, 'utf8')
|
||||
const digest = sha256(data)
|
||||
if (!blobs.has(digest)) {
|
||||
const spillPath = path.join(tmpPath, `spill-${digest}`)
|
||||
await fs.outputFile(spillPath, data)
|
||||
blobs.set(digest, { kind: 'file', path: spillPath })
|
||||
}
|
||||
return { ...record, content: null, contentBlob: digest }
|
||||
},
|
||||
|
||||
/**
|
||||
* Things this exporter knows will not carry over, written into the manifest
|
||||
* and shown in the import log before the import starts.
|
||||
*/
|
||||
async collectWarnings (entities) {
|
||||
const warnings = []
|
||||
|
||||
if (entities.includes('groups')) {
|
||||
const groups = await WIKI.models.groups.query().select('id', 'name', 'permissions')
|
||||
const unmappable = new Set()
|
||||
for (const group of groups) {
|
||||
for (const permission of (group.permissions || [])) {
|
||||
if (!WIKI3_PERMISSIONS.includes(permission)) {
|
||||
unmappable.add(permission)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (unmappable.size > 0) {
|
||||
warnings.push(`Permissions with no Wiki.js 3.x equivalent will be dropped on import: ${[...unmappable].sort().join(', ')}.`)
|
||||
}
|
||||
}
|
||||
|
||||
if (entities.includes('pages') || entities.includes('history')) {
|
||||
warnings.push('Page renders are not carried over. Wiki.js 3.x renders pages with its own pipeline, so imported pages are re-rendered in the background.')
|
||||
}
|
||||
|
||||
if (entities.includes('comments')) {
|
||||
const provider = await WIKI.models.commentProviders.query().where('isEnabled', true).first()
|
||||
if (provider && provider.key !== 'default') {
|
||||
warnings.push(`Comments are handled by the '${provider.key}' provider on this instance; only comments stored by the default provider are included.`)
|
||||
}
|
||||
}
|
||||
|
||||
if (entities.includes('assets')) {
|
||||
warnings.push('Asset folders are carried on each asset as `folderPath`; Wiki.js 2.x has no shared tree for them.')
|
||||
}
|
||||
|
||||
if (entities.includes('settings')) {
|
||||
warnings.push('Settings are exported in full, keyed as Wiki.js 2.x keys them. The importer applies the keys that have a 3.x equivalent and reports the rest.')
|
||||
}
|
||||
|
||||
const missing = ['assets', 'comments', 'groups', 'history', 'navigation', 'pages', 'settings', 'users'].filter(e => !entities.includes(e))
|
||||
if (missing.length > 0) {
|
||||
warnings.push(`Not included in this package: ${missing.join(', ')}.`)
|
||||
}
|
||||
|
||||
return warnings
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,389 @@
|
||||
const fs = require('fs-extra')
|
||||
const path = require('path')
|
||||
const zlib = require('zlib')
|
||||
const { pipeline } = require('node:stream/promises')
|
||||
const { Transform } = require('node:stream')
|
||||
|
||||
/* global BigInt */
|
||||
|
||||
/**
|
||||
* Minimal ZIP writer for the .wkbackup package format.
|
||||
*
|
||||
* The wkbackup spec constrains the writer so that readers can stay small:
|
||||
* - No data descriptors. CRC32 and both sizes always go in the local header,
|
||||
* and general purpose bit 3 is never set. A deflated entry therefore has to
|
||||
* be compressed before its local header can be written, which is why
|
||||
* addStream() stages the compressed bytes in a temp file first.
|
||||
* - General purpose bit 11 (UTF-8 filenames) is always set.
|
||||
* - ZIP64 whenever an entry, the archive, or the entry count requires it.
|
||||
* - Entry order is preserved exactly as added; it is part of the format.
|
||||
* - Compression method is chosen per entry: deflate for text, store for
|
||||
* anything already compressed.
|
||||
*/
|
||||
|
||||
const LOCAL_SIG = 0x04034b50
|
||||
const CENTRAL_SIG = 0x02014b50
|
||||
const EOCD_SIG = 0x06054b50
|
||||
const ZIP64_EOCD_SIG = 0x06064b50
|
||||
const ZIP64_LOCATOR_SIG = 0x07064b50
|
||||
|
||||
const ZIP64_LIMIT = 0xfffffffe
|
||||
const ZIP64_COUNT_LIMIT = 0xfffe
|
||||
|
||||
const METHOD_STORE = 0
|
||||
const METHOD_DEFLATE = 8
|
||||
|
||||
const FLAG_UTF8 = 0x0800
|
||||
|
||||
const VERSION_BASE = 20
|
||||
const VERSION_ZIP64 = 45
|
||||
|
||||
// Version made by: UNIX (3) in the upper byte, spec version in the lower.
|
||||
const VERSION_MADE_BY = (3 << 8) | VERSION_ZIP64
|
||||
|
||||
const crcTable = (() => {
|
||||
const table = new Int32Array(256)
|
||||
for (let n = 0; n < 256; n++) {
|
||||
let c = n
|
||||
for (let k = 0; k < 8; k++) {
|
||||
c = (c & 1) ? (0xedb88320 ^ (c >>> 1)) : (c >>> 1)
|
||||
}
|
||||
table[n] = c
|
||||
}
|
||||
return table
|
||||
})()
|
||||
|
||||
/**
|
||||
* Incremental CRC32, matching the polynomial ZIP uses.
|
||||
*/
|
||||
class CRC32 {
|
||||
constructor () {
|
||||
this.crc = -1
|
||||
}
|
||||
|
||||
update (buf) {
|
||||
let crc = this.crc
|
||||
for (let i = 0; i < buf.length; i++) {
|
||||
crc = crcTable[(crc ^ buf[i]) & 0xff] ^ (crc >>> 8)
|
||||
}
|
||||
this.crc = crc
|
||||
return this
|
||||
}
|
||||
|
||||
get value () {
|
||||
return (this.crc ^ -1) >>> 0
|
||||
}
|
||||
}
|
||||
|
||||
function crc32 (buf) {
|
||||
return new CRC32().update(buf).value
|
||||
}
|
||||
|
||||
/**
|
||||
* MS-DOS date / time pair, as stored in ZIP headers.
|
||||
*/
|
||||
function dosDateTime (date) {
|
||||
const year = date.getFullYear()
|
||||
if (year < 1980) {
|
||||
return { date: 0x21, time: 0 }
|
||||
}
|
||||
return {
|
||||
date: ((year - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(),
|
||||
time: (date.getHours() << 11) | (date.getMinutes() << 5) | Math.floor(date.getSeconds() / 2)
|
||||
}
|
||||
}
|
||||
|
||||
class ZipWriter {
|
||||
/**
|
||||
* @param {string} outputPath Path of the archive to create.
|
||||
* @param {Object} opts
|
||||
* @param {string} opts.tmpPath Directory used to stage compressed entries.
|
||||
*/
|
||||
constructor (outputPath, { tmpPath }) {
|
||||
this.outputPath = outputPath
|
||||
this.tmpPath = tmpPath
|
||||
this.entries = []
|
||||
this.offset = 0
|
||||
this.stream = null
|
||||
this.tmpSeq = 0
|
||||
}
|
||||
|
||||
async open () {
|
||||
await fs.ensureDir(path.dirname(this.outputPath))
|
||||
await fs.ensureDir(this.tmpPath)
|
||||
this.stream = fs.createWriteStream(this.outputPath)
|
||||
await new Promise((resolve, reject) => {
|
||||
this.stream.once('open', resolve)
|
||||
this.stream.once('error', reject)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Write raw bytes to the archive, honouring backpressure.
|
||||
*/
|
||||
async _write (buf) {
|
||||
if (!this.stream.write(buf)) {
|
||||
await new Promise((resolve, reject) => {
|
||||
const onDrain = () => {
|
||||
this.stream.removeListener('error', onError)
|
||||
resolve()
|
||||
}
|
||||
const onError = (err) => {
|
||||
this.stream.removeListener('drain', onDrain)
|
||||
reject(err)
|
||||
}
|
||||
this.stream.once('drain', onDrain)
|
||||
this.stream.once('error', onError)
|
||||
})
|
||||
}
|
||||
this.offset += buf.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an entry whose CRC32 and both sizes are already known, writing the
|
||||
* local header followed by `writeBody()`.
|
||||
*/
|
||||
async _addEntry (name, { method, crc, compressedSize, uncompressedSize, modifiedAt }, writeBody) {
|
||||
const nameBuf = Buffer.from(name, 'utf8')
|
||||
const needsZip64 = compressedSize > ZIP64_LIMIT || uncompressedSize > ZIP64_LIMIT
|
||||
const { date, time } = dosDateTime(modifiedAt || new Date())
|
||||
|
||||
// -> Local file header
|
||||
const extraLen = needsZip64 ? 20 : 0
|
||||
const header = Buffer.alloc(30 + extraLen)
|
||||
header.writeUInt32LE(LOCAL_SIG, 0)
|
||||
header.writeUInt16LE(needsZip64 ? VERSION_ZIP64 : VERSION_BASE, 4)
|
||||
header.writeUInt16LE(FLAG_UTF8, 6)
|
||||
header.writeUInt16LE(method, 8)
|
||||
header.writeUInt16LE(time, 10)
|
||||
header.writeUInt16LE(date, 12)
|
||||
header.writeUInt32LE(crc, 14)
|
||||
header.writeUInt32LE(needsZip64 ? 0xffffffff : compressedSize, 18)
|
||||
header.writeUInt32LE(needsZip64 ? 0xffffffff : uncompressedSize, 22)
|
||||
header.writeUInt16LE(nameBuf.length, 26)
|
||||
header.writeUInt16LE(extraLen, 28)
|
||||
if (needsZip64) {
|
||||
// Both sizes are mandatory in a local header's ZIP64 extra field.
|
||||
header.writeUInt16LE(0x0001, 30)
|
||||
header.writeUInt16LE(16, 32)
|
||||
header.writeBigUInt64LE(BigInt(uncompressedSize), 34)
|
||||
header.writeBigUInt64LE(BigInt(compressedSize), 42)
|
||||
}
|
||||
|
||||
const localHeaderOffset = this.offset
|
||||
await this._write(header)
|
||||
await this._write(nameBuf)
|
||||
await writeBody()
|
||||
|
||||
this.entries.push({
|
||||
nameBuf,
|
||||
method,
|
||||
crc,
|
||||
compressedSize,
|
||||
uncompressedSize,
|
||||
localHeaderOffset,
|
||||
date,
|
||||
time,
|
||||
needsZip64
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an entry from a buffer already held in memory. Used for the manifest,
|
||||
* the small JSON streams and for blobs, whose bytes come out of the database
|
||||
* as a single buffer anyway.
|
||||
*
|
||||
* @param {string} name Entry path within the archive.
|
||||
* @param {Buffer} buf Uncompressed contents.
|
||||
* @param {Object} opts
|
||||
* @param {boolean} opts.compress Deflate the entry (false stores it as-is).
|
||||
*/
|
||||
async addBuffer (name, buf, { compress = true, modifiedAt } = {}) {
|
||||
const uncompressedSize = buf.length
|
||||
const crc = crc32(buf)
|
||||
let method = METHOD_STORE
|
||||
let body = buf
|
||||
|
||||
if (compress && uncompressedSize > 0) {
|
||||
const deflated = await new Promise((resolve, reject) => {
|
||||
zlib.deflateRaw(buf, (err, result) => err ? reject(err) : resolve(result))
|
||||
})
|
||||
// Storing is better than a deflate that made the entry bigger.
|
||||
if (deflated.length < uncompressedSize) {
|
||||
method = METHOD_DEFLATE
|
||||
body = deflated
|
||||
}
|
||||
}
|
||||
|
||||
await this._addEntry(name, {
|
||||
method,
|
||||
crc,
|
||||
compressedSize: body.length,
|
||||
uncompressedSize,
|
||||
modifiedAt
|
||||
}, () => this._write(body))
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an entry produced by a stream of unknown length.
|
||||
*
|
||||
* Because the local header may not be followed by a data descriptor, the
|
||||
* contents are staged to a temp file first: that pass computes the CRC32 and
|
||||
* both sizes, and the staged bytes are then copied into the archive.
|
||||
*
|
||||
* @param {string} name Entry path within the archive.
|
||||
* @param {Readable} source Stream of uncompressed bytes.
|
||||
* @param {Object} opts
|
||||
* @param {boolean} opts.compress Deflate the entry (false stores it as-is).
|
||||
*/
|
||||
async addStream (name, source, { compress = true, modifiedAt } = {}) {
|
||||
const stagePath = path.join(this.tmpPath, `entry-${this.tmpSeq++}.part`)
|
||||
const hasher = new CRC32()
|
||||
let uncompressedSize = 0
|
||||
|
||||
const counter = new Transform({
|
||||
transform (chunk, enc, cb) {
|
||||
hasher.update(chunk)
|
||||
uncompressedSize += chunk.length
|
||||
cb(null, chunk)
|
||||
}
|
||||
})
|
||||
|
||||
try {
|
||||
const stages = [source, counter]
|
||||
if (compress) {
|
||||
stages.push(zlib.createDeflateRaw())
|
||||
}
|
||||
stages.push(fs.createWriteStream(stagePath))
|
||||
await pipeline(...stages)
|
||||
|
||||
const { size: compressedSize } = await fs.stat(stagePath)
|
||||
|
||||
await this._addEntry(name, {
|
||||
method: compress ? METHOD_DEFLATE : METHOD_STORE,
|
||||
crc: hasher.value,
|
||||
compressedSize,
|
||||
uncompressedSize,
|
||||
modifiedAt
|
||||
}, async () => {
|
||||
for await (const chunk of fs.createReadStream(stagePath)) {
|
||||
await this._write(chunk)
|
||||
}
|
||||
})
|
||||
|
||||
return { uncompressedSize, compressedSize }
|
||||
} finally {
|
||||
await fs.remove(stagePath)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the central directory and close the archive.
|
||||
*/
|
||||
async close () {
|
||||
const centralOffset = this.offset
|
||||
|
||||
for (const entry of this.entries) {
|
||||
const zip64Fields = []
|
||||
if (entry.uncompressedSize > ZIP64_LIMIT) { zip64Fields.push(entry.uncompressedSize) }
|
||||
if (entry.compressedSize > ZIP64_LIMIT) { zip64Fields.push(entry.compressedSize) }
|
||||
if (entry.localHeaderOffset > ZIP64_LIMIT) { zip64Fields.push(entry.localHeaderOffset) }
|
||||
|
||||
// The ZIP64 extra field carries only the fields that overflowed, in the
|
||||
// fixed order: uncompressed size, compressed size, local header offset.
|
||||
const extraLen = zip64Fields.length > 0 ? 4 + (zip64Fields.length * 8) : 0
|
||||
const header = Buffer.alloc(46 + extraLen)
|
||||
header.writeUInt32LE(CENTRAL_SIG, 0)
|
||||
header.writeUInt16LE(VERSION_MADE_BY, 4)
|
||||
header.writeUInt16LE(extraLen > 0 ? VERSION_ZIP64 : VERSION_BASE, 6)
|
||||
header.writeUInt16LE(FLAG_UTF8, 8)
|
||||
header.writeUInt16LE(entry.method, 10)
|
||||
header.writeUInt16LE(entry.time, 12)
|
||||
header.writeUInt16LE(entry.date, 14)
|
||||
header.writeUInt32LE(entry.crc, 16)
|
||||
header.writeUInt32LE(entry.compressedSize > ZIP64_LIMIT ? 0xffffffff : entry.compressedSize, 20)
|
||||
header.writeUInt32LE(entry.uncompressedSize > ZIP64_LIMIT ? 0xffffffff : entry.uncompressedSize, 24)
|
||||
header.writeUInt16LE(entry.nameBuf.length, 28)
|
||||
header.writeUInt16LE(extraLen, 30)
|
||||
header.writeUInt16LE(0, 32) // file comment length
|
||||
header.writeUInt16LE(0, 34) // disk number start
|
||||
header.writeUInt16LE(0, 36) // internal attributes
|
||||
header.writeUInt32LE(0o644 << 16, 38) // external attributes
|
||||
header.writeUInt32LE(entry.localHeaderOffset > ZIP64_LIMIT ? 0xffffffff : entry.localHeaderOffset, 42)
|
||||
if (extraLen > 0) {
|
||||
header.writeUInt16LE(0x0001, 46)
|
||||
header.writeUInt16LE(zip64Fields.length * 8, 48)
|
||||
zip64Fields.forEach((value, idx) => {
|
||||
header.writeBigUInt64LE(BigInt(value), 50 + (idx * 8))
|
||||
})
|
||||
}
|
||||
|
||||
await this._write(header)
|
||||
await this._write(entry.nameBuf)
|
||||
}
|
||||
|
||||
const centralSize = this.offset - centralOffset
|
||||
const entryCount = this.entries.length
|
||||
const needsZip64 = entryCount > ZIP64_COUNT_LIMIT ||
|
||||
centralSize > ZIP64_LIMIT ||
|
||||
centralOffset > ZIP64_LIMIT
|
||||
|
||||
if (needsZip64) {
|
||||
const zip64Offset = this.offset
|
||||
|
||||
const record = Buffer.alloc(56)
|
||||
record.writeUInt32LE(ZIP64_EOCD_SIG, 0)
|
||||
record.writeBigUInt64LE(BigInt(44), 4) // size of this record, minus 12
|
||||
record.writeUInt16LE(VERSION_MADE_BY, 12)
|
||||
record.writeUInt16LE(VERSION_ZIP64, 14)
|
||||
record.writeUInt32LE(0, 16) // this disk
|
||||
record.writeUInt32LE(0, 20) // disk with start of central directory
|
||||
record.writeBigUInt64LE(BigInt(entryCount), 24)
|
||||
record.writeBigUInt64LE(BigInt(entryCount), 32)
|
||||
record.writeBigUInt64LE(BigInt(centralSize), 40)
|
||||
record.writeBigUInt64LE(BigInt(centralOffset), 48)
|
||||
await this._write(record)
|
||||
|
||||
const locator = Buffer.alloc(20)
|
||||
locator.writeUInt32LE(ZIP64_LOCATOR_SIG, 0)
|
||||
locator.writeUInt32LE(0, 4) // disk with the ZIP64 end of central directory
|
||||
locator.writeBigUInt64LE(BigInt(zip64Offset), 8)
|
||||
locator.writeUInt32LE(1, 16) // total number of disks
|
||||
await this._write(locator)
|
||||
}
|
||||
|
||||
const eocd = Buffer.alloc(22)
|
||||
eocd.writeUInt32LE(EOCD_SIG, 0)
|
||||
eocd.writeUInt16LE(0, 4) // this disk
|
||||
eocd.writeUInt16LE(0, 6) // disk with start of central directory
|
||||
eocd.writeUInt16LE(entryCount > ZIP64_COUNT_LIMIT ? 0xffff : entryCount, 8)
|
||||
eocd.writeUInt16LE(entryCount > ZIP64_COUNT_LIMIT ? 0xffff : entryCount, 10)
|
||||
eocd.writeUInt32LE(centralSize > ZIP64_LIMIT ? 0xffffffff : centralSize, 12)
|
||||
eocd.writeUInt32LE(centralOffset > ZIP64_LIMIT ? 0xffffffff : centralOffset, 16)
|
||||
eocd.writeUInt16LE(0, 20) // archive comment length
|
||||
await this._write(eocd)
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
this.stream.once('error', reject)
|
||||
this.stream.end(resolve)
|
||||
})
|
||||
this.stream = null
|
||||
}
|
||||
|
||||
/**
|
||||
* Abort the archive, discarding whatever was written so far.
|
||||
*/
|
||||
async abort () {
|
||||
if (this.stream) {
|
||||
await new Promise(resolve => this.stream.end(resolve))
|
||||
this.stream = null
|
||||
}
|
||||
await fs.remove(this.outputPath)
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
ZipWriter,
|
||||
CRC32,
|
||||
crc32
|
||||
}
|
||||
Loading…
Reference in new issue