|
|
|
|
@ -4,8 +4,7 @@ const Promise = require('bluebird')
|
|
|
|
|
const fs = require('fs-extra')
|
|
|
|
|
const path = require('path')
|
|
|
|
|
const zlib = require('zlib')
|
|
|
|
|
const stream = require('stream')
|
|
|
|
|
const pipeline = Promise.promisify(stream.pipeline)
|
|
|
|
|
const { pipeline, Readable, Transform } = require('stream')
|
|
|
|
|
|
|
|
|
|
/* global WIKI */
|
|
|
|
|
|
|
|
|
|
@ -121,7 +120,7 @@ module.exports = {
|
|
|
|
|
|
|
|
|
|
await pipeline(
|
|
|
|
|
WIKI.models.knex.select('filename', 'folderId', 'data').from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(),
|
|
|
|
|
new stream.Transform({
|
|
|
|
|
new Transform({
|
|
|
|
|
objectMode: true,
|
|
|
|
|
transform: async (asset, enc, cb) => {
|
|
|
|
|
const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename
|
|
|
|
|
@ -150,7 +149,7 @@ module.exports = {
|
|
|
|
|
const commentsProgressMultiplier = progressMultiplier / Math.ceil(commentsCount / 50)
|
|
|
|
|
WIKI.logger.info(`Found ${commentsCount} comments to export. Streaming to file...`)
|
|
|
|
|
|
|
|
|
|
const rs = stream.Readable({ objectMode: true })
|
|
|
|
|
const rs = Readable({ objectMode: true })
|
|
|
|
|
rs._read = () => {}
|
|
|
|
|
|
|
|
|
|
const fetchCommentsBatch = async (offset) => {
|
|
|
|
|
@ -177,7 +176,7 @@ module.exports = {
|
|
|
|
|
let marker = 0
|
|
|
|
|
await pipeline(
|
|
|
|
|
rs,
|
|
|
|
|
new stream.Transform({
|
|
|
|
|
new Transform({
|
|
|
|
|
objectMode: true,
|
|
|
|
|
transform (chunk, encoding, callback) {
|
|
|
|
|
marker++
|
|
|
|
|
@ -225,7 +224,7 @@ module.exports = {
|
|
|
|
|
const pagesProgressMultiplier = progressMultiplier / Math.ceil(pagesCount / 10)
|
|
|
|
|
WIKI.logger.info(`Found ${pagesCount} pages history to export. Streaming to file...`)
|
|
|
|
|
|
|
|
|
|
const rs = stream.Readable({ objectMode: true })
|
|
|
|
|
const rs = Readable({ objectMode: true })
|
|
|
|
|
rs._read = () => {}
|
|
|
|
|
|
|
|
|
|
const fetchPagesBatch = async (offset) => {
|
|
|
|
|
@ -255,7 +254,7 @@ module.exports = {
|
|
|
|
|
let marker = 0
|
|
|
|
|
await pipeline(
|
|
|
|
|
rs,
|
|
|
|
|
new stream.Transform({
|
|
|
|
|
new Transform({
|
|
|
|
|
objectMode: true,
|
|
|
|
|
transform (chunk, encoding, callback) {
|
|
|
|
|
marker++
|
|
|
|
|
@ -307,7 +306,7 @@ module.exports = {
|
|
|
|
|
const pagesProgressMultiplier = progressMultiplier / Math.ceil(pagesCount / 10)
|
|
|
|
|
WIKI.logger.info(`Found ${pagesCount} pages to export. Streaming to file...`)
|
|
|
|
|
|
|
|
|
|
const rs = stream.Readable({ objectMode: true })
|
|
|
|
|
const rs = Readable({ objectMode: true })
|
|
|
|
|
rs._read = () => {}
|
|
|
|
|
|
|
|
|
|
const fetchPagesBatch = async (offset) => {
|
|
|
|
|
@ -337,7 +336,7 @@ module.exports = {
|
|
|
|
|
let marker = 0
|
|
|
|
|
await pipeline(
|
|
|
|
|
rs,
|
|
|
|
|
new stream.Transform({
|
|
|
|
|
new Transform({
|
|
|
|
|
objectMode: true,
|
|
|
|
|
transform (chunk, encoding, callback) {
|
|
|
|
|
marker++
|
|
|
|
|
@ -400,7 +399,7 @@ module.exports = {
|
|
|
|
|
const usersProgressMultiplier = progressMultiplier / Math.ceil(usersCount / 50)
|
|
|
|
|
WIKI.logger.info(`Found ${usersCount} users to export. Streaming to file...`)
|
|
|
|
|
|
|
|
|
|
const rs = stream.Readable({ objectMode: true })
|
|
|
|
|
const rs = Readable({ objectMode: true })
|
|
|
|
|
rs._read = () => {}
|
|
|
|
|
|
|
|
|
|
const fetchUsersBatch = async (offset) => {
|
|
|
|
|
@ -427,7 +426,7 @@ module.exports = {
|
|
|
|
|
let marker = 0
|
|
|
|
|
await pipeline(
|
|
|
|
|
rs,
|
|
|
|
|
new stream.Transform({
|
|
|
|
|
new Transform({
|
|
|
|
|
objectMode: true,
|
|
|
|
|
transform (chunk, encoding, callback) {
|
|
|
|
|
marker++
|
|
|
|
|
|