const path = require('path') const sgit = require('simple-git') const fs = require('fs-extra') const _ = require('lodash') const stream = require('stream') const Promise = require('bluebird') const pipeline = Promise.promisify(stream.pipeline) const klaw = require('klaw') const os = require('os') const pageHelper = require('../../../helpers/page') const assetHelper = require('../../../helpers/asset') const commonDisk = require('../disk/common') /* global WIKI */ module.exports = { git: null, repoPath: path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, 'repo'), async activated() { // not used }, async deactivated() { // not used }, /** * INIT */ async init() { WIKI.logger.info('(STORAGE/GIT) Initializing...') this.repoPath = path.resolve(WIKI.ROOTPATH, this.config.localRepoPath) await fs.ensureDir(this.repoPath) this.git = sgit(this.repoPath, { maxConcurrentProcesses: 1 }) // Set custom binary path if (!_.isEmpty(this.config.gitBinaryPath)) { this.git.customBinary(this.config.gitBinaryPath) } // Initialize repo (if needed) WIKI.logger.info('(STORAGE/GIT) Checking repository state...') const isRepo = await this.git.checkIsRepo() if (!isRepo) { WIKI.logger.info('(STORAGE/GIT) Initializing local repository...') await this.git.init() } // Disable quotePath, color output // Link https://git-scm.com/docs/git-config#Documentation/git-config.txt-corequotePath await this.git.raw(['config', '--local', 'core.quotepath', false]) await this.git.raw(['config', '--local', 'color.ui', false]) // Set default author await this.git.raw(['config', '--local', 'user.email', this.config.defaultEmail]) await this.git.raw(['config', '--local', 'user.name', this.config.defaultName]) // Purge existing remotes WIKI.logger.info('(STORAGE/GIT) Listing existing remotes...') const remotes = await this.git.getRemotes() if (remotes.length > 0) { WIKI.logger.info('(STORAGE/GIT) Purging existing remotes...') for (let remote of remotes) { await this.git.removeRemote(remote.name) } } // Add remote WIKI.logger.info('(STORAGE/GIT) Setting SSL Verification config...') await this.git.raw(['config', '--local', '--bool', 'http.sslVerify', _.toString(this.config.verifySSL)]) switch (this.config.authType) { case 'ssh': WIKI.logger.info('(STORAGE/GIT) Setting SSH Command config...') if (this.config.sshPrivateKeyMode === 'contents') { try { this.config.sshPrivateKeyPath = path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, 'secure/git-ssh.pem') await fs.outputFile(this.config.sshPrivateKeyPath, this.config.sshPrivateKeyContent + os.EOL, { encoding: 'utf8', mode: 0o600 }) } catch (err) { WIKI.logger.error(err) throw err } } await this.git.addConfig('core.sshCommand', `ssh -i "${this.config.sshPrivateKeyPath}" -o StrictHostKeyChecking=no`) WIKI.logger.info('(STORAGE/GIT) Adding origin remote via SSH...') await this.git.addRemote('origin', this.config.repoUrl) break default: WIKI.logger.info('(STORAGE/GIT) Adding origin remote via HTTP/S...') let originUrl = '' if (_.startsWith(this.config.repoUrl, 'http')) { originUrl = this.config.repoUrl.replace('://', `://${encodeURI(this.config.basicUsername)}:${encodeURI(this.config.basicPassword)}@`) } else { originUrl = `https://${encodeURI(this.config.basicUsername)}:${encodeURI(this.config.basicPassword)}@${this.config.repoUrl}` } await this.git.addRemote('origin', originUrl) break } // Fetch updates for remote WIKI.logger.info('(STORAGE/GIT) Fetch updates from remote...') await this.git.raw(['remote', 'update', 'origin']) // Checkout branch const branches = await this.git.branch() if (!_.includes(branches.all, this.config.branch) && !_.includes(branches.all, `remotes/origin/${this.config.branch}`)) { throw new Error('Invalid branch! Make sure it exists on the remote first.') } WIKI.logger.info(`(STORAGE/GIT) Checking out branch ${this.config.branch}...`) await this.git.checkout(this.config.branch) // Perform initial sync await this.sync() WIKI.logger.info('(STORAGE/GIT) Initialization completed.') }, /** * SYNC */ async sync() { const currentCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch, '--']), 'latest', {}) const rootUser = await WIKI.models.users.getRootUser() // Pull rebase if (_.includes(['sync', 'pull'], this.mode)) { WIKI.logger.info(`(STORAGE/GIT) Performing pull rebase from origin on branch ${this.config.branch}...`) await this.git.pull('origin', this.config.branch, ['--rebase']) } // Push if (_.includes(['sync', 'push'], this.mode)) { WIKI.logger.info(`(STORAGE/GIT) Performing push to origin on branch ${this.config.branch}...`) let pushOpts = ['--signed=if-asked'] if (this.mode === 'push') { pushOpts.push('--force') } await this.git.push('origin', this.config.branch, pushOpts) } // Process Changes if (_.includes(['sync', 'pull'], this.mode)) { const latestCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch, '--']), 'latest', {}) const diff = await this.git.diffSummary(['-M', currentCommitLog.hash, latestCommitLog.hash]) if (_.get(diff, 'files', []).length > 0) { let filesToProcess = [] const filePattern = /(.*?)(?:{(.*?))? => (?:(.*?)})?(.*)/ for (const f of diff.files) { const fMatch = f.file.match(filePattern) const fNames = { old: null, new: null } if (!fMatch) { fNames.old = f.file fNames.new = f.file } else if (!fMatch[2] && !fMatch[3]) { fNames.old = fMatch[1] fNames.new = fMatch[4] } else { fNames.old = (fMatch[1] + fMatch[2] + fMatch[4]).replace('//', '/') fNames.new = (fMatch[1] + fMatch[3] + fMatch[4]).replace('//', '/') } const fPath = path.join(this.repoPath, fNames.new) let fStats = { size: 0 } try { fStats = await fs.stat(fPath) } catch (err) { if (err.code !== 'ENOENT') { WIKI.logger.warn(`(STORAGE/GIT) Failed to access file ${f.file}! Skipping...`) continue } } filesToProcess.push({ ...f, file: { path: fPath, stats: fStats }, oldPath: fNames.old, relPath: fNames.new }) } await this.processFiles(filesToProcess, rootUser) } } }, /** * Process Files * * @param {Array} files Array of files to process */ async processFiles(files, user) { for (const item of files) { const contentType = pageHelper.getContentType(item.relPath) const fileExists = await fs.pathExists(item.file.path) if (!item.binary && contentType) { // -> Page if (fileExists && !item.importAll && item.relPath !== item.oldPath) { // Page was renamed by git, so rename in DB WIKI.logger.info(`(STORAGE/GIT) Page marked as renamed: from ${item.oldPath} to ${item.relPath}`) const contentPath = pageHelper.getPagePath(item.oldPath) const contentDestinationPath = pageHelper.getPagePath(item.relPath) await WIKI.models.pages.movePage({ user: user, path: contentPath.path, destinationPath: contentDestinationPath.path, locale: contentPath.locale, destinationLocale: contentPath.locale, skipStorage: true }) } else if (!fileExists && !item.importAll && item.deletions > 0 && item.insertions === 0) { // Page was deleted by git, can safely mark as deleted in DB WIKI.logger.info(`(STORAGE/GIT) Page marked as deleted: ${item.relPath}`) const contentPath = pageHelper.getPagePath(item.relPath) await WIKI.models.pages.deletePage({ user: user, path: contentPath.path, locale: contentPath.locale, skipStorage: true }) continue } try { await commonDisk.processPage({ user, relPath: item.relPath, fullPath: this.repoPath, contentType: contentType, moduleName: 'GIT' }) } catch (err) { WIKI.logger.warn(`(STORAGE/GIT) Failed to process ${item.relPath}`) WIKI.logger.warn(err) } } else { // -> Asset if (fileExists && !item.importAll && ((item.before === item.after) || (item.deletions === 0 && item.insertions === 0))) { // Asset was renamed by git, so rename in DB WIKI.logger.info(`(STORAGE/GIT) Asset marked as renamed: from ${item.oldPath} to ${item.relPath}`) const fileHash = assetHelper.generateHash(item.relPath) const assetToRename = await WIKI.models.assets.query().findOne({ hash: fileHash }) if (assetToRename) { await WIKI.models.assets.query().patch({ filename: item.relPath, hash: fileHash }).findById(assetToRename.id) await assetToRename.deleteAssetCache() } else { WIKI.logger.info(`(STORAGE/GIT) Asset was not found in the DB, nothing to rename: ${item.relPath}`) } continue } else if (!fileExists && !item.importAll && ((item.before > 0 && item.after === 0) || (item.deletions > 0 && item.insertions === 0))) { // Asset was deleted by git, can safely mark as deleted in DB WIKI.logger.info(`(STORAGE/GIT) Asset marked as deleted: ${item.relPath}`) const fileHash = assetHelper.generateHash(item.relPath) const assetToDelete = await WIKI.models.assets.query().findOne({ hash: fileHash }) if (assetToDelete) { await WIKI.models.knex('assetData').where('id', assetToDelete.id).del() await WIKI.models.assets.query().deleteById(assetToDelete.id) await assetToDelete.deleteAssetCache() } else { WIKI.logger.info(`(STORAGE/GIT) Asset was not found in the DB, nothing to delete: ${item.relPath}`) } continue } try { await commonDisk.processAsset({ user, relPath: item.relPath, file: item.file, contentType: contentType, moduleName: 'GIT' }) } catch (err) { WIKI.logger.warn(`(STORAGE/GIT) Failed to process asset ${item.relPath}`) WIKI.logger.warn(err) } } } }, /** * CREATE * * @param {Object} page Page to create */ async created(page) { WIKI.logger.info(`(STORAGE/GIT) Committing new file [${page.localeCode}] ${page.path}...`) let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}` if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } const filePath = path.join(this.repoPath, fileName) await fs.outputFile(filePath, page.injectMetadata(), 'utf8') const gitFilePath = `./${fileName}` if ((await this.git.checkIgnore(gitFilePath)).length === 0) { await this.git.add(gitFilePath) await this.git.commit(`docs: create ${page.path}`, fileName, { '--author': `"${page.authorName} <${page.authorEmail}>"` }) } }, /** * UPDATE * * @param {Object} page Page to update */ async updated(page) { WIKI.logger.info(`(STORAGE/GIT) Committing updated file [${page.localeCode}] ${page.path}...`) let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}` if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } const filePath = path.join(this.repoPath, fileName) await fs.outputFile(filePath, page.injectMetadata(), 'utf8') const gitFilePath = `./${fileName}` if ((await this.git.checkIgnore(gitFilePath)).length === 0) { await this.git.add(gitFilePath) await this.git.commit(`docs: update ${page.path}`, fileName, { '--author': `"${page.authorName} <${page.authorEmail}>"` }) } }, /** * DELETE * * @param {Object} page Page to delete */ async deleted(page) { WIKI.logger.info(`(STORAGE/GIT) Committing removed file [${page.localeCode}] ${page.path}...`) let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}` if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } const gitFilePath = `./${fileName}` if ((await this.git.checkIgnore(gitFilePath)).length === 0) { await this.git.rm(gitFilePath) await this.git.commit(`docs: delete ${page.path}`, fileName, { '--author': `"${page.authorName} <${page.authorEmail}>"` }) } }, /** * RENAME * * @param {Object} page Page to rename */ async renamed(page) { WIKI.logger.info(`(STORAGE/GIT) Committing file move from [${page.localeCode}] ${page.path} to [${page.destinationLocaleCode}] ${page.destinationPath}...`) let sourceFileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}` let destinationFileName = `${page.destinationPath}.${pageHelper.getFileExtension(page.contentType)}` if (WIKI.config.lang.namespacing) { if (WIKI.config.lang.code !== page.localeCode) { sourceFileName = `${page.localeCode}/${sourceFileName}` } if (WIKI.config.lang.code !== page.destinationLocaleCode) { destinationFileName = `${page.destinationLocaleCode}/${destinationFileName}` } } const sourceFilePath = path.join(this.repoPath, sourceFileName) const destinationFilePath = path.join(this.repoPath, destinationFileName) await fs.move(sourceFilePath, destinationFilePath) await this.git.rm(`./${sourceFileName}`) await this.git.add(`./${destinationFileName}`) await this.git.commit(`docs: rename ${page.path} to ${page.destinationPath}`, [sourceFilePath, destinationFilePath], { '--author': `"${page.moveAuthorName} <${page.moveAuthorEmail}>"` }) }, /** * ASSET UPLOAD * * @param {Object} asset Asset to upload */ async assetUploaded (asset) { WIKI.logger.info(`(STORAGE/GIT) Committing new file ${asset.path}...`) const filePath = path.join(this.repoPath, asset.path) await fs.outputFile(filePath, asset.data, 'utf8') await this.git.add(`./${asset.path}`) await this.git.commit(`docs: upload ${asset.path}`, asset.path, { '--author': `"${asset.authorName} <${asset.authorEmail}>"` }) }, /** * ASSET DELETE * * @param {Object} asset Asset to upload */ async assetDeleted (asset) { WIKI.logger.info(`(STORAGE/GIT) Committing removed file ${asset.path}...`) await this.git.rm(`./${asset.path}`) await this.git.commit(`docs: delete ${asset.path}`, asset.path, { '--author': `"${asset.authorName} <${asset.authorEmail}>"` }) }, /** * ASSET RENAME * * @param {Object} asset Asset to upload */ async assetRenamed (asset) { WIKI.logger.info(`(STORAGE/GIT) Committing file move from ${asset.path} to ${asset.destinationPath}...`) await this.git.mv(`./${asset.path}`, `./${asset.destinationPath}`) await this.git.commit(`docs: rename ${asset.path} to ${asset.destinationPath}`, [asset.path, asset.destinationPath], { '--author': `"${asset.moveAuthorName} <${asset.moveAuthorEmail}>"` }) }, async getLocalLocation (asset) { return path.join(this.repoPath, asset.path) }, /** * HANDLERS */ async importAll() { WIKI.logger.info(`(STORAGE/GIT) Importing all content from local Git repo to the DB...`) const rootUser = await WIKI.models.users.getRootUser() await pipeline( klaw(this.repoPath, { filter: (f) => { return !_.includes(f, '.git') } }), new stream.Transform({ objectMode: true, transform: async (file, enc, cb) => { const relPath = file.path.substr(this.repoPath.length + 1) if (file.stats.size < 1) { // Skip directories and zero-byte files return cb() } else if (relPath && relPath.length > 3) { WIKI.logger.info(`(STORAGE/GIT) Processing ${relPath}...`) await this.processFiles([{ user: rootUser, relPath, file, deletions: 0, insertions: 0, importAll: true }], rootUser) } cb() } }) ) commonDisk.clearFolderCache() WIKI.logger.info('(STORAGE/GIT) Import completed.') }, async syncUntracked() { WIKI.logger.info(`(STORAGE/GIT) Adding all untracked content...`) // -> Pages await pipeline( WIKI.models.knex.column('id', 'path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt', 'createdAt', 'editorKey').select().from('pages').where({ isPrivate: false }).stream(), new stream.Transform({ objectMode: true, transform: async (page, enc, cb) => { const pageObject = await WIKI.models.pages.query().findById(page.id) page.tags = await pageObject.$relatedQuery('tags') let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}` if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } WIKI.logger.info(`(STORAGE/GIT) Adding page ${fileName}...`) const filePath = path.join(this.repoPath, fileName) await fs.outputFile(filePath, pageHelper.injectPageMetadata(page), 'utf8') await this.git.add(`./${fileName}`) cb() } }) ) // -> Assets const assetFolders = await WIKI.models.assetFolders.getAllPaths() await pipeline( WIKI.models.knex.column('filename', 'folderId', 'data').select().from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(), new stream.Transform({ objectMode: true, transform: async (asset, enc, cb) => { const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename WIKI.logger.info(`(STORAGE/GIT) Adding asset ${filename}...`) await fs.outputFile(path.join(this.repoPath, filename), asset.data) await this.git.add(`./${filename}`) cb() } }) ) await this.git.commit(`docs: add all untracked content`) WIKI.logger.info('(STORAGE/GIT) All content is now tracked.') }, async purge() { WIKI.logger.info(`(STORAGE/GIT) Purging local repository...`) await fs.emptyDir(this.repoPath) WIKI.logger.info('(STORAGE/GIT) Local repository is now empty. Reinitializing...') await this.init() } }