You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
wiki/server/modules/storage/s3/common.js

164 lines
5.8 KiB

const S3 = require('aws-sdk/clients/s3')
const stream = require('stream')
const Promise = require('bluebird')
const pipeline = Promise.promisify(stream.pipeline)
const _ = require('lodash')
const pageHelper = require('../../../helpers/page.js')
/* global WIKI */
/**
* Deduce the file path given the `page` object and the object's key to the page's path.
*/
const getFilePath = (page, pathKey) => {
const fileName = `${page[pathKey]}.${pageHelper.getFileExtension(page.contentType)}`
const withLocaleCode = WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode
return withLocaleCode ? `${page.localeCode}/${fileName}` : fileName
}
/**
* Can be used with S3 compatible storage.
*/
module.exports = class S3CompatibleStorage {
constructor(storageName) {
this.storageName = storageName
}
async activated() {
// not used
}
async deactivated() {
// not used
}
async init() {
WIKI.logger.info(`(STORAGE/${this.storageName}) Initializing...`)
const { accessKeyId, secretAccessKey, bucket } = this.config
const s3Config = {
accessKeyId,
secretAccessKey,
params: { Bucket: bucket },
apiVersions: '2006-03-01'
}
if (!_.isNil(this.config.region)) {
s3Config.region = this.config.region
}
if (!_.isNil(this.config.endpoint)) {
s3Config.endpoint = this.config.endpoint
}
if (!_.isNil(this.config.sslEnabled)) {
s3Config.sslEnabled = this.config.sslEnabled
}
if (!_.isNil(this.config.s3ForcePathStyle)) {
s3Config.s3ForcePathStyle = this.config.s3ForcePathStyle
}
if (!_.isNil(this.config.s3BucketEndpoint)) {
s3Config.s3BucketEndpoint = this.config.s3BucketEndpoint
}
this.s3 = new S3(s3Config)
// determine if a bucket exists and you have permission to access it
await this.s3.headBucket().promise()
WIKI.logger.info(`(STORAGE/${this.storageName}) Initialization completed.`)
}
async created(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Creating file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise()
}
async updated(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Updating file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise()
}
async deleted(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Deleting file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.deleteObject({ Key: filePath }).promise()
}
async renamed(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file ${page.path} to ${page.destinationPath}...`)
let sourceFilePath = getFilePath(page, 'path')
let destinationFilePath = getFilePath(page, 'destinationPath')
if (WIKI.config.lang.namespacing) {
if (WIKI.config.lang.code !== page.localeCode) {
sourceFilePath = `${page.localeCode}/${sourceFilePath}`
}
if (WIKI.config.lang.code !== page.destinationLocaleCode) {
destinationFilePath = `${page.destinationLocaleCode}/${destinationFilePath}`
}
}
await this.s3.copyObject({ CopySource: sourceFilePath, Key: destinationFilePath }).promise()
await this.s3.deleteObject({ Key: sourceFilePath }).promise()
}
/**
* ASSET UPLOAD
*
* @param {Object} asset Asset to upload
*/
async assetUploaded (asset) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Creating new file ${asset.path}...`)
await this.s3.putObject({ Key: asset.path, Body: asset.data }).promise()
}
/**
* ASSET DELETE
*
* @param {Object} asset Asset to delete
*/
async assetDeleted (asset) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Deleting file ${asset.path}...`)
await this.s3.deleteObject({ Key: asset.path }).promise()
}
/**
* ASSET RENAME
*
* @param {Object} asset Asset to rename
*/
async assetRenamed (asset) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
await this.s3.copyObject({ CopySource: asset.path, Key: asset.destinationPath }).promise()
await this.s3.deleteObject({ Key: asset.path }).promise()
}
/**
* HANDLERS
*/
async exportAll() {
WIKI.logger.info(`(STORAGE/${this.storageName}) Exporting all content to the cloud provider...`)
// -> Pages
await pipeline(
WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({
isPrivate: false
}).stream(),
new stream.Transform({
objectMode: true,
transform: async (page, enc, cb) => {
const filePath = getFilePath(page, 'path')
WIKI.logger.info(`(STORAGE/${this.storageName}) Adding page ${filePath}...`)
await this.s3.putObject({ Key: filePath, Body: pageHelper.injectPageMetadata(page) }).promise()
cb()
}
})
)
// -> Assets
const assetFolders = await WIKI.models.assetFolders.getAllPaths()
await pipeline(
WIKI.models.knex.column('filename', 'folderId', 'data').select().from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(),
new stream.Transform({
objectMode: true,
transform: async (asset, enc, cb) => {
const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename
WIKI.logger.info(`(STORAGE/${this.storageName}) Adding asset ${filename}...`)
await this.s3.putObject({ Key: filename, Body: asset.data }).promise()
cb()
}
})
)
WIKI.logger.info(`(STORAGE/${this.storageName}) All content has been pushed to the cloud provider.`)
}
}