feat: AWS S3 + Digitalocean Spaces storage modules (#1015)

* Provide basic implementation of AWS S3 storage module

* Abstract S3 Compatible Storage Module logic

* Refactor `getFileExtension()` into the `page` object

* Add implementation for Digitalocean storage module

* Remove accidental `async`/`await` in S3 Storage Module

* Remove argument from the call to `page.getFileExtension()`

https://github.com/Requarks/wiki/pull/1015#discussion_r321990073
pull/1066/head
Andrew Sim 5 years ago committed by Nicolas Giard
parent 3ab7bcf8ea
commit 5202eadebb

@ -147,6 +147,20 @@ module.exports = class Page extends Model {
return pageHelper.injectPageMetadata(this) return pageHelper.injectPageMetadata(this)
} }
/**
* Get the page's file extension based on content type
*/
getFileExtension() {
switch (this.contentType) {
case 'markdown':
return 'md'
case 'html':
return 'html'
default:
return 'txt'
}
}
/** /**
* Parse injected page metadata from raw content * Parse injected page metadata from raw content
* *

@ -1,28 +1,28 @@
key: digitalocean key: digitalocean
title: DigitalOcean Spaces title: DigitalOcean Spaces
description: DigitalOcean provides developers and businesses a reliable, easy-to-use cloud computing platform of virtual servers (Droplets), object storage (Spaces) and more. description: DigitalOcean provides developers and businesses a reliable, easy-to-use cloud computing platform of virtual servers (Droplets), object storage (Spaces) and more.
author: requarks.io author: andrewsim
logo: https://static.requarks.io/logo/digitalocean.svg logo: https://static.requarks.io/logo/digitalocean.svg
website: https://www.digitalocean.com/products/spaces/ website: https://www.digitalocean.com/products/spaces/
isAvailable: false isAvailable: true
supportedModes: supportedModes:
- push - push
defaultMode: push defaultMode: push
schedule: false schedule: false
props: props:
region: endpoint:
type: String type: String
title: Region title: Endpoint
hint: The DigitalOcean datacenter region where the Space will be created. hint: The DigitalOcean spaces endpoint that has the form ${REGION}.digitaloceanspaces.com
default: nyc3 default: nyc3.digitaloceanspaces.com
enum: enum:
- ams3 - ams3.digitaloceanspaces.com
- fra1 - fra1.digitaloceanspaces.com
- nyc3 - nyc3.digitaloceanspaces.com
- sfo2 - sfo2.digitaloceanspaces.com
- sgp1 - sgp1.digitaloceanspaces.com
order: 1 order: 1
spaceId: bucket:
type: String type: String
title: Space Unique Name title: Space Unique Name
hint: The unique space name to create (e.g. wiki-johndoe) hint: The unique space name to create (e.g. wiki-johndoe)

@ -1,23 +1,3 @@
module.exports = { const S3CompatibleStorage = require('../s3/common')
async activated() {
}, module.exports = new S3CompatibleStorage('Digitalocean')
async deactivated() {
},
async init() {
},
async created() {
},
async updated() {
},
async deleted() {
},
async renamed() {
}
}

@ -10,20 +10,6 @@ const moment = require('moment')
/* global WIKI */ /* global WIKI */
/**
* Get file extension based on content type
*/
const getFileExtension = (contentType) => {
switch (contentType) {
case 'markdown':
return 'md'
case 'html':
return 'html'
default:
return 'txt'
}
}
module.exports = { module.exports = {
async activated() { async activated() {
// not used // not used
@ -58,7 +44,7 @@ module.exports = {
}, },
async created(page) { async created(page) {
WIKI.logger.info(`(STORAGE/DISK) Creating file ${page.path}...`) WIKI.logger.info(`(STORAGE/DISK) Creating file ${page.path}...`)
let fileName = `${page.path}.${getFileExtension(page.contentType)}` let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.code !== page.localeCode) { if (WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}` fileName = `${page.localeCode}/${fileName}`
} }
@ -67,7 +53,7 @@ module.exports = {
}, },
async updated(page) { async updated(page) {
WIKI.logger.info(`(STORAGE/DISK) Updating file ${page.path}...`) WIKI.logger.info(`(STORAGE/DISK) Updating file ${page.path}...`)
let fileName = `${page.path}.${getFileExtension(page.contentType)}` let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.code !== page.localeCode) { if (WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}` fileName = `${page.localeCode}/${fileName}`
} }
@ -76,7 +62,7 @@ module.exports = {
}, },
async deleted(page) { async deleted(page) {
WIKI.logger.info(`(STORAGE/DISK) Deleting file ${page.path}...`) WIKI.logger.info(`(STORAGE/DISK) Deleting file ${page.path}...`)
let fileName = `${page.path}.${getFileExtension(page.contentType)}` let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.code !== page.localeCode) { if (WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}` fileName = `${page.localeCode}/${fileName}`
} }
@ -85,8 +71,8 @@ module.exports = {
}, },
async renamed(page) { async renamed(page) {
WIKI.logger.info(`(STORAGE/DISK) Renaming file ${page.sourcePath} to ${page.destinationPath}...`) WIKI.logger.info(`(STORAGE/DISK) Renaming file ${page.sourcePath} to ${page.destinationPath}...`)
let sourceFilePath = `${page.sourcePath}.${getFileExtension(page.contentType)}` let sourceFilePath = `${page.sourcePath}.${page.getFileExtension()}`
let destinationFilePath = `${page.destinationPath}.${getFileExtension(page.contentType)}` let destinationFilePath = `${page.destinationPath}.${page.getFileExtension()}`
if (WIKI.config.lang.code !== page.localeCode) { if (WIKI.config.lang.code !== page.localeCode) {
sourceFilePath = `${page.localeCode}/${sourceFilePath}` sourceFilePath = `${page.localeCode}/${sourceFilePath}`
@ -107,7 +93,7 @@ module.exports = {
new stream.Transform({ new stream.Transform({
objectMode: true, objectMode: true,
transform: async (page, enc, cb) => { transform: async (page, enc, cb) => {
let fileName = `${page.path}.${getFileExtension(page.contentType)}` let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.code !== page.localeCode) { if (WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}` fileName = `${page.localeCode}/${fileName}`
} }

@ -12,20 +12,6 @@ const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i
/* global WIKI */ /* global WIKI */
/**
* Get file extension based on content type
*/
const getFileExtension = (contentType) => {
switch (contentType) {
case 'markdown':
return 'md'
case 'html':
return 'html'
default:
return 'txt'
}
}
const getContenType = (filePath) => { const getContenType = (filePath) => {
const ext = _.last(filePath.split('.')) const ext = _.last(filePath.split('.'))
switch (ext) { switch (ext) {
@ -256,7 +242,7 @@ module.exports = {
*/ */
async created(page) { async created(page) {
WIKI.logger.info(`(STORAGE/GIT) Committing new file ${page.path}...`) WIKI.logger.info(`(STORAGE/GIT) Committing new file ${page.path}...`)
let fileName = `${page.path}.${getFileExtension(page.contentType)}` let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}` fileName = `${page.localeCode}/${fileName}`
} }
@ -275,7 +261,7 @@ module.exports = {
*/ */
async updated(page) { async updated(page) {
WIKI.logger.info(`(STORAGE/GIT) Committing updated file ${page.path}...`) WIKI.logger.info(`(STORAGE/GIT) Committing updated file ${page.path}...`)
let fileName = `${page.path}.${getFileExtension(page.contentType)}` let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}` fileName = `${page.localeCode}/${fileName}`
} }
@ -294,7 +280,7 @@ module.exports = {
*/ */
async deleted(page) { async deleted(page) {
WIKI.logger.info(`(STORAGE/GIT) Committing removed file ${page.path}...`) WIKI.logger.info(`(STORAGE/GIT) Committing removed file ${page.path}...`)
let fileName = `${page.path}.${getFileExtension(page.contentType)}` let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}` fileName = `${page.localeCode}/${fileName}`
} }
@ -311,8 +297,8 @@ module.exports = {
*/ */
async renamed(page) { async renamed(page) {
WIKI.logger.info(`(STORAGE/GIT) Committing file move from ${page.sourcePath} to ${page.destinationPath}...`) WIKI.logger.info(`(STORAGE/GIT) Committing file move from ${page.sourcePath} to ${page.destinationPath}...`)
let sourceFilePath = `${page.sourcePath}.${getFileExtension(page.contentType)}` let sourceFilePath = `${page.sourcePath}.${page.getFileExtension()}`
let destinationFilePath = `${page.destinationPath}.${getFileExtension(page.contentType)}` let destinationFilePath = `${page.destinationPath}.${page.getFileExtension()}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
sourceFilePath = `${page.localeCode}/${sourceFilePath}` sourceFilePath = `${page.localeCode}/${sourceFilePath}`
@ -363,7 +349,7 @@ module.exports = {
new stream.Transform({ new stream.Transform({
objectMode: true, objectMode: true,
transform: async (page, enc, cb) => { transform: async (page, enc, cb) => {
let fileName = `${page.path}.${getFileExtension(page.contentType)}` let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}` fileName = `${page.localeCode}/${fileName}`
} }

@ -0,0 +1,64 @@
const S3 = require('aws-sdk/clients/s3')
/* global WIKI */
/**
* Deduce the file path given the `page` object and the object's key to the page's path.
*/
const getFilePath = (page, pathKey) => {
const fileName = `${page[pathKey]}.${page.getFileExtension()}`
const withLocaleCode = WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode
return withLocaleCode ? `${page.localeCode}/${fileName}` : fileName
}
/**
* Can be used with S3 compatible storage.
*/
module.exports = class S3CompatibleStorage {
constructor(storageName) {
this.storageName = storageName
}
async activated() {
// not used
}
async deactivated() {
// not used
}
async init() {
WIKI.logger.info(`(STORAGE/${this.storageName}) Initializing...`)
const { accessKeyId, secretAccessKey, region, bucket, endpoint } = this.config
this.s3 = new S3({
accessKeyId,
secretAccessKey,
region,
endpoint,
params: { Bucket: bucket },
apiVersions: '2006-03-01'
})
// determine if a bucket exists and you have permission to access it
await this.s3.headBucket().promise()
WIKI.logger.info(`(STORAGE/${this.storageName}) Initialization completed.`)
}
async created(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Creating file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise()
}
async updated(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Updating file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise()
}
async deleted(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Deleting file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.deleteObject({ Key: filePath }).promise()
}
async renamed(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file ${page.sourcePath} to ${page.destinationPath}...`)
const sourceFilePath = getFilePath(page, 'sourcePath')
const destinationFilePath = getFilePath(page, 'destinationPath')
await this.s3.copyObject({ CopySource: sourceFilePath, Key: destinationFilePath }).promise()
await this.s3.deleteObject({ Key: sourceFilePath }).promise()
}
}

@ -1,11 +1,32 @@
key: s3 key: s3
title: Amazon S3 title: Amazon S3
description: Amazon S3 is a cloud computing web service offered by Amazon Web Services which provides object storage. description: Amazon S3 is a cloud computing web service offered by Amazon Web Services which provides object storage.
author: requarks.io author: andrewsim
logo: https://static.requarks.io/logo/aws-s3.svg logo: https://static.requarks.io/logo/aws-s3.svg
website: https://aws.amazon.com/s3/ website: https://aws.amazon.com/s3/
isAvailable: true
supportedModes:
- push
defaultMode: push
schedule: false
props: props:
accessKeyId: String region:
accessSecret: String type: String
region: String title: Region
bucket: String hint: The AWS datacenter region where the bucket will be created.
order: 1
bucket:
type: String
title: Unique bucket name
hint: The unique bucket name to create (e.g. wiki-johndoe).
order: 2
accessKeyId:
type: String
title: Access Key ID
hint: The Access Key.
order: 3
secretAccessKey:
type: String
title: Secret Access Key
hint: The Secret Access Key for the Access Key ID you created above.
order: 4

@ -1,23 +1,3 @@
module.exports = { const S3CompatibleStorage = require('./common')
async activated() {
}, module.exports = new S3CompatibleStorage('S3')
async deactivated() {
},
async init() {
},
async created() {
},
async updated() {
},
async deleted() {
},
async renamed() {
}
}

Loading…
Cancel
Save