feat: azure blob storage provider + s3 rename fix

5 years ago · 4a2f1d045b
parent 14c842ff9d
commit 4a2f1d045b
5 changed files with 283 additions and 17 deletions
--- a/package.json
+++ b/package.json
@ -36,6 +36,7 @@
  },
  "dependencies": {
    "@aoberoi/passport-slack": "1.0.5",
+    "@azure/storage-blob": "12.0.1",
    "@bugsnag/js": "6.4.3",
    "@exlinc/keycloak-passport": "1.0.2",
    "algoliasearch": "3.35.1",
--- a/server/modules/storage/azure/definition.yml
+++ b/server/modules/storage/azure/definition.yml
@ -4,7 +4,43 @@ description: Azure Blob Storage by Microsoft provides massively scalable object
 author: requarks.io
 logo: https://static.requarks.io/logo/azure.svg
 website: https://azure.microsoft.com/services/storage/blobs/
+isAvailable: true
+supportedModes:
+  - push
+defaultMode: push
+schedule: false
 props:
-  accountName: String
-  accountKey: String
-  container: String
+  accountName:
+    type: String
+    title: Account Name
+    default: ''
+    hint: Your unique account name.
+    order: 1
+  accountKey:
+    type: String
+    title: Account Access Key
+    default: ''
+    hint: Either key 1 or key 2.
+    order: 2
+  containerName:
+    type: String
+    title: Container Name
+    default: 'wiki'
+    hint: Will automatically be created if it doesn't exist yet.
+    order: 3
+  storageTier:
+    type: String
+    title: Storage Tier
+    hint: Represents the access tier on a blob. Use Cool for lower storage costs but at higher retrieval costs.
+    order: 4
+    default: 'Cool'
+    enum:
+        - 'Hot'
+        - 'Cool'
+actions:
+  - handler: exportAll
+    label: Export All
+    hint: Output all content from the DB to Azure Blog Storage, overwriting any existing data. If you enabled Azure Blog Storage after content was created or you temporarily disabled Git, you'll want to execute this action to add the missing content.
+  - handler: importAll
+    label: Import Everything
+    hint: Will import all content currently in Azure Blog Storage. Useful for importing or restoring content from a previously backed up state.
--- a/server/modules/storage/azure/storage.js
+++ b/server/modules/storage/azure/storage.js
@ -1,3 +1,14 @@
+const { BlobServiceClient, StorageSharedKeyCredential } = require('@azure/storage-blob')
+const pageHelper = require('../../../helpers/page.js')
+
+/* global WIKI */
+
+const getFilePath = (page, pathKey) => {
+  const fileName = `${page[pathKey]}.${pageHelper.getFileExtension(page.contentType)}`
+  const withLocaleCode = WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode
+  return withLocaleCode ? `${page.localeCode}/${fileName}` : fileName
+}
+
 module.exports = {
  async activated() {

@ -6,18 +17,98 @@ module.exports = {

  },
  async init() {
-
+    WIKI.logger.info(`(STORAGE/AZURE) Initializing...`)
+    const { accountName, accountKey, containerName } = this.config
+    this.client = new BlobServiceClient(
+      `https://${accountName}.blob.core.windows.net`,
+      new StorageSharedKeyCredential(accountName, accountKey)
+    )
+    this.container = this.client.getContainerClient(containerName)
+    try {
+      await this.container.create()
+    } catch (err) {
+      if (err.statusCode !== 409) {
+        WIKI.logger.warn(err)
+        throw err
+      }
+    }
+    WIKI.logger.info(`(STORAGE/AZURE) Initialization completed.`)
  },
-  async created() {
-
+  async created (page) {
+    WIKI.logger.info(`(STORAGE/AZURE) Creating file ${page.path}...`)
+    const filePath = getFilePath(page, 'path')
+    const pageContent = page.injectMetadata()
+    const blockBlobClient = this.container.getBlockBlobClient(filePath)
+    await blockBlobClient.upload(pageContent, pageContent.length, { tier: this.config.storageTier })
  },
-  async updated() {
-
+  async updated (page) {
+    WIKI.logger.info(`(STORAGE/AZURE) Updating file ${page.path}...`)
+    const filePath = getFilePath(page, 'path')
+    const pageContent = page.injectMetadata()
+    const blockBlobClient = this.container.getBlockBlobClient(filePath)
+    await blockBlobClient.upload(pageContent, pageContent.length, { tier: this.config.storageTier })
  },
-  async deleted() {
-
+  async deleted (page) {
+    WIKI.logger.info(`(STORAGE/AZURE) Deleting file ${page.path}...`)
+    const filePath = getFilePath(page, 'path')
+    const blockBlobClient = this.container.getBlockBlobClient(filePath)
+    await blockBlobClient.delete({
+      deleteSnapshots: 'include'
+    })
  },
-  async renamed() {
-
+  async renamed(page) {
+    WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file ${page.path} to ${page.destinationPath}...`)
+    let sourceFilePath = getFilePath(page, 'path')
+    let destinationFilePath = getFilePath(page, 'destinationPath')
+    if (WIKI.config.lang.namespacing) {
+      if (WIKI.config.lang.code !== page.localeCode) {
+        sourceFilePath = `${page.localeCode}/${sourceFilePath}`
+      }
+      if (WIKI.config.lang.code !== page.destinationLocaleCode) {
+        destinationFilePath = `${page.destinationLocaleCode}/${destinationFilePath}`
+      }
+    }
+    const sourceBlockBlobClient = this.container.getBlockBlobClient(sourceFilePath)
+    const destBlockBlobClient = this.container.getBlockBlobClient(destinationFilePath)
+    await destBlockBlobClient.syncCopyFromURL(sourceBlockBlobClient.url)
+    await sourceBlockBlobClient.delete({
+      deleteSnapshots: 'include'
+    })
+  },
+  /**
+   * ASSET UPLOAD
+   *
+   * @param {Object} asset Asset to upload
+   */
+  async assetUploaded (asset) {
+    WIKI.logger.info(`(STORAGE/AZURE) Creating new file ${asset.path}...`)
+    const blockBlobClient = this.container.getBlockBlobClient(asset.path)
+    await blockBlobClient.upload(asset.data, asset.data.length, { tier: this.config.storageTier })
+  },
+  /**
+   * ASSET DELETE
+   *
+   * @param {Object} asset Asset to delete
+   */
+  async assetDeleted (asset) {
+    WIKI.logger.info(`(STORAGE/AZURE) Deleting file ${asset.path}...`)
+    const blockBlobClient = this.container.getBlockBlobClient(asset.path)
+    await blockBlobClient.delete({
+      deleteSnapshots: 'include'
+    })
+  },
+  /**
+   * ASSET RENAME
+   *
+   * @param {Object} asset Asset to rename
+   */
+  async assetRenamed (asset) {
+    WIKI.logger.info(`(STORAGE/AZURE) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
+    const sourceBlockBlobClient = this.container.getBlockBlobClient(asset.path)
+    const destBlockBlobClient = this.container.getBlockBlobClient(asset.destinationPath)
+    await destBlockBlobClient.syncCopyFromURL(sourceBlockBlobClient.url)
+    await sourceBlockBlobClient.delete({
+      deleteSnapshots: 'include'
+    })
  }
 }
--- a/server/modules/storage/s3/common.js
+++ b/server/modules/storage/s3/common.js
@ -57,8 +57,8 @@ module.exports = class S3CompatibleStorage {
  }
  async renamed(page) {
    WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file ${page.path} to ${page.destinationPath}...`)
-    let sourceFilePath = `${page.path}.${page.getFileExtension()}`
-    let destinationFilePath = `${page.destinationPath}.${page.getFileExtension()}`
+    let sourceFilePath = getFilePath(page, 'path')
+    let destinationFilePath = getFilePath(page, 'destinationPath')
    if (WIKI.config.lang.namespacing) {
      if (WIKI.config.lang.code !== page.localeCode) {
        sourceFilePath = `${page.localeCode}/${sourceFilePath}`
--- a/yarn.lock
+++ b/yarn.lock
@ -45,6 +45,82 @@
  resolved "https://registry.yarnpkg.com/@apollographql/graphql-playground-html/-/graphql-playground-html-1.6.24.tgz#3ce939cb127fb8aaa3ffc1e90dff9b8af9f2e3dc"
  integrity sha512-8GqG48m1XqyXh4mIZrtB5xOhUwSsh1WsrrsaZQOEYYql3YN9DEu9OOSg0ILzXHZo/h2Q74777YE4YzlArQzQEQ==

+"@azure/abort-controller@^1.0.0":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@azure/abort-controller/-/abort-controller-1.0.1.tgz#8510935b25ac051e58920300e9d7b511ca6e656a"
+  integrity sha512-wP2Jw6uPp8DEDy0n4KNidvwzDjyVV2xnycEIq7nPzj1rHyb/r+t3OPeNT1INZePP2wy5ZqlwyuyOMTi0ePyY1A==
+  dependencies:
+    tslib "^1.9.3"
+
+"@azure/core-asynciterator-polyfill@^1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@azure/core-asynciterator-polyfill/-/core-asynciterator-polyfill-1.0.0.tgz#dcccebb88406e5c76e0e1d52e8cc4c43a68b3ee7"
+  integrity sha512-kmv8CGrPfN9SwMwrkiBK9VTQYxdFQEGe0BmQk+M8io56P9KNzpAxcWE/1fxJj7uouwN4kXF0BHW8DNlgx+wtCg==
+
+"@azure/core-auth@^1.0.0":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@azure/core-auth/-/core-auth-1.0.2.tgz#7377c0cacf0e3c988ce321295bf5d2c174e0e288"
+  integrity sha512-zhPJObdrhz2ymIqGL1x8i3meEuaLz0UPjH9mOq9RGOlJB2Pb6K6xPtkHbRsfElgoO9USR4hH2XU5pLa4/JHHIw==
+  dependencies:
+    "@azure/abort-controller" "^1.0.0"
+    "@azure/core-tracing" "1.0.0-preview.7"
+    "@opentelemetry/types" "^0.2.0"
+    tslib "^1.9.3"
+
+"@azure/core-http@^1.0.0":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@azure/core-http/-/core-http-1.0.2.tgz#e281861492cf96459edd035c98ae234401cf8bfd"
+  integrity sha512-wjtY0Ks0+/4SEcIiMAvXdYvZZpyYj0rgs4dbd1MfgBlXgvvzpDOsPpcvI6ty5pyPBrjAyqrpyQeFud6hqsAnDQ==
+  dependencies:
+    "@azure/abort-controller" "^1.0.0"
+    "@azure/core-auth" "^1.0.0"
+    "@azure/core-tracing" "1.0.0-preview.7"
+    "@azure/logger" "^1.0.0"
+    "@opentelemetry/types" "^0.2.0"
+    "@types/node-fetch" "^2.5.0"
+    "@types/tunnel" "^0.0.1"
+    form-data "^3.0.0"
+    node-fetch "^2.6.0"
+    process "^0.11.10"
+    tough-cookie "^3.0.1"
+    tslib "^1.9.3"
+    tunnel "^0.0.6"
+    uuid "^3.3.2"
+    xml2js "^0.4.19"
+
+"@azure/core-lro@^1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@azure/core-lro/-/core-lro-1.0.0.tgz#9837398e03aa04b5b0b09158f4338861348dcce4"
+  integrity sha512-l4abIb8S9qmlv3bJkonLvgGSVQcSXq5jByA7Z28GRGJaQN/mSFal9YQOuLvVag+JXQJsoftuxJFrZiggF2TwOg==
+  dependencies:
+    "@azure/abort-controller" "^1.0.0"
+    "@azure/core-http" "^1.0.0"
+    events "^3.0.0"
+    tslib "^1.9.3"
+
+"@azure/core-paging@^1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@azure/core-paging/-/core-paging-1.0.0.tgz#3aa3855582154260326feea97f9f8322cbfe56d9"
+  integrity sha512-CzaT7LwxU97PZ+/Pn7uAbNGXY2mJ/3b56kmLsZzbR9stfrNfzlILxR94WHG/D1jZEQOk4lUNiaqJ2zP7nSGJhA==
+  dependencies:
+    "@azure/core-asynciterator-polyfill" "^1.0.0"
+
+"@azure/core-tracing@1.0.0-preview.7":
+  version "1.0.0-preview.7"
+  resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.0.0-preview.7.tgz#e9ee9c88f0dcf50d8e5b468fc827203165ecbc3f"
+  integrity sha512-pkFCw6OiJrpR+aH1VQe6DYm3fK2KWCC5Jf3m/Pv1RxF08M1Xm08RCyQ5Qe0YyW5L16yYT2nnV48krVhYZ6SGFA==
+  dependencies:
+    "@opencensus/web-types" "0.0.7"
+    "@opentelemetry/types" "^0.2.0"
+    tslib "^1.9.3"
+
+"@azure/logger@^1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@azure/logger/-/logger-1.0.0.tgz#48b371dfb34288c8797e5c104f6c4fb45bf1772c"
+  integrity sha512-g2qLDgvmhyIxR3JVS8N67CyIOeFRKQlX/llxYJQr1OSGQqM3HTpVP8MjmjcEKbL/OIt2N9C9UFaNQuKOw1laOA==
+  dependencies:
+    tslib "^1.9.3"
+
 "@azure/ms-rest-azure-env@^1.1.2":
  version "1.1.2"
  resolved "https://registry.yarnpkg.com/@azure/ms-rest-azure-env/-/ms-rest-azure-env-1.1.2.tgz#8505873afd4a1227ec040894a64fdd736b4a101f"
@ -73,6 +149,21 @@
    "@azure/ms-rest-js" "^1.8.7"
    adal-node "^0.1.28"

+"@azure/storage-blob@12.0.1":
+  version "12.0.1"
+  resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.0.1.tgz#9478f62e475347b0bf5edd773826fd9f07e4784a"
+  integrity sha512-NxMpOnN2jmWXgCZJqlGLuN5axXpi379PmVuCXWX6WLDwoY/jSJ1WOwWbS9h0Lgs/Lnd7rPrIijJEwUNQeGEtFw==
+  dependencies:
+    "@azure/abort-controller" "^1.0.0"
+    "@azure/core-http" "^1.0.0"
+    "@azure/core-lro" "^1.0.0"
+    "@azure/core-paging" "^1.0.0"
+    "@azure/core-tracing" "1.0.0-preview.7"
+    "@azure/logger" "^1.0.0"
+    "@opentelemetry/types" "^0.2.0"
+    events "^3.0.0"
+    tslib "^1.9.3"
+
 "@babel/cli@^7.7.5":
  version "7.7.5"
  resolved "https://registry.yarnpkg.com/@babel/cli/-/cli-7.7.5.tgz#25702cc65418efc06989af3727897b9f4c8690b6"
@ -1337,6 +1428,16 @@
  resolved "https://registry.yarnpkg.com/@mdi/font/-/font-4.7.95.tgz#46fddf35aad64dd623a8b1837f78ca4ed7bc48b1"
  integrity sha512-/SWooHIFz2dXkQJk3VhEXSbBplOU1lIkGSELAmw0peFEgR8KPqyM//M3vD8WDZETuEOSRVhVqLevP3okrsM5dw==

+"@opencensus/web-types@0.0.7":
+  version "0.0.7"
+  resolved "https://registry.yarnpkg.com/@opencensus/web-types/-/web-types-0.0.7.tgz#4426de1fe5aa8f624db395d2152b902874f0570a"
+  integrity sha512-xB+w7ZDAu3YBzqH44rCmG9/RlrOmFuDPt/bpf17eJr8eZSrLt7nc7LnWdxM9Mmoj/YKMHpxRg28txu3TcpiL+g==
+
+"@opentelemetry/types@^0.2.0":
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/@opentelemetry/types/-/types-0.2.0.tgz#2a0afd40fa7026e39ea56a454642bda72b172f80"
+  integrity sha512-GtwNB6BNDdsIPAYEdpp3JnOGO/3AJxjPvny53s3HERBdXSJTGQw8IRhiaTEX0b3w9P8+FwFZde4k+qkjn67aVw==
+
 "@panter/vue-i18next@0.15.1":
  version "0.15.1"
  resolved "https://registry.yarnpkg.com/@panter/vue-i18next/-/vue-i18next-0.15.1.tgz#3d27e57fa702611df52bb73042729df5c0ccde2b"
@ -1645,6 +1746,13 @@
  resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
  integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==

+"@types/node-fetch@^2.5.0":
+  version "2.5.4"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.4.tgz#5245b6d8841fc3a6208b82291119bc11c4e0ce44"
+  integrity sha512-Oz6id++2qAOFuOlE1j0ouk1dzl3mmI1+qINPNBhi9nt/gVOz0G+13Ao6qjhdF0Ys+eOkhu6JnFmt38bR3H0POQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/node@*", "@types/node@>=6":
  version "12.7.4"
  resolved "https://registry.yarnpkg.com/@types/node/-/node-12.7.4.tgz#64db61e0359eb5a8d99b55e05c729f130a678b04"
@ -1728,6 +1836,13 @@
  dependencies:
    "@types/node" "*"

+"@types/tunnel@^0.0.1":
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/@types/tunnel/-/tunnel-0.0.1.tgz#0d72774768b73df26f25df9184273a42da72b19c"
+  integrity sha512-AOqu6bQu5MSWwYvehMXLukFHnupHrpZ8nvgae5Ggie9UwzDR1CCwoXgSSWNZJuyOlCdfdsWMA5F2LlmvyoTv8A==
+  dependencies:
+    "@types/node" "*"
+
 "@types/uglify-js@*":
  version "3.0.4"
  resolved "https://registry.yarnpkg.com/@types/uglify-js/-/uglify-js-3.0.4.tgz#96beae23df6f561862a830b4288a49e86baac082"
@ -4100,7 +4215,7 @@ colorspace@1.1.x:
    color "3.0.x"
    text-hex "1.0.x"

-combined-stream@^1.0.6, combined-stream@~1.0.6:
+combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6:
  version "1.0.8"
  resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f"
  integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==
@ -6100,6 +6215,15 @@ form-data@^2.3.1, form-data@^2.3.2:
    combined-stream "^1.0.6"
    mime-types "^2.1.12"

+form-data@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.0.tgz#31b7e39c85f1355b7139ee0c647cf0de7f83c682"
+  integrity sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==
+  dependencies:
+    asynckit "^0.4.0"
+    combined-stream "^1.0.8"
+    mime-types "^2.1.12"
+
 form-data@~2.3.2:
  version "2.3.3"
  resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6"
@ -7071,6 +7195,11 @@ invert-kv@^2.0.0:
  resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02"
  integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==

+ip-regex@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9"
+  integrity sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=
+
 ipaddr.js@1.9.0:
  version "1.9.0"
  resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.0.tgz#37df74e430a0e47550fe54a2defe30d8acd95f65"
@ -9180,7 +9309,7 @@ node-fetch@1.7.3:
    encoding "^0.1.11"
    is-stream "^1.0.1"

-node-fetch@^2.1.2, node-fetch@^2.2.0, node-fetch@^2.3.0:
+node-fetch@^2.1.2, node-fetch@^2.2.0, node-fetch@^2.3.0, node-fetch@^2.6.0:
  version "2.6.0"
  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
  integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
@ -13491,6 +13620,15 @@ tough-cookie@^2.3.3, tough-cookie@^2.3.4, tough-cookie@^2.4.3:
    psl "^1.1.28"
    punycode "^2.1.1"

+tough-cookie@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-3.0.1.tgz#9df4f57e739c26930a018184887f4adb7dca73b2"
+  integrity sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==
+  dependencies:
+    ip-regex "^2.1.0"
+    psl "^1.1.28"
+    punycode "^2.1.1"
+
 tough-cookie@~2.4.3:
  version "2.4.3"
  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781"
@ -13552,7 +13690,7 @@ tunnel-agent@^0.6.0:
  dependencies:
    safe-buffer "^5.0.1"

-tunnel@0.0.6:
+tunnel@0.0.6, tunnel@^0.0.6:
  version "0.0.6"
  resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c"
  integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==