fix: stream assets from storage local locations (#2087)

server/models/assets.js

@@ -6,6 +6,7 @@ const path = require('path')
 const fs = require('fs-extra')
 const _ = require('lodash')
 const assetHelper = require('../helpers/asset')
+const Promise = require('bluebird')
 
 /**
  * Users model
@@ -150,32 +151,53 @@ module.exports = class Asset extends Model {
   }
 
   static async getAsset(assetPath, res) {
-    let assetExists = await WIKI.models.assets.getAssetFromCache(assetPath, res)
+    try {
-    if (!assetExists) {
-      await WIKI.models.assets.getAssetFromDb(assetPath, res)
-    }
-  }
-
-  static async getAssetFromCache(assetPath, res) {
       const fileHash = assetHelper.generateHash(assetPath)
       const cachePath = path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, `cache/${fileHash}.dat`)
+      if (await WIKI.models.assets.getAssetFromCache(assetPath, cachePath, res)) {
+        return
+      }
+      if (await WIKI.models.assets.getAssetFromStorage(assetPath, res)) {
+        return
+      }
+      await WIKI.models.assets.getAssetFromDb(assetPath, fileHash, cachePath, res)
+    } catch (err) {
+      if (err.code === `ECONNABORTED` || err.code === `EPIPE`) {
+        return
+      }
+      WIKI.logger.error(err)
+      res.sendStatus(500)
+    }
+  }
 
-    return new Promise((resolve, reject) => {
+  static async getAssetFromCache(assetPath, cachePath, res) {
+    try {
+      await fs.access(cachePath, fs.constants.R_OK)
+    } catch (err) {
+      return false
+    }
+    const sendFile = Promise.promisify(res.sendFile, {context: res})
     res.type(path.extname(assetPath))
-      res.sendFile(cachePath, { dotfiles: 'deny' }, err => {
+    await sendFile(cachePath, { dotfiles: 'deny' })
-        if (err) {
+    return true
-          resolve(false)
+  }
-        } else {
+
-          resolve(true)
+  static async getAssetFromStorage(assetPath, res) {
+    const localLocations = await WIKI.models.storage.getLocalLocations({
+      asset: {
+        path: assetPath,
       }
     })
-    })
+    for (let location of _.filter(localLocations, location => Boolean(location.path))) {
+      const assetExists = await WIKI.models.assets.getAssetFromCache(assetPath, location.path, res)
+      if (assetExists) {
+        return true
+      }
+    }
+    return false
   }
 
-  static async getAssetFromDb(assetPath, res) {
+  static async getAssetFromDb(assetPath, fileHash, cachePath, res) {
-    const fileHash = assetHelper.generateHash(assetPath)
-    const cachePath = path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, `cache/${fileHash}.dat`)
-
     const asset = await WIKI.models.assets.query().where('hash', fileHash).first()
     if (asset) {
       const assetData = await WIKI.models.knex('assetData').where('id', asset.id).first()

server/models/storage.js

@@ -199,6 +199,23 @@ module.exports = class Storage extends Model {
     }
   }
 
+  static async getLocalLocations({ asset }) {
+    const locations = []
+    const promises = this.targets.map(async (target) => {
+      try {
+        const path = await target.fn.getLocalLocation(asset)
+        locations.push({
+          path,
+          key: target.key
+        })
+      } catch (err) {
+        WIKI.logger.warn(err)
+      }
+    })
+    await Promise.all(promises)
+    return locations
+  }
+
   static async executeAction(targetKey, handler) {
     try {
       const target = _.find(this.targets, ['key', targetKey])

server/modules/storage/azure/storage.js

@@ -114,6 +114,9 @@ module.exports = {
     await sourceBlockBlobClient.delete({
       deleteSnapshots: 'include'
     })
+  },
+  async getLocalLocation () {
+
   },
   /**
    * HANDLERS

server/modules/storage/box/storage.js

@@ -19,5 +19,8 @@ module.exports = {
   },
   async renamed() {
 
+  },
+  async getLocalLocation () {
+
   }
 }

server/modules/storage/disk/storage.js

@@ -116,7 +116,9 @@ module.exports = {
     WIKI.logger.info(`(STORAGE/DISK) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
     await fs.move(path.join(this.config.path, asset.path), path.join(this.config.path, asset.destinationPath), { overwrite: true })
   },
-
+  async getLocalLocation (asset) {
+    return path.join(this.config.path, asset.path)
+  },
   /**
    * HANDLERS
    */

server/modules/storage/dropbox/storage.js

@@ -19,5 +19,8 @@ module.exports = {
   },
   async renamed() {
 
+  },
+  async getLocalLocation () {
+
   }
 }

server/modules/storage/gdrive/storage.js

@@ -19,5 +19,8 @@ module.exports = {
   },
   async renamed() {
 
+  },
+  async getLocalLocation () {
+
   }
 }

server/modules/storage/git/storage.js

@@ -363,6 +363,9 @@ module.exports = {
       '--author': `"${asset.moveAuthorName} <${asset.moveAuthorEmail}>"`
     })
   },
+  async getLocalLocation (asset) {
+    return path.join(this.repoPath, asset.path)
+  },
   /**
    * HANDLERS
    */

server/modules/storage/onedrive/storage.js

@@ -19,5 +19,8 @@ module.exports = {
   },
   async renamed() {
 
+  },
+  async getLocalLocation () {
+
   }
 }

server/modules/storage/s3/common.js

@@ -119,6 +119,9 @@ module.exports = class S3CompatibleStorage {
     WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
     await this.s3.copyObject({ CopySource: asset.path, Key: asset.destinationPath }).promise()
     await this.s3.deleteObject({ Key: asset.path }).promise()
+  }
+  async getLocalLocation () {
+
   }
   /**
    * HANDLERS

server/modules/storage/sftp/storage.js

@@ -103,6 +103,9 @@ module.exports = {
     WIKI.logger.info(`(STORAGE/SFTP) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
     await this.ensureDirectory(asset.destinationPath)
     await this.sftp.rename(path.posix.join(this.config.basePath, asset.path), path.posix.join(this.config.basePath, asset.destinationPath))
+  },
+  async getLocalLocation () {
+
   },
   /**
    * HANDLERS