From b2ff064d34c8251fe717eee5a5c6603b075c0b1c Mon Sep 17 00:00:00 2001 From: Regev Brody Date: Sun, 12 Jul 2020 19:19:01 +0300 Subject: [PATCH] fix: stream assets from storage local locations (#2087) --- server/models/assets.js | 62 +++++++++++++++------- server/models/storage.js | 17 ++++++ server/modules/storage/azure/storage.js | 3 ++ server/modules/storage/box/storage.js | 3 ++ server/modules/storage/disk/storage.js | 4 +- server/modules/storage/dropbox/storage.js | 3 ++ server/modules/storage/gdrive/storage.js | 3 ++ server/modules/storage/git/storage.js | 3 ++ server/modules/storage/onedrive/storage.js | 3 ++ server/modules/storage/s3/common.js | 3 ++ server/modules/storage/sftp/storage.js | 3 ++ 11 files changed, 86 insertions(+), 21 deletions(-) diff --git a/server/models/assets.js b/server/models/assets.js index 514c29bd..396299fb 100644 --- a/server/models/assets.js +++ b/server/models/assets.js @@ -6,6 +6,7 @@ const path = require('path') const fs = require('fs-extra') const _ = require('lodash') const assetHelper = require('../helpers/asset') +const Promise = require('bluebird') /** * Users model @@ -150,32 +151,53 @@ module.exports = class Asset extends Model { } static async getAsset(assetPath, res) { - let assetExists = await WIKI.models.assets.getAssetFromCache(assetPath, res) - if (!assetExists) { - await WIKI.models.assets.getAssetFromDb(assetPath, res) + try { + const fileHash = assetHelper.generateHash(assetPath) + const cachePath = path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, `cache/${fileHash}.dat`) + if (await WIKI.models.assets.getAssetFromCache(assetPath, cachePath, res)) { + return + } + if (await WIKI.models.assets.getAssetFromStorage(assetPath, res)) { + return + } + await WIKI.models.assets.getAssetFromDb(assetPath, fileHash, cachePath, res) + } catch (err) { + if (err.code === `ECONNABORTED` || err.code === `EPIPE`) { + return + } + WIKI.logger.error(err) + res.sendStatus(500) } } - static async getAssetFromCache(assetPath, res) { - const fileHash = assetHelper.generateHash(assetPath) - const cachePath = path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, `cache/${fileHash}.dat`) - - return new Promise((resolve, reject) => { - res.type(path.extname(assetPath)) - res.sendFile(cachePath, { dotfiles: 'deny' }, err => { - if (err) { - resolve(false) - } else { - resolve(true) - } - }) - }) + static async getAssetFromCache(assetPath, cachePath, res) { + try { + await fs.access(cachePath, fs.constants.R_OK) + } catch (err) { + return false + } + const sendFile = Promise.promisify(res.sendFile, {context: res}) + res.type(path.extname(assetPath)) + await sendFile(cachePath, { dotfiles: 'deny' }) + return true } - static async getAssetFromDb(assetPath, res) { - const fileHash = assetHelper.generateHash(assetPath) - const cachePath = path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, `cache/${fileHash}.dat`) + static async getAssetFromStorage(assetPath, res) { + const localLocations = await WIKI.models.storage.getLocalLocations({ + asset: { + path: assetPath, + } + }) + for (let location of _.filter(localLocations, location => Boolean(location.path))) { + const assetExists = await WIKI.models.assets.getAssetFromCache(assetPath, location.path, res) + if (assetExists) { + return true + } + } + return false + } + static async getAssetFromDb(assetPath, fileHash, cachePath, res) { const asset = await WIKI.models.assets.query().where('hash', fileHash).first() if (asset) { const assetData = await WIKI.models.knex('assetData').where('id', asset.id).first() diff --git a/server/models/storage.js b/server/models/storage.js index cbfc58b0..a982b7ec 100644 --- a/server/models/storage.js +++ b/server/models/storage.js @@ -199,6 +199,23 @@ module.exports = class Storage extends Model { } } + static async getLocalLocations({ asset }) { + const locations = [] + const promises = this.targets.map(async (target) => { + try { + const path = await target.fn.getLocalLocation(asset) + locations.push({ + path, + key: target.key + }) + } catch (err) { + WIKI.logger.warn(err) + } + }) + await Promise.all(promises) + return locations + } + static async executeAction(targetKey, handler) { try { const target = _.find(this.targets, ['key', targetKey]) diff --git a/server/modules/storage/azure/storage.js b/server/modules/storage/azure/storage.js index 1228d767..69a453aa 100644 --- a/server/modules/storage/azure/storage.js +++ b/server/modules/storage/azure/storage.js @@ -114,6 +114,9 @@ module.exports = { await sourceBlockBlobClient.delete({ deleteSnapshots: 'include' }) + }, + async getLocalLocation () { + }, /** * HANDLERS diff --git a/server/modules/storage/box/storage.js b/server/modules/storage/box/storage.js index ab25ce97..8ae6cc29 100644 --- a/server/modules/storage/box/storage.js +++ b/server/modules/storage/box/storage.js @@ -19,5 +19,8 @@ module.exports = { }, async renamed() { + }, + async getLocalLocation () { + } } diff --git a/server/modules/storage/disk/storage.js b/server/modules/storage/disk/storage.js index 6fc916ac..34117378 100644 --- a/server/modules/storage/disk/storage.js +++ b/server/modules/storage/disk/storage.js @@ -116,7 +116,9 @@ module.exports = { WIKI.logger.info(`(STORAGE/DISK) Renaming file from ${asset.path} to ${asset.destinationPath}...`) await fs.move(path.join(this.config.path, asset.path), path.join(this.config.path, asset.destinationPath), { overwrite: true }) }, - + async getLocalLocation (asset) { + return path.join(this.config.path, asset.path) + }, /** * HANDLERS */ diff --git a/server/modules/storage/dropbox/storage.js b/server/modules/storage/dropbox/storage.js index ab25ce97..8ae6cc29 100644 --- a/server/modules/storage/dropbox/storage.js +++ b/server/modules/storage/dropbox/storage.js @@ -19,5 +19,8 @@ module.exports = { }, async renamed() { + }, + async getLocalLocation () { + } } diff --git a/server/modules/storage/gdrive/storage.js b/server/modules/storage/gdrive/storage.js index ab25ce97..8ae6cc29 100644 --- a/server/modules/storage/gdrive/storage.js +++ b/server/modules/storage/gdrive/storage.js @@ -19,5 +19,8 @@ module.exports = { }, async renamed() { + }, + async getLocalLocation () { + } } diff --git a/server/modules/storage/git/storage.js b/server/modules/storage/git/storage.js index 7f5c1e66..4da51a01 100644 --- a/server/modules/storage/git/storage.js +++ b/server/modules/storage/git/storage.js @@ -363,6 +363,9 @@ module.exports = { '--author': `"${asset.moveAuthorName} <${asset.moveAuthorEmail}>"` }) }, + async getLocalLocation (asset) { + return path.join(this.repoPath, asset.path) + }, /** * HANDLERS */ diff --git a/server/modules/storage/onedrive/storage.js b/server/modules/storage/onedrive/storage.js index ab25ce97..8ae6cc29 100644 --- a/server/modules/storage/onedrive/storage.js +++ b/server/modules/storage/onedrive/storage.js @@ -19,5 +19,8 @@ module.exports = { }, async renamed() { + }, + async getLocalLocation () { + } } diff --git a/server/modules/storage/s3/common.js b/server/modules/storage/s3/common.js index 66068c4b..b11bce00 100644 --- a/server/modules/storage/s3/common.js +++ b/server/modules/storage/s3/common.js @@ -119,6 +119,9 @@ module.exports = class S3CompatibleStorage { WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file from ${asset.path} to ${asset.destinationPath}...`) await this.s3.copyObject({ CopySource: asset.path, Key: asset.destinationPath }).promise() await this.s3.deleteObject({ Key: asset.path }).promise() + } + async getLocalLocation () { + } /** * HANDLERS diff --git a/server/modules/storage/sftp/storage.js b/server/modules/storage/sftp/storage.js index 5ded65c5..0684f4cb 100644 --- a/server/modules/storage/sftp/storage.js +++ b/server/modules/storage/sftp/storage.js @@ -103,6 +103,9 @@ module.exports = { WIKI.logger.info(`(STORAGE/SFTP) Renaming file from ${asset.path} to ${asset.destinationPath}...`) await this.ensureDirectory(asset.destinationPath) await this.sftp.rename(path.posix.join(this.config.basePath, asset.path), path.posix.join(this.config.basePath, asset.destinationPath)) + }, + async getLocalLocation () { + }, /** * HANDLERS