Added thumbnail generation + insert image files display + Create page fix

agent.js

@@ -39,6 +39,7 @@ global.WSInternalKey = process.argv[2];
 winston.info('[AGENT] Background Agent is initializing...');
 
 var appconfig = require('./models/config')('./config.yml');
+let lcdata = require('./models/localdata').init(appconfig, 'agent');
 
 global.git = require('./models/git').init(appconfig);
 global.entries = require('./models/entries').init(appconfig);
@@ -50,8 +51,12 @@ var Promise = require('bluebird');
 var fs = Promise.promisifyAll(require("fs-extra"));
 var path = require('path');
 var cron = require('cron').CronJob;
-var wsClient = require('socket.io-client');
+var readChunk = require('read-chunk');
-global.ws = wsClient('http://localhost:' + appconfig.wsPort, { reconnectionAttempts: 10 });
+var fileType = require('file-type');
+
+global.ws = require('socket.io-client')('http://localhost:' + appconfig.wsPort, { reconnectionAttempts: 10 });
+
+const mimeImgTypes = ['image/png', 'image/jpg']
 
 // ----------------------------------------
 // Start Cron
@@ -75,6 +80,7 @@ var job = new cron({
 
 		let jobs = [];
 		let repoPath = path.resolve(ROOTPATH, appconfig.datadir.repo);
+		let dataPath = path.resolve(ROOTPATH, appconfig.datadir.db);
 		let uploadsPath = path.join(repoPath, 'uploads');
 
 		// ----------------------------------------
@@ -151,11 +157,97 @@ var job = new cron({
 
 			return Promise.map(ls, (f) => {
 				return fs.statAsync(path.join(uploadsPath, f)).then((s) => { return { filename: f, stat: s }; });
-			}).filter((s) => { return s.stat.isDirectory(); }).then((arrStats) => {
+			}).filter((s) => { return s.stat.isDirectory(); }).then((arrDirs) => {
+
+				let folderNames = _.map(arrDirs, 'filename');
+				folderNames.unshift('');
+
 				ws.emit('uploadsSetFolders', {
 					auth: WSInternalKey,
-					content: _.map(arrStats, 'filename')
+					content: folderNames
+				});
+
+				let allFiles = [];
+
+				// Travel each directory
+
+				return Promise.map(folderNames, (fldName) => {
+					let fldPath = path.join(uploadsPath, fldName);
+					return fs.readdirAsync(fldPath).then((fList) => {
+						return Promise.map(fList, (f) => {
+							let fPath = path.join(fldPath, f);
+							let fPathObj = path.parse(fPath);
+
+							return fs.statAsync(fPath)
+								.then((s) => {
+
+									if(!s.isFile()) { return false; }
+
+									// Get MIME info
+
+									let mimeInfo = fileType(readChunk.sync(fPath, 0, 262));
+
+									// Images
+
+									if(s.size < 3145728) { // ignore files larger than 3MB
+										if(_.includes(['image/png', 'image/jpeg', 'image/gif', 'image/webp'], mimeInfo.mime)) {
+											return lcdata.getImageMetadata(fPath).then((mData) => {
+
+												let cacheThumbnailPath = path.parse(path.join(dataPath, 'thumbs', fldName, fPathObj.name + '.png'));
+												let cacheThumbnailPathStr = path.format(cacheThumbnailPath);
+
+												mData = _.pick(mData, ['format', 'width', 'height', 'density', 'hasAlpha', 'orientation']);
+												mData.category = 'image';
+												mData.mime = mimeInfo.mime;
+												mData.folder = fldName;
+												mData.filename = f;
+												mData.basename = fPathObj.name;
+												mData.filesize = s.size;
+												mData.uploadedOn = moment().utc();
+												allFiles.push(mData);
+
+												// Generate thumbnail
+
+												return fs.statAsync(cacheThumbnailPathStr).then((st) => {
+													return st.isFile();
+												}).catch((err) => {
+													return false;
+												}).then((thumbExists) => {
+
+													return (thumbExists) ? true : fs.ensureDirAsync(cacheThumbnailPath.dir).then(() => {
+														return lcdata.generateThumbnail(fPath, cacheThumbnailPathStr);
+													});
+
+												});
+
+											})
+										}
+									}
+
+									// Other Files
+									
+									allFiles.push({
+										category: 'file',
+										mime: mimeInfo.mime,
+										folder: fldName,
+										filename: f,
+										basename: fPathObj.name,
+										filesize: s.size,
+										uploadedOn: moment().utc()
+									});
+
+								});
+						}, {concurrency: 3});
+					});
+				}, {concurrency: 1}).finally(() => {
+
+					ws.emit('uploadsSetFiles', {
+						auth: WSInternalKey,
+						content: allFiles
+					});
+
 				});
+
 				return true;
 			});
 

client/js/components/editor-image.js

@@ -46,7 +46,10 @@ let vueImage = new Vue({
 			vueImage.isLoadingText = 'Fetching images...';
 			Vue.nextTick(() => {
 				socket.emit('uploadsGetImages', { folder: vueImage.currentFolder }, (data) => {
-					vueImage.images = data;
+					vueImage.images = _.map(data, (f) => {
+						f.thumbpath = (f.folder === '') ? f.basename + '.png' : _.join([ f.folder, f.basename + '.png' ], '/');
+						return f;
+					});
 					vueImage.isLoading = false;
 				});
 			});

client/scss/app.scss

@@ -7,6 +7,7 @@ $orange: #FB8C00;
 $blue: #039BE5;
 $turquoise: #00ACC1;
 $green: #7CB342;
+$purple: #673AB7;
 
 $warning: $orange;
 

client/scss/layout/_base.scss

@@ -9,7 +9,7 @@ html {
 	padding-top: 52px;
 }
 
-//$family-sans-serif: "Roboto", "Helvetica", "Arial", sans-serif;
+$family-monospace: monospace;
 $family-sans-serif: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
 
 [v-cloak] {

client/scss/layout/_content.scss

@@ -35,7 +35,7 @@
 		border-bottom: 1px dotted $grey-light;
 		padding-bottom: 4px;
 		font-weight: 400;
-		color: desaturate(darken($purple, 15%), 10%);
+		color: desaturate($purple, 20%);
 	}
 
 	a.toc-anchor {

controllers/uploads.js

@@ -5,11 +5,32 @@ var router = express.Router();
 var _ = require('lodash');
 
 var validPathRe = new RegExp("^([a-z0-9\\/-]+\\.[a-z0-9]+)$");
+var validPathThumbsRe = new RegExp("^([a-z0-9\\/-]+\\.png)$");
 
 // ==========================================
 // SERVE UPLOADS FILES
 // ==========================================
 
+router.get('/t/*', (req, res, next) => {
+
+	let fileName = req.params[0];
+	if(!validPathThumbsRe.test(fileName)) {
+		return res.sendStatus(404).end();
+	}
+
+	//todo: Authentication-based access
+
+	res.sendFile(fileName, {
+		root: lcdata.getThumbsPath(),
+		dotfiles: 'deny'
+	}, (err) => {
+		if (err) {
+			res.status(err.status).end();
+		}
+	});
+
+});
+
 router.get('/*', (req, res, next) => {
 
 	let fileName = req.params[0];

models/entries.js

@@ -5,8 +5,6 @@ var Promise = require('bluebird'),
 	fs = Promise.promisifyAll(require("fs-extra")),
 	_ = require('lodash'),
 	farmhash = require('farmhash'),
-	BSONModule = require('bson'),
-	BSON = new BSONModule.BSONPure.BSON(),
 	moment = require('moment');
 
 /**
@@ -82,7 +80,7 @@ module.exports = {
 				// Load from cache
 
 				return fs.readFileAsync(cpath).then((contents) => {
-					return BSON.deserialize(contents);
+					return JSON.parse(contents);
 				}).catch((err) => {
 					winston.error('Corrupted cache file. Deleting it...');
 					fs.unlinkSync(cpath);
@@ -156,7 +154,7 @@ module.exports = {
 						// Cache to disk
 
 						if(options.cache) {
-							let cacheData = BSON.serialize(_.pick(pageData, ['html', 'meta', 'tree', 'parent']), false, false, false);
+							let cacheData = JSON.stringify(_.pick(pageData, ['html', 'meta', 'tree', 'parent']), false, false, false);
 							return fs.writeFileAsync(cpath, cacheData).catch((err) => {
 								winston.error('Unable to write to cache! Performance may be affected.');
 								return true;
@@ -257,7 +255,7 @@ module.exports = {
 	 * @return     {String}  The full cache path.
 	 */
 	getCachePath(entryPath) {
-		return path.join(this._cachePath, farmhash.fingerprint32(entryPath) + '.bson');
+		return path.join(this._cachePath, farmhash.fingerprint32(entryPath) + '.json');
 	},
 
 	/**

models/localdata.js

@@ -2,6 +2,8 @@
 
 var fs = require('fs'),
 	path = require('path'),
+	loki = require('lokijs'),
+	Promise = require('bluebird'),
 	_ = require('lodash');
 
 /**
@@ -12,7 +14,9 @@ var fs = require('fs'),
 module.exports = {
 
 	_uploadsPath: './repo/uploads',
+	_uploadsThumbsPath: './data/thumbs',
 	_uploadsFolders: [],
+	_uploadsDb: null,
 
 	/**
 	 * Initialize Local Data Storage model
@@ -20,22 +24,88 @@ module.exports = {
 	 * @param      {Object}  appconfig  The application config
 	 * @return     {Object}  Local Data Storage model instance
 	 */
-	init(appconfig, skipFolderCreation = false) {
+	init(appconfig, mode = 'server') {
 
 		let self = this;
 
-		self._uploadsPath = path.join(ROOTPATH, appconfig.datadir.db, 'uploads');
+		self._uploadsPath = path.resolve(ROOTPATH, appconfig.datadir.repo, 'uploads');
+		self._uploadsThumbsPath = path.resolve(ROOTPATH, appconfig.datadir.db, 'thumbs');
 
-		// Create data directories
 
-		if(!skipFolderCreation) {
+		// Start in full or bare mode
-			self.createBaseDirectories(appconfig);
+
+		switch(mode) {
+			case 'agent':
+				//todo
+			break;
+			case 'server':
+				self.createBaseDirectories(appconfig);
+			break;
+			case 'ws':
+				self.initDb(appconfig);
+			break;
 		}
 
 		return self;
 
 	},
 
+	/**
+	 * Initialize Uploads DB
+	 *
+	 * @param      {Object}   appconfig  The application config
+	 * @return     {boolean}  Void
+	 */
+	initDb(appconfig) {
+
+		let self = this;
+
+		let dbReadyResolve;
+		let dbReady = new Promise((resolve, reject) => {
+			dbReadyResolve = resolve;
+		});
+
+		// Initialize Loki.js
+
+		let dbModel = {
+			Store: new loki(path.join(appconfig.datadir.db, 'uploads.db'), {
+				env: 'NODEJS',
+				autosave: true,
+				autosaveInterval: 15000
+			}),
+			onReady: dbReady
+		};
+
+		// Load Models
+
+		dbModel.Store.loadDatabase({}, () => {
+
+			dbModel.Files = dbModel.Store.getCollection('Files');
+			if(!dbModel.Files) {
+				dbModel.Files = dbModel.Store.addCollection('Files', {
+					indices: ['category', 'folder']
+				});
+			}
+
+			dbReadyResolve();
+
+		});
+
+		self._uploadsDb = dbModel;
+
+		return true;
+
+	},
+
+	/**
+	 * Gets the thumbnails folder path.
+	 *
+	 * @return     {String}  The thumbs path.
+	 */
+	getThumbsPath() {
+		return this._uploadsThumbsPath;
+	},
+
 	/**
 	 * Creates a base directories (Synchronous).
 	 *
@@ -99,6 +169,77 @@ module.exports = {
 	 */
 	getUploadsFolders() {
 		return this._uploadsFolders;
+	},
+
+	/**
+	 * Sets the uploads files.
+	 *
+	 * @param      {Array<Object>}  arrFiles  The uploads files
+	 * @return     {Void}  Void
+	 */
+	setUploadsFiles(arrFiles) {
+
+		let self = this;
+
+		if(_.isArray(arrFiles) && arrFiles.length > 0) {
+			self._uploadsDb.Files.clear();
+			self._uploadsDb.Files.insert(arrFiles);
+			self._uploadsDb.Files.ensureIndex('category', true);
+			self._uploadsDb.Files.ensureIndex('folder', true);
+		}
+
+		return;
+
+	},
+
+	/**
+	 * Gets the uploads files.
+	 *
+	 * @param      {String}  cat     Category type
+	 * @param      {String}  fld     Folder
+	 * @return     {Array<Object>}  The files matching the query
+	 */
+	getUploadsFiles(cat, fld) {
+
+		return this._uploadsDb.Files.find({
+			'$and': [{ 'category' : cat	},{ 'folder' : fld }]
+		});
+
+	},
+
+	/**
+	 * Generate thumbnail of image
+	 *
+	 * @param      {String}           sourcePath  The source path
+	 * @return     {Promise<Object>}  Promise returning the resized image info
+	 */
+	generateThumbnail(sourcePath, destPath) {
+
+		let sharp = require('sharp');
+
+		return sharp(sourcePath)
+						.withoutEnlargement()
+						.resize(150,150)
+						.background('white')
+						.embed()
+						.flatten()
+						.toFormat('png')
+						.toFile(destPath);
+
+	},
+
+	/**
+	 * Gets the image metadata.
+	 *
+	 * @param      {String}  sourcePath  The source path
+	 * @return     {Object}  The image metadata.
+	 */
+	getImageMetadata(sourcePath) {
+
+		let sharp = require('sharp');
+
+		return sharp(sourcePath).metadata();
+
 	}
 
 };

models/search.js

@@ -22,7 +22,7 @@ module.exports = {
 	init(appconfig) {
 
 		let self = this;
-		let dbPath = path.resolve(ROOTPATH, appconfig.datadir.db, 'search-index');
+		let dbPath = path.resolve(ROOTPATH, appconfig.datadir.db, 'search');
 
 		searchIndex({
 			deletable: true,

package.json

@@ -49,31 +49,33 @@
     "express-brute": "^1.0.0",
     "express-brute-loki": "^1.0.0",
     "express-session": "^1.14.1",
-    "express-validator": "^2.20.8",
+    "express-validator": "^2.20.10",
     "farmhash": "^1.2.1",
+    "file-type": "^3.8.0",
     "fs-extra": "^0.30.0",
     "git-wrapper2-promise": "^0.2.9",
-    "highlight.js": "^9.6.0",
+    "highlight.js": "^9.7.0",
-    "i18next": "^3.4.2",
+    "i18next": "^3.4.3",
     "i18next-express-middleware": "^1.0.2",
     "i18next-node-fs-backend": "^0.1.2",
     "js-yaml": "^3.6.1",
-    "lodash": "^4.15.0",
+    "lodash": "^4.16.1",
     "lokijs": "^1.4.1",
     "markdown-it": "^8.0.0",
     "markdown-it-abbr": "^1.0.4",
     "markdown-it-anchor": "^2.5.0",
-    "markdown-it-attrs": "^0.7.0",
+    "markdown-it-attrs": "^0.7.1",
     "markdown-it-emoji": "^1.2.0",
     "markdown-it-expand-tabs": "^1.0.11",
     "markdown-it-external-links": "0.0.5",
     "markdown-it-footnote": "^3.0.1",
     "markdown-it-task-lists": "^1.4.1",
-    "moment": "^2.15.0",
+    "moment": "^2.15.1",
     "moment-timezone": "^0.5.5",
     "passport": "^0.3.2",
     "passport-local": "^1.0.0",
     "pug": "^2.0.0-beta6",
+    "read-chunk": "^2.0.0",
     "remove-markdown": "^0.1.0",
     "search-index": "^0.8.15",
     "serve-favicon": "^2.3.0",
@@ -89,7 +91,7 @@
   "devDependencies": {
     "ace-builds": "^1.2.5",
     "babel-preset-es2015": "^6.14.0",
-    "bulma": "^0.1.2",
+    "bulma": "^0.2.0",
     "chai": "^3.5.0",
     "chai-as-promised": "^5.3.0",
     "codacy-coverage": "^2.0.0",
@@ -107,7 +109,7 @@
     "gulp-uglify": "^2.0.0",
     "gulp-zip": "^3.2.0",
     "istanbul": "^0.4.5",
-    "jquery": "^3.1.0",
+    "jquery": "^3.1.1",
     "jquery-smooth-scroll": "^2.0.0",
     "merge-stream": "^1.0.0",
     "mocha": "^3.0.2",
@@ -115,7 +117,7 @@
     "nodemon": "^1.10.2",
     "sticky-js": "^1.0.5",
     "twemoji-awesome": "^1.0.4",
-    "vue": "^1.0.26"
+    "vue": "^1.0.27"
   },
   "snyk": true
 }

server.js

@@ -7,7 +7,7 @@
 global.ROOTPATH = __dirname;
 
 // ----------------------------------------
-// Load global modules
+// Load Winston
 // ----------------------------------------
 
 var _isDebug = process.env.NODE_ENV === 'development';
@@ -24,9 +24,12 @@ winston.add(winston.transports.Console, {
 
 winston.info('[SERVER] Requarks Wiki is initializing...');
 
-var appconfig = require('./models/config')('./config.yml');
+// ----------------------------------------
-let lcdata = require('./models/localdata').init(appconfig, false);
+// Load global modules
+// ----------------------------------------
 
+var appconfig = require('./models/config')('./config.yml');
+global.lcdata = require('./models/localdata').init(appconfig, 'server');
 global.db = require('./models/db')(appconfig);
 global.git = require('./models/git').init(appconfig, false);
 global.entries = require('./models/entries').init(appconfig);

views/modals/editor-image.pug

@@ -37,17 +37,13 @@
 						p.menu-label
 							| Folders
 						ul.menu-list
-							li
-								a(v-on:click="selectFolder('')", v-bind:class="{ 'is-active': currentFolder === '' }")
-									span.icon.is-small: i.fa.fa-folder-o
-									span /
 							li(v-for="fld in folders")
 								a(v-on:click="selectFolder(fld)", v-bind:class="{ 'is-active': currentFolder === fld }")
 									span.icon.is-small: i.fa.fa-folder
-									span / {{ fld }}
+									span /{{ fld }}
 			.column
-				figure.image.is-128x128
+				figure.image.is-128x128(v-for="img in images")
-					img(src='http://placehold.it/128x128')
+					img(v-bind:src="'/uploads/t/' + img.thumbpath")
 
 	.modal(v-bind:class="{ 'is-active': newFolderShow }")
 		.modal-background

views/pages/create.pug

@@ -21,4 +21,7 @@ block content
 		.editor-area
 			textarea#mk-editor= pageData.markdown
 
-	include ../modals/create-discard.pug
+	include ../modals/create-discard.pug
+	include ../modals/editor-link.pug
+	include ../modals/editor-image.pug
+	include ../modals/editor-codeblock.pug

ws-server.js

@@ -33,20 +33,20 @@ if(!process.argv[2] || process.argv[2].length !== 40) {
 global.internalAuth = require('./lib/internalAuth').init(process.argv[2]);;
 
 // ----------------------------------------
-// Load modules
+// Load global modules
 // ----------------------------------------
 
 winston.info('[WS] WS Server is initializing...');
 
 var appconfig = require('./models/config')('./config.yml');
-let lcdata = require('./models/localdata').init(appconfig, true);
+let lcdata = require('./models/localdata').init(appconfig, 'ws');
 
 global.entries = require('./models/entries').init(appconfig);
 global.mark = require('./models/markdown');
 global.search = require('./models/search').init(appconfig);
 
 // ----------------------------------------
-// Load modules
+// Load local modules
 // ----------------------------------------
 
 var _ = require('lodash');
@@ -141,8 +141,14 @@ io.on('connection', (socket) => {
 		cb(lcdata.getUploadsFolders());
 	});
 
+	socket.on('uploadsSetFiles', (data, cb) => {
+		if(internalAuth.validateKey(data.auth)) {
+			lcdata.setUploadsFiles(data.content);
+		}
+	});
+
 	socket.on('uploadsGetImages', (data, cb) => {
-		cb([]);
+		cb(lcdata.getUploadsFiles('image', data.folder));
 	});
 
 });