msb-public
/
wiki
mirror of https://github.com/requarks/wiki
1
0
Code Issues Projects Releases Wiki Activity

Search results picker + create/update index update

Browse Source
pull/1/head
NGPixel 8 years ago
parent dca6b71610
commit 48e2afa5c0
16 changed files with 169 additions and 102 deletions
Show Stats Download Patch File Download Diff File

22
agent.js
Unescape View File

@ -68,6 +68,7 @@ var job = new cron({
winston.warn('[AGENT] Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)'); winston.warn('[AGENT] Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)');
return; return;
} }
winston.info('[AGENT] Running all jobs...');
jobIsBusy = true; jobIsBusy = true;
// Prepare async job collector // Prepare async job collector
@ -87,6 +88,10 @@ var job = new cron({
//-> Stream all documents //-> Stream all documents
let cacheJobs = []; let cacheJobs = [];
let jobCbStreamDocs_resolve = null,
jobCbStreamDocs = new Promise((resolve, reject) => {
jobCbStreamDocs_resolve = resolve;
});
fs.walk(repoPath).on('data', function (item) { fs.walk(repoPath).on('data', function (item) {
if(path.extname(item.path) === '.md') { if(path.extname(item.path) === '.md') {
@ -113,15 +118,10 @@ var job = new cron({
}).then((fileStatus) => { }).then((fileStatus) => {
//-> Update search index //-> Update cache and search index
if(fileStatus !== 'active') { if(fileStatus !== 'active') {
return entries.fetchIndexableVersion(entryPath).then((content) => { return entries.updateCache(entryPath);
ws.emit('searchAdd', {
auth: WSInternalKey,
content
});
});
} }
return true; return true;
@ -131,9 +131,11 @@ var job = new cron({
); );
} }
}).on('end', () => {
jobCbStreamDocs_resolve(Promise.all(cacheJobs));
}); });
return Promise.all(cacheJobs); return jobCbStreamDocs;
}); });
})); }));
@ -143,7 +145,7 @@ var job = new cron({
// ---------------------------------------- // ----------------------------------------
Promise.all(jobs).then(() => { Promise.all(jobs).then(() => {
winston.info('[AGENT] All jobs completed successfully! Going to sleep for now...'); winston.info('[AGENT] All jobs completed successfully! Going to sleep for now.');
}).catch((err) => { }).catch((err) => {
winston.error('[AGENT] One or more jobs have failed: ', err); winston.error('[AGENT] One or more jobs have failed: ', err);
}).finally(() => { }).finally(() => {
@ -161,8 +163,8 @@ var job = new cron({
// ---------------------------------------- // ----------------------------------------
ws.on('connect', function () { ws.on('connect', function () {
job.start();
winston.info('[AGENT] Background Agent started successfully! [RUNNING]'); winston.info('[AGENT] Background Agent started successfully! [RUNNING]');
job.start();
}); });
ws.on('connect_error', function () { ws.on('connect_error', function () {

2
assets/css/app.css
View File

File diff suppressed because one or more lines are too long

2
assets/js/app.js
View File

File diff suppressed because one or more lines are too long

23
client/js/components/search.js
Unescape View File

@ -27,24 +27,32 @@ jQuery( document ).ready(function( $ ) {
}, },
watch: { watch: {
searchq: (val, oldVal) => { searchq: (val, oldVal) => {
searchmoveidx: 0; vueHeader.searchmoveidx = 0;
if(val.length >= 3) { if(val.length >= 3) {
vueHeader.searchactive = true; vueHeader.searchactive = true;
vueHeader.searchload++; vueHeader.searchload++;
socket.emit('search', { terms: val }, (data) => { socket.emit('search', { terms: val }, (data) => {
vueHeader.searchres = data.match; vueHeader.searchres = data.match;
vueHeader.searchsuggest = data.suggest; vueHeader.searchsuggest = data.suggest;
vueHeader.searchmovearr = _.concat([], vueHeader.searchres, vueHeader.searchsuggest);
if(vueHeader.searchload > 0) { vueHeader.searchload--; } if(vueHeader.searchload > 0) { vueHeader.searchload--; }
}); });
} else { } else {
vueHeader.searchactive = false; vueHeader.searchactive = false;
vueHeader.searchres = []; vueHeader.searchres = [];
vueHeader.searchsuggest = []; vueHeader.searchsuggest = [];
vueHeader.searchmovearr = [];
vueHeader.searchload = 0; vueHeader.searchload = 0;
} }
}, },
searchmoveidx: (val, oldVal) => { searchmoveidx: (val, oldVal) => {
if(val > 0) {
vueHeader.searchmovekey = (vueHeader.searchmovearr[val - 1].document) ?
'res.' + vueHeader.searchmovearr[val - 1].document.entryPath :
'sug.' + vueHeader.searchmovearr[val - 1];
} else {
vueHeader.searchmovekey = '';
}
} }
}, },
methods: { methods: {
@ -53,13 +61,20 @@ jQuery( document ).ready(function( $ ) {
}, },
closeSearch: () => { closeSearch: () => {
vueHeader.searchq = ''; vueHeader.searchq = '';
vueHeader.searchactive = false;
}, },
moveSelectSearch: () => { moveSelectSearch: () => {
if(vueHeader.searchmoveidx < 1) { return; }
let i = vueHeader.searchmoveidx - 1;
if(vueHeader.searchmovearr[i].document) {
window.location.assign('/' + vueHeader.searchmovearr[i].document.entryPath);
} else {
vueHeader.searchq = vueHeader.searchmovearr[i];
}
}, },
moveDownSearch: () => { moveDownSearch: () => {
if(vueHeader.searchmoveidx < vueHeader.searchmovearr) { if(vueHeader.searchmoveidx < vueHeader.searchmovearr.length) {
vueHeader.searchmoveidx++; vueHeader.searchmoveidx++;
} }
}, },

2
client/scss/components/_alerts.scss
Unescape View File

@ -3,7 +3,7 @@
top: 60px; top: 60px;
right: 10px; right: 10px;
width: 350px; width: 350px;
z-index: 2; z-index: 10;
text-shadow: 1px 1px 0 rgba(0,0,0,0.1); text-shadow: 1px 1px 0 rgba(0,0,0,0.1);
.notification { .notification {

6
controllers/pages.js
Unescape View File

@ -71,7 +71,7 @@ router.get('/create/*', (req, res, next) => {
entries.exists(safePath).then((docExists) => { entries.exists(safePath).then((docExists) => {
if(!docExists) { if(!docExists) {
entries.getStarter(safePath).then((contents) => { return entries.getStarter(safePath).then((contents) => {
let pageData = { let pageData = {
markdown: contents, markdown: contents,
@ -80,7 +80,9 @@ router.get('/create/*', (req, res, next) => {
path: safePath path: safePath
} }
}; };
return res.render('pages/create', { pageData }); res.render('pages/create', { pageData });
return true;
}).catch((err) => { }).catch((err) => {
throw new Error('Could not load starter content!'); throw new Error('Could not load starter content!');

71
models/entries.js
Unescape View File

@ -156,7 +156,7 @@ module.exports = {
// Cache to disk // Cache to disk
if(options.cache) { if(options.cache) {
let cacheData = BSON.serialize(pageData, false, false, false); let cacheData = BSON.serialize(_.pick(pageData, ['html', 'meta', 'tree', 'parent']), false, false, false);
return fs.writeFileAsync(cpath, cacheData).catch((err) => { return fs.writeFileAsync(cpath, cacheData).catch((err) => {
winston.error('Unable to write to cache! Performance may be affected.'); winston.error('Unable to write to cache! Performance may be affected.');
return true; return true;
@ -177,34 +177,6 @@ module.exports = {
}, },
/**
* Fetches a text version of a Markdown-formatted document
*
* @param {String} entryPath The entry path
* @return {String} Text-only version
*/
fetchIndexableVersion(entryPath) {
let self = this;
return self.fetchOriginal(entryPath, {
parseMarkdown: false,
parseMeta: true,
parseTree: false,
includeMarkdown: true,
includeParentInfo: true,
cache: false
}).then((pageData) => {
return {
entryPath,
meta: pageData.meta,
parent: pageData.parent || {},
text: mark.removeMarkdown(pageData.markdown)
};
});
},
/** /**
* Parse raw url path and make it safe * Parse raw url path and make it safe
* *
@ -314,13 +286,48 @@ module.exports = {
return fs.statAsync(fpath).then((st) => { return fs.statAsync(fpath).then((st) => {
if(st.isFile()) { if(st.isFile()) {
return self.makePersistent(entryPath, contents).then(() => { return self.makePersistent(entryPath, contents).then(() => {
return self.fetchOriginal(entryPath, {}); return self.updateCache(entryPath);
}); });
} else { } else {
return Promise.reject(new Error('Entry does not exist!')); return Promise.reject(new Error('Entry does not exist!'));
} }
}).catch((err) => { }).catch((err) => {
return Promise.reject(new Error('Entry does not exist!')); winston.error(err);
return Promise.reject(new Error('Failed to save document.'));
});
},
/**
* Update local cache and search index
*
* @param {String} entryPath The entry path
* @return {Promise} Promise of the operation
*/
updateCache(entryPath) {
let self = this;
return self.fetchOriginal(entryPath, {
parseMarkdown: true,
parseMeta: true,
parseTree: true,
includeMarkdown: true,
includeParentInfo: true,
cache: true
}).then((pageData) => {
return {
entryPath,
meta: pageData.meta,
parent: pageData.parent || {},
text: mark.removeMarkdown(pageData.markdown)
};
}).then((content) => {
ws.emit('searchAdd', {
auth: WSInternalKey,
content
});
return true;
}); });
}, },
@ -339,7 +346,7 @@ module.exports = {
return self.exists(entryPath).then((docExists) => { return self.exists(entryPath).then((docExists) => {
if(!docExists) { if(!docExists) {
return self.makePersistent(entryPath, contents).then(() => { return self.makePersistent(entryPath, contents).then(() => {
return self.fetchOriginal(entryPath, {}); return self.updateCache(entryPath);
}); });
} else { } else {
return Promise.reject(new Error('Entry already exists!')); return Promise.reject(new Error('Entry already exists!'));

4
models/git.js
Unescape View File

@ -195,7 +195,9 @@ module.exports = {
commitMsg = (isTracked) ? 'Updated ' + gitFilePath : 'Added ' + gitFilePath; commitMsg = (isTracked) ? 'Updated ' + gitFilePath : 'Added ' + gitFilePath;
return self._git.add(gitFilePath); return self._git.add(gitFilePath);
}).then(() => { }).then(() => {
return self._git.commit(commitMsg); return self._git.commit(commitMsg).catch((err) => {
if(_.includes(err.stdout, 'nothing to commit')) { return true; }
});
}); });
} }

90
models/search.js
Unescape View File

@ -100,42 +100,62 @@ module.exports = {
let self = this; let self = this;
return self._si.addAsync({ return self._si.searchAsync({
entryPath: content.entryPath, query: {
title: content.meta.title, AND: [{ 'entryPath': [content.entryPath] }]
subtitle: content.meta.subtitle || '', }
parent: content.parent.title || '', }).then((results) => {
content: content.text || ''
}, { if(results.totalHits > 0) {
fieldOptions: [{ let delIds = _.map(results.hits, 'id');
fieldName: 'entryPath', return self._si.delAsync(delIds);
searchable: true, } else {
weight: 2 return true;
}, }
{
fieldName: 'title',
nGramLength: [1, 2],
searchable: true,
weight: 3
},
{
fieldName: 'subtitle',
searchable: true,
weight: 1,
store: false
},
{
fieldName: 'subtitle',
searchable: false,
},
{
fieldName: 'content',
searchable: true,
weight: 0,
store: false
}]
}).then(() => { }).then(() => {
winston.info('Entry ' + content.entryPath + ' added to index.');
return self._si.addAsync({
entryPath: content.entryPath,
title: content.meta.title,
subtitle: content.meta.subtitle || '',
parent: content.parent.title || '',
content: content.text || ''
}, {
fieldOptions: [{
fieldName: 'entryPath',
searchable: true,
weight: 2
},
{
fieldName: 'title',
nGramLength: [1, 2],
searchable: true,
weight: 3
},
{
fieldName: 'subtitle',
searchable: true,
weight: 1,
store: false
},
{
fieldName: 'parent',
searchable: false,
},
{
fieldName: 'content',
searchable: true,
weight: 0,
store: false
}]
}).then(() => {
winston.info('Entry ' + content.entryPath + ' added/updated to index.');
return true;
}).catch((err) => {
winston.error(err);
});
}).catch((err) => { }).catch((err) => {
winston.error(err); winston.error(err);
}); });

27
server.js
Unescape View File

@ -206,13 +206,32 @@ server.on('listening', () => {
// ---------------------------------------- // ----------------------------------------
var fork = require('child_process').fork, var fork = require('child_process').fork,
libInternalAuth = require('./lib/internalAuth'), libInternalAuth = require('./lib/internalAuth');
internalAuthKey = libInternalAuth.generateKey();
var wsSrv = fork('ws-server.js', [internalAuthKey]), global.WSInternalKey = libInternalAuth.generateKey();
bgAgent = fork('agent.js', [internalAuthKey]);
var wsSrv = fork('ws-server.js', [WSInternalKey]),
bgAgent = fork('agent.js', [WSInternalKey]);
process.on('exit', (code) => { process.on('exit', (code) => {
wsSrv.disconnect(); wsSrv.disconnect();
bgAgent.disconnect(); bgAgent.disconnect();
});
// ----------------------------------------
// Connect to local WebSocket server
// ----------------------------------------
var wsClient = require('socket.io-client');
global.ws = wsClient('http://localhost:' + appconfig.wsPort, { reconnectionAttempts: 10 });
ws.on('connect', function () {
winston.info('[SERVER] Connected to WebSocket server successfully!');
});
ws.on('connect_error', function () {
winston.warn('[SERVER] Unable to connect to WebSocket server! Retrying...');
});
ws.on('reconnect_failed', function () {
winston.error('[SERVER] Failed to reconnect to WebSocket server too many times! Stopping...');
process.exit(1);
}); });

4
views/common/header.pug
Unescape View File

@ -11,7 +11,7 @@
block rootNavCenter block rootNavCenter
.nav-item .nav-item
p.control(v-bind:class="{ 'is-loading': searchload > 0 }") p.control(v-bind:class="{ 'is-loading': searchload > 0 }")
input.input#search-input(type='text', v-model='searchq', @keyup.esc='closeSearch', @keyup.down='moveDownSearch', @keyup.up='moveUpSearch', debounce='400', placeholder='Search...') input.input#search-input(type='text', v-model='searchq', @keyup.esc='closeSearch', @keyup.down='moveDownSearch', @keyup.up='moveUpSearch', @keyup.enter='moveSelectSearch', debounce='400', placeholder='Search...')
span.nav-toggle span.nav-toggle
span span
span span
@ -46,6 +46,6 @@
| Did you mean...? | Did you mean...?
ul.menu-list(v-if='searchsuggest.length > 0') ul.menu-list(v-if='searchsuggest.length > 0')
li(v-for='sug in searchsuggest') li(v-for='sug in searchsuggest')
a(v-on:click="useSuggestion(sug)") {{ sug }} a(v-on:click="useSuggestion(sug)", v-bind:class="{ 'is-active': searchmovekey === 'sug.' + sug }") {{ sug }}

6
views/layout.pug
Unescape View File

@ -31,10 +31,10 @@ html
body body
#root #root
include ./common/header include ./common/header.pug
include ./common/alerts include ./common/alerts.pug
main main
block content block content
include ./common/footer include ./common/footer.pug
block outside block outside

2
views/pages/create.pug
Unescape View File

@ -1,4 +1,4 @@
extends ../layout extends ../layout.pug
block rootNavCenter block rootNavCenter
h2.nav-item Create New Document h2.nav-item Create New Document

4
views/pages/edit.pug
Unescape View File

@ -1,4 +1,4 @@
extends ../layout extends ../layout.pug
block rootNavCenter block rootNavCenter
h2.nav-item= pageData.meta.title h2.nav-item= pageData.meta.title
@ -24,4 +24,4 @@ block content
#page-type-edit(data-entrypath=pageData.meta.path) #page-type-edit(data-entrypath=pageData.meta.path)
textarea#mk-editor= pageData.markdown textarea#mk-editor= pageData.markdown
include ../modals/edit include ../modals/edit.pug

2
views/pages/source.pug
Unescape View File

@ -1,4 +1,4 @@
extends ../layout extends ../layout.pug
block content block content

4
views/pages/view.pug
Unescape View File

@ -1,4 +1,4 @@
extends ../layout extends ../layout.pug
mixin tocMenu(ti) mixin tocMenu(ti)
each node in ti each node in ti
@ -61,4 +61,4 @@ block content
.content.mkcontent .content.mkcontent
!= pageData.html != pageData.html
include ../modals/create include ../modals/create.pug

Write Preview
Loading…
Cancel
Save