From a323608faf46c60dd20c354081ed38cfb5e82c78 Mon Sep 17 00:00:00 2001 From: Lawrence Ching Date: Fri, 13 Oct 2023 16:10:41 +0000 Subject: [PATCH] feat: add function of generating sitemap under a toggle --- config.sample.yml | 6 +++ package.json | 1 + server/controllers/common.js | 86 ++++++++++++++++++++++++++++++++++++ server/setup.js | 6 ++- yarn.lock | 27 +++++++++++ 5 files changed, 125 insertions(+), 1 deletion(-) diff --git a/config.sample.yml b/config.sample.yml index 47edd8d2..92544575 100644 --- a/config.sample.yml +++ b/config.sample.yml @@ -145,3 +145,9 @@ dataPath: ./data # file uploads. bodyParserLimit: 5mb + + +seo: + sitemap: + enabled: false # sitemap is experimental feature, do not enable in production + cacheExpireTime: 3600 # in seconds \ No newline at end of file diff --git a/package.json b/package.json index e78c91b1..08fd8b35 100644 --- a/package.json +++ b/package.json @@ -176,6 +176,7 @@ "semver": "7.3.8", "serve-favicon": "2.5.0", "simple-git": "3.16.0", + "sitemap": "7.1.1", "solr-node": "1.2.1", "sqlite3": "5.1.4", "ssh2": "1.11.0", diff --git a/server/controllers/common.js b/server/controllers/common.js index 55cc4d33..1e658e87 100644 --- a/server/controllers/common.js +++ b/server/controllers/common.js @@ -5,6 +5,7 @@ const _ = require('lodash') const CleanCSS = require('clean-css') const moment = require('moment') const qs = require('querystring') +const { SitemapStream, streamToPromise } = require('sitemap') /* global WIKI */ @@ -22,6 +23,91 @@ router.get('/robots.txt', (req, res, next) => { } }) +/** + * sitemap.xml + */ +// TODO: toggle off before the feature is ready +if(WIKI.config.seo.sitemap.enabled) { + WIKI.logger.info(`Experimental feature sitemap is enabled`) + router.get('/sitemap.xml', async (req, res + + ) => { + const host = WIKI.config.host + const {enabled, cacheExpireTime} = WIKI.config.seo.sitemap + !enabled && res.status(404).end() + + try { + + res.header('Content-Type', 'application/xml'); + + // TODO: if we have a cached entry send it + // if (sitemap) { + // res.send(sitemap) + // return + // } + + const smStream = new SitemapStream({ hostname: host}) + + /** + * [ + Page { + id: 1, + path: 'home', + hash: 'b29b5d2ce62e55412776ab98f05631e0aa96597b', + title: 'Untitled Page', + description: '', + isPrivate: 0, + isPublished: 1, + privateNS: null, + publishStartDate: '', + publishEndDate: '', + content: '# Header\nYour content here', + render: '

ΒΆ Header

\n' + + '

Your content here

\n', + toc: '[{"title":"Header","anchor":"#header","children":[]}]', + contentType: 'markdown', + createdAt: '2023-10-12T16:07:42.594Z', + updatedAt: '2023-10-12T16:07:44.791Z', + editorKey: 'markdown', + localeCode: 'en', + authorId: 1, + creatorId: 1, + extra: { js: '', css: '' } + } + ] + */ + const pages = await WIKI.models.pages.query() + .where('isPublished', 1) + .where('isPrivate', 0) + .orderBy('updatedAt', 'desc') + + for (const page of pages) { + // TODO: + /** + * TypeError: user.getGlobalPermissions is not a function + at Object.checkAccess (/home/xxx/workspace/wiki/server/core/auth.js:218:72) + at /home/xxx/workspace/wiki/server/controllers/common.js:92:22 + */ + // if (!WIKI.auth.checkAccess(WIKI.auth.guest, ['read:pages'], page)) continue + + smStream.write({ + url: `/${page.localeCode}/${page.path}`, + lastmod: page.updatedAt + }) + } + + smStream.end(); + smStream.pipe(res).on('error', (e) => { throw e }) + } catch (e) { + console.error(e) + res.status(500).end() + } + + }) +} + + + /** * Health Endpoint */ diff --git a/server/setup.js b/server/setup.js index 44da308b..52ed8a08 100644 --- a/server/setup.js +++ b/server/setup.js @@ -117,7 +117,11 @@ module.exports = () => { description: '', robots: ['index', 'follow'], analyticsService: '', - analyticsId: '' + analyticsId: '', + sitemap: { + enabled: false, + cacheExpireTime: 3600 // in seconds + } }) _.set(WIKI.config, 'sessionSecret', (await crypto.randomBytesAsync(32)).toString('hex')) _.set(WIKI.config, 'telemetry', { diff --git a/yarn.lock b/yarn.lock index d7003475..1cda95fd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3839,6 +3839,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.14.tgz#1c1d6e3c75dba466e0326948d56e8bd72a1903d2" integrity sha512-u/SJDyXwuihpwjXy7hOOghagLEV1KdAST6syfnOk6QZAMzZuWZqXy5aYYZbh8Jdpd4escVFP0MvftHNDb9pruA== +"@types/node@^17.0.5": + version "17.0.45" + resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.45.tgz#2c0fafd78705e7a18b7906b5201a522719dc5190" + integrity sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw== + "@types/node@^8.0.47": version "8.10.53" resolved "https://registry.yarnpkg.com/@types/node/-/node-8.10.53.tgz#5fa08eef810b08b2c03073e360b54f7bad899db1" @@ -3890,6 +3895,13 @@ "@types/node" "*" safe-buffer "*" +"@types/sax@^1.2.1": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@types/sax/-/sax-1.2.5.tgz#4392799e1770d24b6dc8d0c66c8882f8e1c38b3d" + integrity sha512-9jWta97bBVC027/MShr3gLab8gPhKy4l6qpb+UJLF5pDm3501NvA7uvqVCW+REFtx00oTi6Cq9JzLwgq6evVgw== + dependencies: + "@types/node" "*" + "@types/serve-static@*": version "1.13.3" resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.13.3.tgz#eb7e1c41c4468272557e897e9171ded5e2ded9d1" @@ -5107,6 +5119,11 @@ are-we-there-yet@~1.1.2: delegates "^1.0.0" readable-stream "^2.0.6" +arg@^5.0.0: + version "5.0.2" + resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c" + integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg== + argparse@^1.0.7: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" @@ -18189,6 +18206,16 @@ sisteransi@^1.0.3: resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.3.tgz#98168d62b79e3a5e758e27ae63c4a053d748f4eb" integrity sha512-SbEG75TzH8G7eVXFSN5f9EExILKfly7SUvVY5DhhYLvfhKqhDFY0OzevWa/zwak0RLRfWS5AvfMWpd9gJvr5Yg== +sitemap@7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-7.1.1.tgz#eeed9ad6d95499161a3eadc60f8c6dce4bea2bef" + integrity sha512-mK3aFtjz4VdJN0igpIJrinf3EO8U8mxOPsTBzSsy06UtjZQJ3YY3o3Xa7zSc5nMqcMrRwlChHZ18Kxg0caiPBg== + dependencies: + "@types/node" "^17.0.5" + "@types/sax" "^1.2.1" + arg "^5.0.0" + sax "^1.2.4" + slash@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55"