mirror of https://github.com/requarks/wiki
NGPixel
3 weeks ago
parent
91eede058e
commit
49555480a7
@ -0,0 +1,510 @@
|
||||
import fs from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { dump as dumpYaml, load as loadYaml } from 'js-yaml'
|
||||
import { pageEditorForExtension, pageFileExtension } from '../models/pages.ts'
|
||||
import type { StoragePageContent, StoragePageRef, StorageTarget } from '../models/storage.ts'
|
||||
|
||||
/**
|
||||
* What a storage module needs in order to keep the wiki's content as a tree of ordinary files.
|
||||
*
|
||||
* Shared by every target that addresses content by path — the local disk and git today — because the
|
||||
* two have to agree about it exactly. A page written by one and read back by the other has to come
|
||||
* back as the same page, so the front matter, the file name a page is filed under, the rule for what
|
||||
* makes a file a page rather than an attachment and the walk that reads a folder back all live here
|
||||
* rather than in either module.
|
||||
*
|
||||
* What does *not* live here is anything about where the root is or what happens after a file is
|
||||
* written: the disk target is finished at that point, and git has a commit to make.
|
||||
*
|
||||
* Not under `modules/storage/` deliberately. `refreshFromDisk` reads every directory there and
|
||||
* expects a `definition.yml` in it, and a directory without one takes every storage module down with
|
||||
* it.
|
||||
*/
|
||||
|
||||
/** Leading YAML front matter, as `serializePage` writes it. */
|
||||
const FRONT_MATTER = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?/
|
||||
|
||||
/** The extension whose pages are written as one JSON document rather than front matter and a body. */
|
||||
export const JSON_EXTENSION = 'json'
|
||||
|
||||
/**
|
||||
* The editor an imported page falls back to.
|
||||
*
|
||||
* Only reached for a file the site reserves as a page extension but which no editor writes — `txt` on
|
||||
* a default site. Markdown renders plain prose as prose, so it is the least surprising answer.
|
||||
*/
|
||||
const DEFAULT_PAGE_EDITOR = 'markdown'
|
||||
|
||||
/**
|
||||
* What the site's root page is filed as.
|
||||
*
|
||||
* A page path may be empty, which is the page a site serves at `/`. It still needs a name of its own
|
||||
* on disk, and `index` is the one every other tool that writes a tree of documents picks.
|
||||
*/
|
||||
export const ROOT_PAGE_NAME = 'index'
|
||||
|
||||
/**
|
||||
* Names never walked by an import.
|
||||
*
|
||||
* Anything starting with a dot, which covers a half-written `.tmp`, the `.DS_Store` a Mac leaves in
|
||||
* every folder it has looked at, and — the reason this is tested against every segment of the path
|
||||
* rather than only the file name — the whole of a `.git` directory. None of that is content somebody
|
||||
* meant to put in their wiki, and a repository's own internals least of all.
|
||||
*/
|
||||
const IGNORED_SEGMENT = /^\.|\.tmp$/
|
||||
|
||||
/**
|
||||
* A configured root as an absolute path.
|
||||
*
|
||||
* A relative setting is resolved from the install directory rather than from the working directory,
|
||||
* so that `./data/content` means the same folder whichever way the server was started.
|
||||
*/
|
||||
export function resolveRoot(configured: string | undefined, fallback: string): string {
|
||||
return path.resolve(WIKI.ROOTPATH, configured || fallback)
|
||||
}
|
||||
|
||||
/**
|
||||
* Where an asset belongs under the root, as a slash-separated relative path.
|
||||
*
|
||||
* What brackets the tree — the site, the locale, both or neither — is the site's own answer and is
|
||||
* `pathPrefixFor`'s to give; everything below it is the tree as the file manager shows it.
|
||||
*
|
||||
* @returns Null for content this site's layout has no place for, which a caller reads as "this target
|
||||
* does not hold that": a secondary locale on a site storing only its primary one
|
||||
*/
|
||||
export function assetRelPath(
|
||||
target: StorageTarget,
|
||||
ref: { locale: string; folderPath: string; fileName: string }
|
||||
): string | null {
|
||||
const prefix = WIKI.models.storage.pathPrefixFor(target.siteId, ref.locale)
|
||||
if (!prefix) {
|
||||
return null
|
||||
}
|
||||
return [...prefix, ...ref.folderPath.split('/').filter(Boolean), ref.fileName].join('/')
|
||||
}
|
||||
|
||||
/**
|
||||
* Where a page's copy belongs, alongside the assets of the same folder.
|
||||
*
|
||||
* The extension is the one its editor writes, and is exactly what the wiki reserves against uploads —
|
||||
* `models/assets.ts` refuses an attachment that would take this name, so the two never meet here.
|
||||
*
|
||||
* @returns Null under the same circumstances as `assetRelPath`
|
||||
*/
|
||||
export function pageRelPath(target: StorageTarget, ref: StoragePageRef): string | null {
|
||||
const prefix = WIKI.models.storage.pathPrefixFor(target.siteId, ref.locale)
|
||||
if (!prefix) {
|
||||
return null
|
||||
}
|
||||
const segments = ref.path.split('/').filter(Boolean)
|
||||
const fileName = segments.pop() ?? ROOT_PAGE_NAME
|
||||
return [...prefix, ...segments, `${fileName}.${pageFileExtension(ref.contentType)}`].join('/')
|
||||
}
|
||||
|
||||
/**
|
||||
* The absolute path of a stored file, refusing anything that would land outside the root.
|
||||
*
|
||||
* Every segment reaching this is either a UUID or a name the tree has already normalized, so this
|
||||
* catches a stored path that has been tampered with rather than an ordinary mistake — but it is the
|
||||
* only thing between a `..` in the database and the rest of the file system.
|
||||
*/
|
||||
export function absPathIn(root: string, relPath: string): string {
|
||||
const resolved = path.resolve(root, relPath)
|
||||
if (resolved !== root && !resolved.startsWith(root + path.sep)) {
|
||||
throw new Error(`The stored path "${relPath}" resolves outside the storage folder.`)
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove the folders a deleted file leaves behind, stopping at the first one still in use.
|
||||
*
|
||||
* Best effort throughout: a folder that turns out not to be empty, or that another request is
|
||||
* writing into at that moment, is simply left alone.
|
||||
*/
|
||||
export async function pruneEmptyDirs(root: string, fromDir: string): Promise<void> {
|
||||
let dir = fromDir
|
||||
while (dir !== root && dir.startsWith(root + path.sep)) {
|
||||
try {
|
||||
await fs.rmdir(dir)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
dir = path.dirname(dir)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a file, creating its folder and leaving nothing half-written behind.
|
||||
*
|
||||
* Written under a temporary name and renamed, so a reader either finds the previous contents or the
|
||||
* new ones — never the middle of a write. That matters here more than it does for a cache: this is
|
||||
* the only copy of an asset once the database target has been purged.
|
||||
*/
|
||||
export async function writeFileAtomic(filePath: string, data: Buffer | string): Promise<void> {
|
||||
const tempPath = `${filePath}.${process.pid}.tmp`
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true })
|
||||
try {
|
||||
await fs.writeFile(tempPath, data)
|
||||
await fs.rename(tempPath, filePath)
|
||||
} catch (err) {
|
||||
await fs.rm(tempPath, { force: true }).catch(() => {})
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Move a file, coping with a root that spans devices and with the file not being there.
|
||||
*
|
||||
* @returns Whether anything was moved
|
||||
*/
|
||||
export async function moveFile(from: string, to: string): Promise<boolean> {
|
||||
await fs.mkdir(path.dirname(to), { recursive: true })
|
||||
try {
|
||||
await fs.rename(from, to)
|
||||
return true
|
||||
} catch (err: any) {
|
||||
if (err.code === 'ENOENT') {
|
||||
return false
|
||||
}
|
||||
if (err.code !== 'EXDEV') {
|
||||
throw err
|
||||
}
|
||||
// -> `rename` cannot cross a mount point
|
||||
await fs.copyFile(from, to)
|
||||
await fs.rm(from, { force: true })
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Follow a rename, given where the file was and where it now belongs.
|
||||
*
|
||||
* Either end may be nowhere: the layout can have no place for a locale, and a move may cross into or
|
||||
* out of it. Moving *into* it has nothing to move — the copy was never written — whereas moving out
|
||||
* of it leaves a file behind at the old path, so that one is a delete.
|
||||
*
|
||||
* @returns What actually happened, which a target keeping history needs in order to record it
|
||||
*/
|
||||
export async function moveStored(
|
||||
root: string,
|
||||
fromRelPath: string | null,
|
||||
toRelPath: string | null
|
||||
): Promise<'moved' | 'deleted' | 'nothing'> {
|
||||
if (!fromRelPath) {
|
||||
return 'nothing'
|
||||
}
|
||||
const from = absPathIn(root, fromRelPath)
|
||||
if (!toRelPath) {
|
||||
await fs.rm(from, { force: true })
|
||||
await pruneEmptyDirs(root, path.dirname(from))
|
||||
return 'deleted'
|
||||
}
|
||||
if (await moveFile(from, absPathIn(root, toRelPath))) {
|
||||
await pruneEmptyDirs(root, path.dirname(from))
|
||||
return 'moved'
|
||||
}
|
||||
return 'nothing'
|
||||
}
|
||||
|
||||
/** The metadata every page file carries, whichever of the two forms it is written in. */
|
||||
function pageMeta(page: StoragePageContent): Record<string, any> {
|
||||
return {
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
published: page.isPublished,
|
||||
date: page.updatedAt.toISOString(),
|
||||
tags: page.tags,
|
||||
// -> The declaration that makes this a page rather than a file that happens to sit here. Nothing
|
||||
// is imported as a page without it, so it is the one key that must always be written.
|
||||
editor: page.editor,
|
||||
dateCreated: page.createdAt.toISOString()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A page as a file that stands on its own, in one of two forms.
|
||||
*
|
||||
* A **text** page is YAML front matter and then the source as the author wrote it — the convention
|
||||
* every static site generator reads and the one Wiki.js 2.x wrote, so the folder is worth something
|
||||
* to tools that have never heard of this wiki.
|
||||
*
|
||||
* A **JSON** page — a redirection today — is a single JSON document with the same metadata at its top
|
||||
* level and the source under `content`. Front matter would leave a `.json` file that is not JSON,
|
||||
* which is worth avoiding for the one editor whose source is already structured.
|
||||
*
|
||||
* Either way the metadata is the point: a bare body says nothing about whether it was published or
|
||||
* what it was called, and none of that is recoverable from the prose.
|
||||
*/
|
||||
export function serializePage(ref: StoragePageRef, page: StoragePageContent): string {
|
||||
if (pageFileExtension(ref.contentType) === JSON_EXTENSION) {
|
||||
let content: any = page.content
|
||||
try {
|
||||
content = JSON.parse(page.content)
|
||||
} catch {
|
||||
// -> Kept as the string it is. The column is written by the editor and should always parse,
|
||||
// and a file that says what it holds beats one this refused to write.
|
||||
}
|
||||
return `${JSON.stringify({ ...pageMeta(page), content }, null, 2)}\n`
|
||||
}
|
||||
return `---\n${dumpYaml(pageMeta(page))}---\n\n${page.content}\n`
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a page file back: its declaration, and the source below it.
|
||||
*
|
||||
* A page file says it is one by carrying an `editor` — in its front matter, or at the top level of
|
||||
* the JSON document for the one editor written that way. That declaration is what lets a page and an
|
||||
* attachment share a folder without a module having to guess which is which from an extension, and
|
||||
* it is how every file written here comes back.
|
||||
*
|
||||
* Returning null does not settle it. A file whose extension the site reserves for pages is a page
|
||||
* whatever it does or does not declare — see `importTree`, which owns that rule and fills the editor
|
||||
* in from the extension. This only reports whether the file said so itself.
|
||||
*
|
||||
* Beyond the one key it is forgiving: a file may have been hand-written or generated by something
|
||||
* that has never seen this wiki, so a missing title or date is filled in by the caller, and front
|
||||
* matter that is not YAML is treated as no declaration rather than as a fault.
|
||||
*
|
||||
* @returns The declaration and the source, or null for a file that does not declare itself a page
|
||||
*/
|
||||
export function deserializePage(
|
||||
raw: string,
|
||||
ext: string
|
||||
): { meta: Record<string, any>; content: string } | null {
|
||||
if (ext === JSON_EXTENSION) {
|
||||
let parsed: any
|
||||
try {
|
||||
parsed = JSON.parse(raw)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
if (!parsed || typeof parsed !== 'object' || typeof parsed.editor !== 'string') {
|
||||
return null
|
||||
}
|
||||
const { content, ...meta } = parsed
|
||||
return {
|
||||
meta,
|
||||
content: typeof content === 'string' ? content : JSON.stringify(content ?? {})
|
||||
}
|
||||
}
|
||||
|
||||
const match = FRONT_MATTER.exec(raw)
|
||||
if (!match) {
|
||||
return null
|
||||
}
|
||||
let meta: Record<string, any>
|
||||
try {
|
||||
const parsed = loadYaml(match[1])
|
||||
if (!parsed || typeof parsed !== 'object') {
|
||||
return null
|
||||
}
|
||||
meta = parsed as Record<string, any>
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
if (typeof meta.editor !== 'string' || !meta.editor) {
|
||||
return null
|
||||
}
|
||||
return { meta, content: raw.slice(match[0].length).trim() }
|
||||
}
|
||||
|
||||
/** A front matter date, or undefined for one that is missing or not a date at all. */
|
||||
export function parseFileDate(value: unknown): Date | undefined {
|
||||
if (value instanceof Date) {
|
||||
return value
|
||||
}
|
||||
if (typeof value !== 'string') {
|
||||
return undefined
|
||||
}
|
||||
const date = new Date(value)
|
||||
return Number.isNaN(date.getTime()) ? undefined : date
|
||||
}
|
||||
|
||||
/** One stored file, as `walkStored` reports it. */
|
||||
export interface StoredFile {
|
||||
/** Absolute. */
|
||||
filePath: string
|
||||
/** Relative to the root, split — the whole path including the file name. */
|
||||
segments: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Every file under a root, with anything hidden — and so a repository's `.git` — left out.
|
||||
*
|
||||
* @returns Null when the root does not exist yet, which is not a fault: a target can be configured
|
||||
* long before anything is written to it
|
||||
*/
|
||||
export async function walkStored(root: string): Promise<StoredFile[] | null> {
|
||||
let entries
|
||||
try {
|
||||
entries = await fs.readdir(root, { recursive: true, withFileTypes: true })
|
||||
} catch (err: any) {
|
||||
if (err.code !== 'ENOENT') {
|
||||
throw err
|
||||
}
|
||||
return null
|
||||
}
|
||||
const files: StoredFile[] = []
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) {
|
||||
continue
|
||||
}
|
||||
const filePath = path.join(entry.parentPath, entry.name)
|
||||
const segments = path.relative(root, filePath).split(path.sep)
|
||||
if (segments.some((segment) => IGNORED_SEGMENT.test(segment))) {
|
||||
continue
|
||||
}
|
||||
files.push({ filePath, segments })
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
/**
|
||||
* How a run of `importTree` went, for the module to report in its own words.
|
||||
*
|
||||
* A module says what the run *was* — an import from a folder, a pull from a remote — and this says
|
||||
* what it did, which is the same either way.
|
||||
*/
|
||||
export interface ImportSummary {
|
||||
pages: number
|
||||
assets: number
|
||||
/** Left alone because the wiki already had something at that path and `overwrite` was off. */
|
||||
skipped: number
|
||||
/** Unusable: an empty body, an editor this wiki does not have, a path it cannot address. */
|
||||
failed: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Take a tree of files into the wiki, either filling in what is missing or letting the files win.
|
||||
*
|
||||
* **Two ways a file is a page**, and everything else is an attachment, read where it lies and adopted
|
||||
* in place rather than copied:
|
||||
*
|
||||
* 1. **Its extension is one the site reserves for pages.** Those extensions address a page by URL and
|
||||
* cannot be uploaded as attachments, so nothing else can be sitting under one — which makes a
|
||||
* hand-written `.md` dropped into the folder a page, as whoever dropped it meant.
|
||||
* 2. **It declares an editor**, in its front matter or at the top level of its JSON. This is what
|
||||
* every file a target writes carries, and it is the only way in for an extension the site does not
|
||||
* reserve — `.adoc` on a default site, where an attachment could just as well be sitting.
|
||||
*
|
||||
* Which editor it belongs to comes from that declaration, and only falls back to the extension for a
|
||||
* reserved one that made none.
|
||||
*
|
||||
* `overwrite` is the only thing separating the safe direction from the authoritative one. Off, a path
|
||||
* the wiki already has is left alone on both sides, which makes the run repeatable and makes it no
|
||||
* use for a file edited on both sides — reconciling those is a merge, and a target without history
|
||||
* cannot do one. On, the file wins: for a restore, or for a target whose remote is the authority.
|
||||
*
|
||||
* @param files What to take in, or null to walk the root
|
||||
* @param readFile How to read one, for a tree that is not on this machine — the SFTP target hands in
|
||||
* its own and everything else about the walk is the same
|
||||
*/
|
||||
export async function importTree({
|
||||
target,
|
||||
root,
|
||||
actorId,
|
||||
overwrite,
|
||||
files,
|
||||
readFile = (filePath) => fs.readFile(filePath)
|
||||
}: {
|
||||
target: StorageTarget
|
||||
root: string
|
||||
actorId: string
|
||||
overwrite: boolean
|
||||
files?: StoredFile[] | null
|
||||
readFile?: (filePath: string) => Promise<Buffer>
|
||||
}): Promise<ImportSummary | null> {
|
||||
const found = files === undefined ? await walkStored(root) : files
|
||||
if (!found) {
|
||||
return null
|
||||
}
|
||||
|
||||
const reserved: string[] = WIKI.sites[target.siteId]?.config?.pageExtensions ?? []
|
||||
const summary: ImportSummary = { pages: 0, assets: 0, skipped: 0, failed: 0 }
|
||||
|
||||
for (const { filePath, segments } of found) {
|
||||
// -> The prefix the site's layout writes, read back off the path. Null for a file that is not
|
||||
// part of this site's tree: another site's folder, or one sitting outside the layout.
|
||||
const stored = WIKI.models.storage.parseStoredPath(target.siteId, segments)
|
||||
if (!stored) {
|
||||
continue
|
||||
}
|
||||
const locale = stored.locale
|
||||
const rest = stored.segments
|
||||
const fileName = rest.pop()!
|
||||
const ext = path.extname(fileName).replace(/^\./, '').toLowerCase()
|
||||
const folderPath = rest.join('/')
|
||||
|
||||
let raw: Buffer
|
||||
try {
|
||||
raw = await readFile(filePath)
|
||||
} catch (err: any) {
|
||||
// -> A path a diff named but that is no longer there. Nothing to import and nothing wrong.
|
||||
if (err.code === 'ENOENT') {
|
||||
continue
|
||||
}
|
||||
throw err
|
||||
}
|
||||
const declared = deserializePage(raw.toString('utf8'), ext)
|
||||
const isReservedExtension = Boolean(ext) && reserved.includes(ext)
|
||||
|
||||
if (!declared && !isReservedExtension) {
|
||||
const asset = await WIKI.models.assets.adoptStoredFile({
|
||||
siteId: target.siteId,
|
||||
locale,
|
||||
folderPath,
|
||||
fileName,
|
||||
data: raw,
|
||||
authorId: actorId,
|
||||
overwrite
|
||||
})
|
||||
if (asset) {
|
||||
summary.assets++
|
||||
} else {
|
||||
summary.skipped++
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// -> A page is addressed without its extension: `guides/setup.md` is the page at `guides/setup`
|
||||
const name = fileName.slice(0, fileName.length - (ext ? ext.length + 1 : 0))
|
||||
const meta = declared?.meta ?? {}
|
||||
try {
|
||||
const imported = await WIKI.models.pages.adoptStoredPage({
|
||||
siteId: target.siteId,
|
||||
locale,
|
||||
// -> The root page is filed under a name of its own, and takes the empty path back
|
||||
path: [...rest, ...(name === ROOT_PAGE_NAME ? [] : [name])].join('/'),
|
||||
// -> The declaration first, then the file itself: a page written by hand carries no title,
|
||||
// and its name is the next best thing
|
||||
title: typeof meta.title === 'string' && meta.title ? meta.title : name,
|
||||
description: typeof meta.description === 'string' ? meta.description : '',
|
||||
editor:
|
||||
typeof meta.editor === 'string' && meta.editor
|
||||
? meta.editor
|
||||
: (pageEditorForExtension(ext) ?? DEFAULT_PAGE_EDITOR),
|
||||
tags: Array.isArray(meta.tags) ? meta.tags.map(String) : [],
|
||||
isPublished: meta.published !== false,
|
||||
// -> Undeclared means there was no front matter to strip, so the file is all body
|
||||
content: declared?.content ?? raw.toString('utf8').trim(),
|
||||
createdAt: parseFileDate(meta.dateCreated),
|
||||
updatedAt: parseFileDate(meta.date),
|
||||
authorId: actorId,
|
||||
overwrite
|
||||
})
|
||||
if (imported) {
|
||||
summary.pages++
|
||||
} else {
|
||||
summary.skipped++
|
||||
}
|
||||
} catch (err: any) {
|
||||
// -> One unusable file must not stop the rest of the tree from being imported
|
||||
summary.failed++
|
||||
WIKI.logger.warn(`Could not import the page at ${filePath} [ SKIPPED ]`)
|
||||
WIKI.logger.warn(err.message)
|
||||
}
|
||||
}
|
||||
|
||||
return summary
|
||||
}
|
||||
@ -0,0 +1,268 @@
|
||||
import mime from 'mime'
|
||||
import { assetRelPath, pageRelPath, serializePage } from './storageFiles.ts'
|
||||
import type { StorageModule, StorageTarget } from '../models/storage.ts'
|
||||
|
||||
/**
|
||||
* The shared half of every object-store target — S3, Azure Blob Storage, Google Cloud Storage.
|
||||
*
|
||||
* All three answer the same four questions (put, get, remove, copy) against a flat namespace of keys,
|
||||
* and everything above that is identical between them: which key a page or an asset takes, how a
|
||||
* rename is done where there is no rename, what a bulk export walks. That part lives here, so a
|
||||
* module is its client and nothing else.
|
||||
*
|
||||
* **A key is a path**, the same one the disk target would write — `pathPrefixFor` decides what
|
||||
* brackets it, and pages and assets sit beside each other in it exactly as they do in a folder. An
|
||||
* object store has no directories, so the slashes are just characters in a name, which is why there is
|
||||
* nothing here about creating or pruning them.
|
||||
*
|
||||
* Not under `modules/storage/`, for the reason `storageFiles.ts` gives: a directory there without a
|
||||
* `definition.yml` takes every storage module down with it.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A direct-access URL as the shared layer asks for one.
|
||||
*
|
||||
* `key` rather than a ref, because signing is about an object and not about the wiki: the store has
|
||||
* to know what to declare the response as and whether to make the browser save it, both of which the
|
||||
* wiki knows and the object may not have been stored with.
|
||||
*/
|
||||
export interface PresignRequest {
|
||||
key: string
|
||||
expiresInSeconds: number
|
||||
contentType: string
|
||||
/** The file name to save as, when the browser should save rather than display. */
|
||||
downloadAs?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* The origin a signed URL should be built on, or null for the store's own.
|
||||
*
|
||||
* Normalized to no trailing slash so that a module can always join it to a key with one, however the
|
||||
* administrator typed it.
|
||||
*/
|
||||
export function signingBaseUrl(target: StorageTarget): string | null {
|
||||
const configured = target.assetDelivery.baseUrl?.trim()
|
||||
return configured ? configured.replace(/\/+$/, '') : null
|
||||
}
|
||||
|
||||
/** What a store has to be able to do for `objectStorageModule` to build a target out of it. */
|
||||
export interface ObjectStoreClient {
|
||||
/** Write an object, replacing whatever was at that key. */
|
||||
put: (target: StorageTarget, key: string, data: Buffer, contentType: string) => Promise<void>
|
||||
/** Read one back, or null when the store does not have it. Must not throw for a missing key. */
|
||||
get: (target: StorageTarget, key: string) => Promise<Buffer | null>
|
||||
/** Drop one. Must not throw for a key that is already gone. */
|
||||
remove: (target: StorageTarget, key: string) => Promise<void>
|
||||
/**
|
||||
* Copy one key onto another, server-side where the store can.
|
||||
*
|
||||
* @returns Whether there was anything at the source. False rather than a throw, because a target
|
||||
* enabled after an upload legitimately has no copy of the file being moved.
|
||||
*/
|
||||
copy: (target: StorageTarget, fromKey: string, toKey: string) => Promise<boolean>
|
||||
/**
|
||||
* Sign a URL a reader can fetch the object from without going through the wiki.
|
||||
*
|
||||
* Optional only in the type: all three object stores implement it, and a store that could not
|
||||
* would declare `isDirectAccessSupported: false` and never be asked.
|
||||
*/
|
||||
presign?: (target: StorageTarget, request: PresignRequest) => Promise<string | null>
|
||||
}
|
||||
|
||||
/**
|
||||
* What to declare an object as, so that a store handing it straight to a browser says the right thing.
|
||||
*
|
||||
* Guessed from the name rather than taken from the asset, because the reference a target is given
|
||||
* carries the file's size and kind but not its type — and the name is what the wiki itself resolves
|
||||
* the served type from, so guessing the same way keeps the two in step.
|
||||
*/
|
||||
function contentTypeOf(fileName: string): string {
|
||||
return mime.getType(fileName) ?? 'application/octet-stream'
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn a client into a storage module.
|
||||
*
|
||||
* The eight contract methods plus `exportAll`, which is the one action all three declare. A module
|
||||
* spreads the result and adds nothing, unless its store can do something the others cannot.
|
||||
*/
|
||||
export function objectStorageModule(client: ObjectStoreClient): StorageModule {
|
||||
const module: StorageModule = {
|
||||
canStore(target, ref) {
|
||||
return WIKI.models.storage.pathPrefixFor(target.siteId, ref.locale) !== null
|
||||
},
|
||||
|
||||
async putAsset(target, ref, data) {
|
||||
const key = assetRelPath(target, ref)
|
||||
// -> Guarded rather than skipped: the model asks `canStore` before dispatching a write, so
|
||||
// reaching this means somebody wrote without asking, and an asset's bytes may exist nowhere
|
||||
// else
|
||||
if (!key) {
|
||||
throw new Error(
|
||||
`${target.title} has no path for ${ref.locale} content, so ${ref.fileName} cannot be stored there.`
|
||||
)
|
||||
}
|
||||
await client.put(target, key, data, contentTypeOf(ref.fileName))
|
||||
},
|
||||
|
||||
async getAsset(target, ref) {
|
||||
const key = assetRelPath(target, ref)
|
||||
return key ? client.get(target, key) : null
|
||||
},
|
||||
|
||||
async deleteAsset(target, ref) {
|
||||
const key = assetRelPath(target, ref)
|
||||
if (key) {
|
||||
await client.remove(target, key)
|
||||
}
|
||||
},
|
||||
|
||||
async moveAsset(target, ref, previous) {
|
||||
await moveObject(
|
||||
client,
|
||||
target,
|
||||
assetRelPath(target, { ...ref, ...previous }),
|
||||
assetRelPath(target, ref)
|
||||
)
|
||||
},
|
||||
|
||||
async putPage(target, ref, page) {
|
||||
const key = pageRelPath(target, ref)
|
||||
// -> Unlike an asset, a page with no place here is not worth failing over: it is in the
|
||||
// database, which is where a page always is, and this copy is the thing the site declined
|
||||
if (!key) {
|
||||
return
|
||||
}
|
||||
await client.put(
|
||||
target,
|
||||
key,
|
||||
Buffer.from(serializePage(ref, page), 'utf8'),
|
||||
contentTypeOf(key)
|
||||
)
|
||||
},
|
||||
|
||||
async deletePage(target, ref) {
|
||||
const key = pageRelPath(target, ref)
|
||||
if (key) {
|
||||
await client.remove(target, key)
|
||||
}
|
||||
},
|
||||
|
||||
async movePage(target, ref, previousPath) {
|
||||
await moveObject(
|
||||
client,
|
||||
target,
|
||||
pageRelPath(target, { ...ref, path: previousPath }),
|
||||
pageRelPath(target, ref)
|
||||
)
|
||||
},
|
||||
|
||||
...(client.presign
|
||||
? {
|
||||
async presignAsset(target, ref, options) {
|
||||
const key = assetRelPath(target, ref)
|
||||
if (!key) {
|
||||
return null
|
||||
}
|
||||
return client.presign!(target, { key, ...options })
|
||||
}
|
||||
}
|
||||
: {}),
|
||||
|
||||
/**
|
||||
* Write a copy of everything this target is configured to hold into the store.
|
||||
*
|
||||
* How content that predates the target being enabled gets into it: an upload only ever goes to
|
||||
* the targets enabled at the time, so a store turned on today holds nothing from yesterday. A
|
||||
* plain copy and nothing more — no database row is touched, nothing is repointed, and running it
|
||||
* twice does the same work to the same effect.
|
||||
*/
|
||||
async exportAll(target: StorageTarget): Promise<string> {
|
||||
let assets = 0
|
||||
let unreadable = 0
|
||||
let unstored = 0
|
||||
|
||||
for (const asset of await WIKI.models.assets.listStoredAssets(target.siteId)) {
|
||||
const contentType = WIKI.models.storage.contentTypeFor(
|
||||
target.siteId,
|
||||
asset.kind,
|
||||
asset.fileSize
|
||||
)
|
||||
if (!target.contentTypes.activeTypes.includes(contentType)) {
|
||||
continue
|
||||
}
|
||||
if (!assetRelPath(target, asset)) {
|
||||
unstored++
|
||||
continue
|
||||
}
|
||||
const data = await WIKI.models.storage.getAsset(asset)
|
||||
if (!data) {
|
||||
unreadable++
|
||||
continue
|
||||
}
|
||||
await module.putAsset(target, asset, data)
|
||||
assets++
|
||||
}
|
||||
|
||||
let pages = 0
|
||||
if (target.contentTypes.activeTypes.includes('pages')) {
|
||||
for (const { ref, content } of await WIKI.models.pages.listForStorage(target.siteId)) {
|
||||
if (!pageRelPath(target, ref)) {
|
||||
unstored++
|
||||
continue
|
||||
}
|
||||
await module.putPage(target, ref, content)
|
||||
pages++
|
||||
}
|
||||
}
|
||||
|
||||
WIKI.logger.info(`Exported ${assets} asset(s) and ${pages} page(s) to ${target.title} [ OK ]`)
|
||||
const parts = []
|
||||
if (assets > 0 || pages > 0) {
|
||||
parts.push(`Exported ${pages} page(s) and ${assets} asset(s).`)
|
||||
} else {
|
||||
parts.push('There was nothing to export.')
|
||||
}
|
||||
if (unreadable > 0) {
|
||||
parts.push(`${unreadable} asset(s) could not be read and were skipped.`)
|
||||
}
|
||||
if (unstored > 0) {
|
||||
const { primaryLocale } = WIKI.models.storage.pathLayoutFor(target.siteId)
|
||||
parts.push(
|
||||
`${unstored} item(s) are not in the ${primaryLocale} locale, which is the only one this site stores.`
|
||||
)
|
||||
}
|
||||
return parts.join(' ')
|
||||
}
|
||||
}
|
||||
|
||||
return module
|
||||
}
|
||||
|
||||
/**
|
||||
* Follow a rename, which in an object store is a copy and a delete.
|
||||
*
|
||||
* Either end may be nowhere, as on disk: the layout can have no path for a locale, and a move may
|
||||
* cross into or out of it. Moving *into* it has nothing to copy from; moving out of it leaves an
|
||||
* object behind at the old key, so that one is a delete.
|
||||
*
|
||||
* The delete only happens once the copy has reported success, so a store that fails halfway leaves
|
||||
* the file at its old key rather than nowhere.
|
||||
*/
|
||||
async function moveObject(
|
||||
client: ObjectStoreClient,
|
||||
target: StorageTarget,
|
||||
fromKey: string | null,
|
||||
toKey: string | null
|
||||
): Promise<void> {
|
||||
if (!fromKey) {
|
||||
return
|
||||
}
|
||||
if (!toKey) {
|
||||
await client.remove(target, fromKey)
|
||||
return
|
||||
}
|
||||
if (await client.copy(target, fromKey, toKey)) {
|
||||
await client.remove(target, fromKey)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
key: azure
|
||||
title: Azure Blob Storage
|
||||
icon: '/_assets/icons/ultraviolet-azure.svg'
|
||||
banner: '/_assets/storage/azure.jpg'
|
||||
description: Azure Blob Storage is Microsoft's object storage solution for the cloud, optimized for storing large amounts of unstructured data.
|
||||
assetDelivery:
|
||||
isDirectAccessSupported: true
|
||||
contentTypes:
|
||||
defaultTypesEnabled: ['images', 'documents', 'others', 'large']
|
||||
props:
|
||||
accountName:
|
||||
type: String
|
||||
title: Account Name
|
||||
default: ''
|
||||
hint: Your unique storage account name.
|
||||
icon: 3d-touch
|
||||
order: 1
|
||||
accountKey:
|
||||
type: String
|
||||
title: Account Access Key
|
||||
default: ''
|
||||
hint: Either key 1 or key 2 from the storage account. Leave empty to use the credentials the machine already has, such as a managed identity.
|
||||
icon: key
|
||||
sensitive: true
|
||||
order: 2
|
||||
containerName:
|
||||
type: String
|
||||
title: Container Name
|
||||
default: wiki
|
||||
hint: The container to store content in. It is created on first use if it does not exist yet.
|
||||
icon: shipping-container
|
||||
order: 3
|
||||
accessTier:
|
||||
type: String
|
||||
title: Access Tier
|
||||
default: Cool
|
||||
hint: What new blobs are stored as. Cool costs less to keep and more to read, which suits content that is served from another target and kept here as a copy.
|
||||
icon: scan-stock
|
||||
order: 4
|
||||
enum:
|
||||
- Hot|Hot
|
||||
- Cool|Cool
|
||||
- Cold|Cold
|
||||
- Archive|Archive
|
||||
actions:
|
||||
exportAll:
|
||||
label: Export Everything
|
||||
hint: Write a copy of every page and asset this target is configured to hold to the container, overwriting whatever is already there. Nothing in the database is changed and nothing is moved, so this is how content created before the target was enabled gets onto it.
|
||||
icon: this-way-up
|
||||
@ -0,0 +1,171 @@
|
||||
import {
|
||||
BlobSASPermissions,
|
||||
BlobServiceClient,
|
||||
StorageSharedKeyCredential,
|
||||
generateBlobSASQueryParameters
|
||||
} from '@azure/storage-blob'
|
||||
import { DefaultAzureCredential } from '@azure/identity'
|
||||
import { objectStorageModule, signingBaseUrl } from '../../../helpers/storageObjects.ts'
|
||||
import type { ContainerClient } from '@azure/storage-blob'
|
||||
import type { ObjectStoreClient } from '../../../helpers/storageObjects.ts'
|
||||
import type { StorageTarget } from '../../../models/storage.ts'
|
||||
|
||||
/** Live container clients, keyed by target, plus whether the container has been ensured. */
|
||||
const containers = new Map<string, { container: ContainerClient; fingerprint: string }>()
|
||||
|
||||
/** The settings a client is built from — a change to any of them needs a new one. */
|
||||
function configFingerprint(target: StorageTarget): string {
|
||||
const c = target.config
|
||||
return JSON.stringify([c.accountName, c.accountKey, c.containerName])
|
||||
}
|
||||
|
||||
/**
|
||||
* The container client for this target, created once and kept.
|
||||
*
|
||||
* The container is created on the way, which is the one place these three modules differ in what they
|
||||
* will do for you: a container is namespaced under the storage account and costs nothing to make,
|
||||
* whereas an S3 bucket is a global name and a GCS bucket is billable, so both of those are the
|
||||
* administrator's to create.
|
||||
*
|
||||
* **The account key is optional.** Left empty, `DefaultAzureCredential` is used instead — a managed
|
||||
* identity on an Azure VM or container app, or the standard `AZURE_*` environment variables — which is
|
||||
* how a deployment avoids putting a long-lived key in the database at all.
|
||||
*/
|
||||
async function containerFor(target: StorageTarget): Promise<ContainerClient> {
|
||||
const fingerprint = configFingerprint(target)
|
||||
const cached = containers.get(target.id)
|
||||
if (cached && cached.fingerprint === fingerprint) {
|
||||
return cached.container
|
||||
}
|
||||
const { accountName, accountKey, containerName } = target.config
|
||||
const url = `https://${accountName}.blob.core.windows.net`
|
||||
const service = accountKey
|
||||
? new BlobServiceClient(url, new StorageSharedKeyCredential(accountName, accountKey))
|
||||
: new BlobServiceClient(url, new DefaultAzureCredential())
|
||||
const container = service.getContainerClient(containerName || 'wiki')
|
||||
await container.createIfNotExists()
|
||||
containers.set(target.id, { container, fingerprint })
|
||||
return container
|
||||
}
|
||||
|
||||
/**
|
||||
* A user delegation key, for an account authenticating as itself rather than with a shared key.
|
||||
*
|
||||
* The managed-identity path: with no account key there is nothing to sign a SAS with, so Azure is
|
||||
* asked for a short-lived key to sign with instead. It needs the **Storage Blob Delegator** role on
|
||||
* the account, and it is what makes direct access work without a long-lived secret in the database.
|
||||
*
|
||||
* Cached until shortly before it expires, since fetching one is a round trip and every image on every
|
||||
* page would otherwise pay for it.
|
||||
*/
|
||||
const delegationKeys = new Map<string, { key: any; expiresAt: number }>()
|
||||
|
||||
/** How long a delegation key is asked for, and how much of that is left unused as a safety margin. */
|
||||
const DELEGATION_KEY_MINUTES = 60
|
||||
const DELEGATION_KEY_MARGIN_MS = 5 * 60_000
|
||||
|
||||
async function delegationKeyFor(target: StorageTarget): Promise<any> {
|
||||
const cached = delegationKeys.get(target.id)
|
||||
const now = Date.now()
|
||||
if (cached && cached.expiresAt - DELEGATION_KEY_MARGIN_MS > now) {
|
||||
return cached.key
|
||||
}
|
||||
const service = new BlobServiceClient(
|
||||
`https://${target.config.accountName}.blob.core.windows.net`,
|
||||
new DefaultAzureCredential()
|
||||
)
|
||||
const expiresAt = now + DELEGATION_KEY_MINUTES * 60_000
|
||||
const key = await service.getUserDelegationKey(new Date(now), new Date(expiresAt))
|
||||
delegationKeys.set(target.id, { key, expiresAt })
|
||||
return key
|
||||
}
|
||||
|
||||
/** Whether the service is telling us the blob simply is not there. */
|
||||
function isNotFound(err: any): boolean {
|
||||
return err?.statusCode === 404 || err?.details?.errorCode === 'BlobNotFound'
|
||||
}
|
||||
|
||||
const azureClient: ObjectStoreClient = {
|
||||
async put(target, key, data, contentType) {
|
||||
const blob = (await containerFor(target)).getBlockBlobClient(key)
|
||||
await blob.uploadData(data, {
|
||||
blobHTTPHeaders: { blobContentType: contentType },
|
||||
...(target.config.accessTier ? { tier: target.config.accessTier } : {})
|
||||
})
|
||||
},
|
||||
|
||||
async get(target, key) {
|
||||
try {
|
||||
return await (await containerFor(target)).getBlockBlobClient(key).downloadToBuffer()
|
||||
} catch (err: any) {
|
||||
if (isNotFound(err)) {
|
||||
// -> This target does not have the file: enabled after the upload, or removed from outside
|
||||
// the wiki. Not a fault — the caller asks the next target.
|
||||
return null
|
||||
}
|
||||
throw err
|
||||
}
|
||||
},
|
||||
|
||||
async remove(target, key) {
|
||||
await (await containerFor(target)).getBlockBlobClient(key).deleteIfExists()
|
||||
},
|
||||
|
||||
async copy(target, fromKey, toKey) {
|
||||
const container = await containerFor(target)
|
||||
const source = container.getBlockBlobClient(fromKey)
|
||||
if (!(await source.exists())) {
|
||||
return false
|
||||
}
|
||||
// -> Server-side, and awaited: the destination has to be complete before the caller deletes the
|
||||
// source, and `beginCopyFromURL` is only a promise that the copy has *started*
|
||||
const copy = await container.getBlockBlobClient(toKey).beginCopyFromURL(source.url)
|
||||
await copy.pollUntilDone()
|
||||
return true
|
||||
},
|
||||
|
||||
async presign(target, { key, expiresInSeconds, contentType, downloadAs }) {
|
||||
const { accountName, accountKey, containerName } = target.config
|
||||
const container = containerName || 'wiki'
|
||||
const now = Date.now()
|
||||
const values = {
|
||||
containerName: container,
|
||||
blobName: key,
|
||||
permissions: BlobSASPermissions.parse('r'),
|
||||
// -> A minute of slack at the front, because the reader's clock and Azure's need not agree and
|
||||
// a SAS that is not valid yet fails exactly as hard as one that has expired
|
||||
startsOn: new Date(now - 60_000),
|
||||
expiresOn: new Date(now + expiresInSeconds * 1000),
|
||||
contentType,
|
||||
...(downloadAs
|
||||
? { contentDisposition: `attachment; filename="${encodeURIComponent(downloadAs)}"` }
|
||||
: {})
|
||||
}
|
||||
|
||||
const sas = accountKey
|
||||
? generateBlobSASQueryParameters(
|
||||
values,
|
||||
new StorageSharedKeyCredential(accountName, accountKey)
|
||||
)
|
||||
: generateBlobSASQueryParameters(values, await delegationKeyFor(target), accountName)
|
||||
|
||||
/*
|
||||
Azure signs the canonicalized resource — the account, the container and the blob — and not the
|
||||
host, which is the one thing that makes this simpler than S3 and GCS: a CDN or Front Door
|
||||
endpoint in front of the container can be put in front of a signature made for the account, and
|
||||
Azure still validates it when the request reaches the origin.
|
||||
*/
|
||||
const base = signingBaseUrl(target)
|
||||
const origin = base ?? `https://${accountName}.blob.core.windows.net/${container}`
|
||||
return `${origin}/${key.split('/').map(encodeURIComponent).join('/')}?${sas.toString()}`
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Azure Blob Storage module
|
||||
*
|
||||
* Blob names are the same paths the disk target writes, so a container and a folder hold the wiki's
|
||||
* content laid out identically, and the shape of both is the site's `pathPrefixFor` to decide. See
|
||||
* `helpers/storageObjects.ts` for everything above the four calls below.
|
||||
*/
|
||||
export default objectStorageModule(azureClient)
|
||||
@ -0,0 +1,57 @@
|
||||
key: gcs
|
||||
title: Google Cloud Storage
|
||||
icon: '/_assets/icons/ultraviolet-google.svg'
|
||||
banner: '/_assets/storage/gcs.jpg'
|
||||
description: Google Cloud Storage is an online file storage web service for storing and accessing data on Google Cloud Platform infrastructure.
|
||||
assetDelivery:
|
||||
isDirectAccessSupported: true
|
||||
contentTypes:
|
||||
defaultTypesEnabled: ['images', 'documents', 'others', 'large']
|
||||
props:
|
||||
projectId:
|
||||
type: String
|
||||
title: Project ID
|
||||
default: ''
|
||||
hint: The project ID from the Google Cloud console, e.g. grape-spaceship-123. Optional when the credentials below name one.
|
||||
icon: 3d-touch
|
||||
order: 1
|
||||
credentialsJSON:
|
||||
type: String
|
||||
title: JSON Credentials
|
||||
default: ''
|
||||
hint: Contents of the JSON key file for a service account with Storage Object Admin on the bucket. Leave empty to use Application Default Credentials, which is what a workload identity on GKE or Cloud Run provides.
|
||||
icon: key
|
||||
multiline: true
|
||||
sensitive: true
|
||||
order: 2
|
||||
bucket:
|
||||
type: String
|
||||
title: Bucket Name
|
||||
default: ''
|
||||
hint: The bucket to store content in. It must already exist - this target will not create it.
|
||||
icon: open-box
|
||||
order: 3
|
||||
storageClass:
|
||||
type: String
|
||||
title: Storage Class
|
||||
default: STANDARD
|
||||
hint: What new objects are stored as. The colder classes cost less to keep and more to read, and charge for a minimum storage duration.
|
||||
icon: scan-stock
|
||||
order: 4
|
||||
enum:
|
||||
- STANDARD|Standard
|
||||
- NEARLINE|Nearline
|
||||
- COLDLINE|Coldline
|
||||
- ARCHIVE|Archive
|
||||
apiEndpoint:
|
||||
type: String
|
||||
title: API Endpoint
|
||||
default: ''
|
||||
hint: Leave empty for Google Cloud Storage itself. Only set this to point at an emulator or a private service endpoint.
|
||||
icon: api
|
||||
order: 5
|
||||
actions:
|
||||
exportAll:
|
||||
label: Export Everything
|
||||
hint: Write a copy of every page and asset this target is configured to hold to the bucket, overwriting whatever is already there. Nothing in the database is changed and nothing is moved, so this is how content created before the target was enabled gets onto it.
|
||||
icon: this-way-up
|
||||
@ -0,0 +1,133 @@
|
||||
import { Storage } from '@google-cloud/storage'
|
||||
import { objectStorageModule, signingBaseUrl } from '../../../helpers/storageObjects.ts'
|
||||
import type { Bucket } from '@google-cloud/storage'
|
||||
import type { ObjectStoreClient } from '../../../helpers/storageObjects.ts'
|
||||
import type { StorageTarget } from '../../../models/storage.ts'
|
||||
|
||||
/** Live buckets, keyed by target. See `bucketFor`. */
|
||||
const buckets = new Map<string, { bucket: Bucket; fingerprint: string }>()
|
||||
|
||||
/** The settings a client is built from — a change to any of them needs a new one. */
|
||||
function configFingerprint(target: StorageTarget): string {
|
||||
const c = target.config
|
||||
return JSON.stringify([c.projectId, c.credentialsJSON, c.bucket, c.apiEndpoint])
|
||||
}
|
||||
|
||||
/**
|
||||
* The bucket handle for this target, built once and kept.
|
||||
*
|
||||
* **The credentials are optional.** Left empty, the client falls back to Application Default
|
||||
* Credentials — the workload identity attached to a GKE pod or a Cloud Run service, or the
|
||||
* `GOOGLE_APPLICATION_CREDENTIALS` file — which is how a deployment on Google's own infrastructure
|
||||
* avoids putting a service account key in the database at all.
|
||||
*
|
||||
* @throws When the pasted credentials are not JSON, which is worth saying plainly: it is a long blob
|
||||
* somebody pasted into a form, and the client's own error for it is not obviously about that
|
||||
*/
|
||||
function bucketFor(target: StorageTarget): Bucket {
|
||||
const fingerprint = configFingerprint(target)
|
||||
const cached = buckets.get(target.id)
|
||||
if (cached && cached.fingerprint === fingerprint) {
|
||||
return cached.bucket
|
||||
}
|
||||
const { projectId, credentialsJSON, bucket, apiEndpoint } = target.config
|
||||
|
||||
let credentials
|
||||
if (credentialsJSON?.trim()) {
|
||||
try {
|
||||
credentials = JSON.parse(credentialsJSON)
|
||||
} catch {
|
||||
throw new Error(
|
||||
'The JSON credentials for this target are not valid JSON. Paste the whole contents of the service account key file.'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const storage = new Storage({
|
||||
...(projectId ? { projectId } : {}),
|
||||
...(credentials ? { credentials } : {}),
|
||||
...(apiEndpoint ? { apiEndpoint } : {})
|
||||
})
|
||||
const handle = storage.bucket(bucket)
|
||||
buckets.set(target.id, { bucket: handle, fingerprint })
|
||||
return handle
|
||||
}
|
||||
|
||||
/** Whether the service is telling us the object simply is not there. */
|
||||
function isNotFound(err: any): boolean {
|
||||
return err?.code === 404
|
||||
}
|
||||
|
||||
const gcsClient: ObjectStoreClient = {
|
||||
async put(target, key, data, contentType) {
|
||||
await bucketFor(target)
|
||||
.file(key)
|
||||
.save(data, {
|
||||
contentType,
|
||||
...(target.config.storageClass && target.config.storageClass !== 'STANDARD'
|
||||
? { metadata: { storageClass: target.config.storageClass } }
|
||||
: {})
|
||||
})
|
||||
},
|
||||
|
||||
async get(target, key) {
|
||||
try {
|
||||
const [contents] = await bucketFor(target).file(key).download()
|
||||
return contents
|
||||
} catch (err: any) {
|
||||
if (isNotFound(err)) {
|
||||
// -> This target does not have the file: enabled after the upload, or removed from outside
|
||||
// the wiki. Not a fault — the caller asks the next target.
|
||||
return null
|
||||
}
|
||||
throw err
|
||||
}
|
||||
},
|
||||
|
||||
async remove(target, key) {
|
||||
await bucketFor(target).file(key).delete({ ignoreNotFound: true })
|
||||
},
|
||||
|
||||
async copy(target, fromKey, toKey) {
|
||||
const bucket = bucketFor(target)
|
||||
try {
|
||||
await bucket.file(fromKey).copy(bucket.file(toKey))
|
||||
return true
|
||||
} catch (err: any) {
|
||||
if (isNotFound(err)) {
|
||||
return false
|
||||
}
|
||||
throw err
|
||||
}
|
||||
},
|
||||
|
||||
async presign(target, { key, expiresInSeconds, contentType, downloadAs }) {
|
||||
const baseUrl = signingBaseUrl(target)
|
||||
const [url] = await bucketFor(target)
|
||||
.file(key)
|
||||
.getSignedUrl({
|
||||
version: 'v4',
|
||||
action: 'read',
|
||||
expires: Date.now() + expiresInSeconds * 1000,
|
||||
responseType: contentType,
|
||||
...(downloadAs ? { promptSaveAs: downloadAs } : {}),
|
||||
/*
|
||||
A V4 signature covers the host, so a URL signed for `storage.googleapis.com` and then moved
|
||||
onto a custom domain is a signature for the wrong host. `cname` is how the client is told to
|
||||
sign for that domain in the first place — the same reason the S3 module builds a second
|
||||
client rather than rewriting its output.
|
||||
*/
|
||||
...(baseUrl ? { cname: baseUrl } : {})
|
||||
})
|
||||
return url
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Google Cloud Storage module
|
||||
*
|
||||
* Object names are the same paths the disk target writes, so a bucket and a folder hold the wiki's
|
||||
* content laid out identically, and the shape of both is the site's `pathPrefixFor` to decide. See
|
||||
* `helpers/storageObjects.ts` for everything above the four calls below.
|
||||
*/
|
||||
export default objectStorageModule(gcsClient)
|
||||
@ -0,0 +1,157 @@
|
||||
key: git
|
||||
title: Git
|
||||
icon: '/_assets/icons/ultraviolet-git.svg'
|
||||
banner: '/_assets/storage/git.jpg'
|
||||
description: Keep this site's content in a Git repository, committed as it changes and synchronized with a remote. Every page and file is an ordinary versioned file, so the wiki's history is also the repository's.
|
||||
assetDelivery:
|
||||
isDirectAccessSupported: false
|
||||
contentTypes:
|
||||
defaultTypesEnabled: ['pages', 'images', 'documents', 'others']
|
||||
props:
|
||||
authType:
|
||||
type: String
|
||||
default: 'ssh'
|
||||
title: Authentication Type
|
||||
hint: Use SSH for maximum security.
|
||||
icon: security-configuration
|
||||
enum:
|
||||
- basic|Basic
|
||||
- ssh|SSH
|
||||
enumDisplay: buttons
|
||||
order: 1
|
||||
repoUrl:
|
||||
type: String
|
||||
title: Repository URI
|
||||
hint: Git-compliant URI (e.g. git@server.com:org/repo.git for SSH, https://server.com/org/repo.git for basic). Leave empty to keep a purely local repository, committed but never pushed.
|
||||
icon: dns
|
||||
order: 2
|
||||
branch:
|
||||
type: String
|
||||
default: 'main'
|
||||
title: Branch
|
||||
hint: The branch to use during pull / push. It must already exist on the remote.
|
||||
icon: code-fork
|
||||
order: 3
|
||||
syncMode:
|
||||
type: String
|
||||
default: 'sync'
|
||||
title: Sync Direction
|
||||
hint: Sync pulls and then pushes. Push force-pushes the wiki's commits and never takes anything back, so the wiki always wins. Pull only takes changes in, and is what makes the remote the authority.
|
||||
icon: synchronize
|
||||
enum:
|
||||
- sync|Sync
|
||||
- push|Push only
|
||||
- pull|Pull only
|
||||
enumDisplay: buttons
|
||||
order: 4
|
||||
sshPrivateKeyMode:
|
||||
type: String
|
||||
title: SSH Private Key Mode
|
||||
hint: The mode to use to load the private key. Fill in the corresponding field below.
|
||||
icon: grand-master-key
|
||||
order: 11
|
||||
default: inline
|
||||
enum:
|
||||
- path|File Path
|
||||
- inline|Inline Contents
|
||||
enumDisplay: buttons
|
||||
if:
|
||||
- { key: 'authType', eq: 'ssh' }
|
||||
sshPrivateKeyPath:
|
||||
type: String
|
||||
title: SSH Private Key Path
|
||||
hint: Absolute path to the key. The key must NOT be passphrase-protected.
|
||||
icon: key
|
||||
order: 12
|
||||
if:
|
||||
- { key: 'authType', eq: 'ssh' }
|
||||
- { key: 'sshPrivateKeyMode', eq: 'path' }
|
||||
sshPrivateKeyContent:
|
||||
type: String
|
||||
title: SSH Private Key Contents
|
||||
hint: Paste the contents of the private key. The key must NOT be passphrase-protected. It is written to a file readable only by the wiki's own user.
|
||||
icon: key
|
||||
multiline: true
|
||||
sensitive: true
|
||||
order: 13
|
||||
if:
|
||||
- { key: 'authType', eq: 'ssh' }
|
||||
- { key: 'sshPrivateKeyMode', eq: 'inline' }
|
||||
verifySSL:
|
||||
type: Boolean
|
||||
default: true
|
||||
title: Verify SSL Certificate
|
||||
hint: Some hosts require SSL certificate checking to be disabled. Leave enabled for proper security.
|
||||
icon: security-ssl
|
||||
order: 14
|
||||
basicUsername:
|
||||
type: String
|
||||
title: Username
|
||||
hint: Basic Authentication Only
|
||||
icon: test-account
|
||||
order: 20
|
||||
if:
|
||||
- { key: 'authType', eq: 'basic' }
|
||||
basicPassword:
|
||||
type: String
|
||||
title: Password / PAT
|
||||
hint: Basic Authentication Only
|
||||
icon: password
|
||||
sensitive: true
|
||||
order: 21
|
||||
if:
|
||||
- { key: 'authType', eq: 'basic' }
|
||||
alwaysUseDefaultAuthor:
|
||||
type: Boolean
|
||||
default: false
|
||||
title: Always Commit as the Default Author
|
||||
hint: Attribute every commit to the default author below instead of to the user who made the change, so that no account name or email address reaches the repository. Turn this on if the remote is somewhere your users' identities should not be published.
|
||||
icon: data-protection
|
||||
order: 29
|
||||
defaultName:
|
||||
type: String
|
||||
title: Default Author Name
|
||||
default: 'Wiki.js'
|
||||
hint: The commit author when the change was not made by one person - a scheduled sync, or a folder rename that moved a hundred files - and every commit when the option above is on.
|
||||
icon: customer
|
||||
order: 30
|
||||
defaultEmail:
|
||||
type: String
|
||||
title: Default Author Email
|
||||
default: 'wiki@example.com'
|
||||
hint: The commit author email in the same cases as the name above.
|
||||
icon: email
|
||||
order: 31
|
||||
localRepoPath:
|
||||
type: String
|
||||
title: Local Repository Path
|
||||
default: './data/repo'
|
||||
hint: Where the working copy is kept. Give each site its own path unless you turn on Add Site ID Prefix under Configuration, since two sites sharing a repository would otherwise write over each other. Relative paths are resolved from the Wiki.js install directory.
|
||||
icon: symlink-directory
|
||||
order: 32
|
||||
gitBinaryPath:
|
||||
type: String
|
||||
title: Git Binary Path
|
||||
default: ''
|
||||
hint: Optional - Absolute path to the Git binary, when not available in PATH. Leave empty to use the default PATH location (recommended).
|
||||
icon: run-command
|
||||
order: 50
|
||||
actions:
|
||||
sync:
|
||||
label: Force Sync
|
||||
hint: Run a sync straight away rather than waiting for the next scheduled one. The Sync Direction above is respected, and a pull applies what it brings in to the wiki.
|
||||
icon: synchronize
|
||||
syncUntracked:
|
||||
label: Add Untracked Changes
|
||||
hint: Write every page and file this target is configured to hold into the repository and commit whatever is missing. Content created before Git was enabled - or while it was turned off - is untracked until this is run.
|
||||
icon: database-daily-export
|
||||
importAll:
|
||||
label: Import Everything
|
||||
hint: Take everything currently in the local repository into the wiki, whatever the last commit did. For picking up content from a remote repository that existed before Git was enabled here.
|
||||
warn: The repository wins every collision. A page it replaces keeps its previous version in its history, but a file has none - the bytes it replaces are gone from every storage target holding them.
|
||||
icon: database-daily-import
|
||||
purge:
|
||||
label: Purge Local Repository
|
||||
hint: Empty the local working copy and clone it again from the remote. This is the way out of unrelated merge histories or a working copy that git can no longer make sense of. The remote is not touched and nothing is committed.
|
||||
warn: Any commit that exists only in the local repository and has never been pushed is lost. Run a Force Sync first if you are not sure.
|
||||
icon: trash
|
||||
@ -0,0 +1,934 @@
|
||||
import fs from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { CheckRepoActions, simpleGit } from 'simple-git'
|
||||
import {
|
||||
absPathIn,
|
||||
assetRelPath,
|
||||
importTree,
|
||||
moveStored,
|
||||
pageRelPath,
|
||||
resolveRoot,
|
||||
serializePage,
|
||||
walkStored
|
||||
} from '../../../helpers/storageFiles.ts'
|
||||
import type { ImportSummary, StoredFile } from '../../../helpers/storageFiles.ts'
|
||||
import type { SimpleGit } from 'simple-git'
|
||||
import type { StorageModule, StoragePageRef, StorageTarget } from '../../../models/storage.ts'
|
||||
|
||||
/** Where the working copy goes when the target has no path configured, per the definition default. */
|
||||
const DEFAULT_REPO_PATH = './data/repo'
|
||||
|
||||
/** What a change with no one person behind it is committed as, per the definition defaults. */
|
||||
const FALLBACK_AUTHOR = { name: 'Wiki.js', email: 'wiki@example.com' }
|
||||
|
||||
/**
|
||||
* One repository, as this module keeps it between operations.
|
||||
*
|
||||
* Cached against the configuration it was set up from, so that changing a setting in the admin area
|
||||
* takes effect on the next operation rather than at the next restart.
|
||||
*/
|
||||
interface Repo {
|
||||
git: SimpleGit
|
||||
root: string
|
||||
/** The configuration this was prepared from — see `configFingerprint`. */
|
||||
fingerprint: string
|
||||
/** Whether the remote has been contacted since this entry was made. See `ensureRemote`. */
|
||||
remoteReady: boolean
|
||||
/** Serializes work on this repository. See `withRepo`. */
|
||||
queue: Promise<unknown>
|
||||
}
|
||||
|
||||
const repos = new Map<string, Repo>()
|
||||
|
||||
/** The working copy for this target, as an absolute path. */
|
||||
function repoDir(target: StorageTarget): string {
|
||||
return resolveRoot(target.config.localRepoPath, DEFAULT_REPO_PATH)
|
||||
}
|
||||
|
||||
/**
|
||||
* What the cached repository was prepared from.
|
||||
*
|
||||
* Every setting `prepareRepo` writes into the repository or uses to reach the remote. A change to any
|
||||
* of them has to run the setup again — a new branch, a rotated key, a different URL. The default
|
||||
* author is in here because it becomes the repository's own `user.name` and `user.email`, i.e. the
|
||||
* committer of every commit; `alwaysUseDefaultAuthor` is not, because `commitAuthor` reads it per
|
||||
* commit and there is nothing prepared from it.
|
||||
*/
|
||||
function configFingerprint(target: StorageTarget): string {
|
||||
const c = target.config
|
||||
return JSON.stringify([
|
||||
c.localRepoPath,
|
||||
c.authType,
|
||||
c.repoUrl,
|
||||
c.branch,
|
||||
c.sshPrivateKeyMode,
|
||||
c.sshPrivateKeyPath,
|
||||
c.sshPrivateKeyContent,
|
||||
c.verifySSL,
|
||||
c.basicUsername,
|
||||
c.basicPassword,
|
||||
c.gitBinaryPath,
|
||||
c.defaultName,
|
||||
c.defaultEmail
|
||||
])
|
||||
}
|
||||
|
||||
/** Where an inline SSH key is written, one file per target so two of them cannot collide. */
|
||||
function sshKeyPath(target: StorageTarget): string {
|
||||
return path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, 'secure', `git-ssh-${target.id}.pem`)
|
||||
}
|
||||
|
||||
/**
|
||||
* The remote URL to talk to, with basic credentials folded in where there are any to fold.
|
||||
*
|
||||
* Built here rather than stored, and never logged: the password is a config value an administrator
|
||||
* can rotate, and a URL with it baked in would otherwise sit in the repository's own `.git/config`
|
||||
* under a name this module had stopped looking at.
|
||||
*
|
||||
* Credentials only ever go into an HTTP URL, which is the only scheme that has anywhere to put them.
|
||||
* Everything else is passed through untouched — an `ssh://` or `git@host:path` remote authenticates
|
||||
* with a key, and a bare path or `file://` is a repository on this machine and authenticates with
|
||||
* nothing at all. A URL with no scheme is the one case worth guessing about: `server.com/org/repo.git`
|
||||
* is what somebody configuring basic auth types, so it becomes HTTPS.
|
||||
*/
|
||||
function remoteUrl(target: StorageTarget): string {
|
||||
const { authType, repoUrl, basicUsername, basicPassword } = target.config
|
||||
if (authType !== 'basic') {
|
||||
return repoUrl
|
||||
}
|
||||
// -> A local path, or any scheme that is not HTTP. `git@host:path` counts: the colon is scp syntax
|
||||
// and there is no scheme at all, so the slash test is what tells it from `host/org/repo.git`.
|
||||
const isHttp = /^https?:\/\//i.test(repoUrl)
|
||||
if (!isHttp && (repoUrl.startsWith('/') || /^[a-z][a-z0-9+.-]*:/i.test(repoUrl))) {
|
||||
return repoUrl
|
||||
}
|
||||
// -> Nothing to fold in. `https://:@host` is not the same request as `https://host` and some hosts
|
||||
// refuse it outright, so an unset username means the URL is left as it is.
|
||||
if (!basicUsername) {
|
||||
return isHttp ? repoUrl : `https://${repoUrl}`
|
||||
}
|
||||
const credentials = `${encodeURIComponent(basicUsername)}:${encodeURIComponent(basicPassword ?? '')}`
|
||||
return isHttp
|
||||
? repoUrl.replace(/^(https?:\/\/)/i, `$1${credentials}@`)
|
||||
: `https://${credentials}@${repoUrl}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare the local repository, without touching the network.
|
||||
*
|
||||
* Everything an ordinary page save needs: a working copy that exists, is a repository, knows who it
|
||||
* is committing as and has `origin` pointing where the configuration says. Deliberately *not* the
|
||||
* fetch and the checkout — see `ensureRemote`. A page save must not wait on a remote, and with the
|
||||
* commits made locally and pushed by the sync there is no reason for it to.
|
||||
*/
|
||||
async function prepareRepo(target: StorageTarget): Promise<Repo> {
|
||||
const root = repoDir(target)
|
||||
await fs.mkdir(root, { recursive: true })
|
||||
/*
|
||||
`core.sshCommand` is arbitrary command execution, so simple-git refuses to set it unless the
|
||||
caller says it means to — a library cannot tell a path an administrator typed from one that
|
||||
arrived in a query string, and for most of its users the value would be the latter.
|
||||
|
||||
Here it is neither: it comes from this target's own configuration, which is only writable through
|
||||
`PUT /sites/:siteId/storage` behind `manage:system` — a permission that bypasses every check in
|
||||
the wiki, so anybody who can set this can already do anything. Granted only for the auth type that
|
||||
actually needs it, so a target authenticating over HTTPS carries no allowance at all.
|
||||
*/
|
||||
const git = simpleGit(
|
||||
root,
|
||||
target.config.authType === 'ssh' ? { unsafe: { allowUnsafeSshCommand: true } } : {}
|
||||
)
|
||||
if (target.config.gitBinaryPath) {
|
||||
git.customBinary(target.config.gitBinaryPath)
|
||||
}
|
||||
|
||||
/*
|
||||
`IS_REPO_ROOT`, emphatically not a bare `checkIsRepo()`. That defaults to
|
||||
`rev-parse --is-inside-work-tree`, which is true for any directory *inside* a repository — and the
|
||||
default working copy is `<install>/data/repo`, so on any instance whose install directory is itself
|
||||
a git checkout (every dev install, and any deployment that pulled the source down with git) the
|
||||
answer is yes and `git init` is skipped. Every command after that then runs against the wiki's own
|
||||
source repository, which is how this came to fail on a `.gitignore` rule that has nothing to do
|
||||
with storage. `--git-dir` resolving to `.git` is the question actually being asked here: is this
|
||||
directory the root of its own repository.
|
||||
|
||||
A repository nested inside another one's working tree is fine, and is what this creates: git uses
|
||||
the innermost `.git` for commands run here, and reads ignore rules from this root downwards, so
|
||||
the outer checkout's `.gitignore` stops applying the moment this exists.
|
||||
*/
|
||||
if (!(await git.checkIsRepo(CheckRepoActions.IS_REPO_ROOT))) {
|
||||
WIKI.logger.info(`(STORAGE/GIT) Initializing local repository at ${root}...`)
|
||||
await git.init(['--initial-branch', target.config.branch || 'main'])
|
||||
}
|
||||
|
||||
// -> Without this git escapes any non-ASCII path in its own output, and every path this module
|
||||
// reads back out of a diff would arrive quoted and mangled
|
||||
await git.addConfig('core.quotepath', 'false')
|
||||
await git.addConfig('user.name', target.config.defaultName || FALLBACK_AUTHOR.name)
|
||||
await git.addConfig('user.email', target.config.defaultEmail || FALLBACK_AUTHOR.email)
|
||||
await git.addConfig('http.sslVerify', target.config.verifySSL === false ? 'false' : 'true')
|
||||
|
||||
if (target.config.authType === 'ssh') {
|
||||
let keyPath = target.config.sshPrivateKeyPath
|
||||
if (target.config.sshPrivateKeyMode === 'inline') {
|
||||
keyPath = sshKeyPath(target)
|
||||
await fs.mkdir(path.dirname(keyPath), { recursive: true })
|
||||
// -> Trailing newline and 0600, both of which ssh insists on: it refuses a key file other
|
||||
// users can read, and a key without the final newline
|
||||
await fs.writeFile(keyPath, `${(target.config.sshPrivateKeyContent ?? '').trimEnd()}\n`, {
|
||||
encoding: 'utf8',
|
||||
mode: 0o600
|
||||
})
|
||||
}
|
||||
if (keyPath) {
|
||||
await git.addConfig(
|
||||
'core.sshCommand',
|
||||
`ssh -i "${keyPath}" -o StrictHostKeyChecking=no -o IdentitiesOnly=yes`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// -> Rewritten rather than added to: the URL carries the credentials, so a remote left over from a
|
||||
// previous configuration would still be reachable under its old ones
|
||||
const remotes = await git.getRemotes()
|
||||
for (const remote of remotes) {
|
||||
await git.removeRemote(remote.name)
|
||||
}
|
||||
if (target.config.repoUrl) {
|
||||
await git.addRemote('origin', remoteUrl(target))
|
||||
}
|
||||
|
||||
return {
|
||||
git,
|
||||
root,
|
||||
fingerprint: configFingerprint(target),
|
||||
remoteReady: false,
|
||||
queue: Promise.resolve()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run something against this target's repository, one operation at a time.
|
||||
*
|
||||
* Git takes a lock on the index for the length of a write, so two uploads landing together would have
|
||||
* one of them fail on `index.lock` rather than wait. Serializing here is what makes concurrent saves
|
||||
* safe, and the queue is per target because two targets are two working copies.
|
||||
*/
|
||||
async function withRepo<T>(target: StorageTarget, run: (repo: Repo) => Promise<T>): Promise<T> {
|
||||
let repo = repos.get(target.id)
|
||||
if (!repo || repo.fingerprint !== configFingerprint(target)) {
|
||||
repo = await prepareRepo(target)
|
||||
repos.set(target.id, repo)
|
||||
}
|
||||
const entry = repo
|
||||
const result = entry.queue.then(
|
||||
() => run(entry),
|
||||
() => run(entry)
|
||||
)
|
||||
// -> The queue holds the settled outcome rather than the result, so one failed operation does not
|
||||
// reject every operation queued behind it
|
||||
entry.queue = result.catch(() => {})
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Bring the working copy onto the configured branch, having contacted the remote.
|
||||
*
|
||||
* The part of the 2.x module's `init` that costs a network round trip, split off so that only a sync
|
||||
* pays for it. A repository with no remote configured is left as the purely local one it is.
|
||||
*
|
||||
* @returns Whether the remote already has the branch, which decides whether there is anything to pull
|
||||
*/
|
||||
async function ensureRemote(repo: Repo, target: StorageTarget): Promise<{ onRemote: boolean }> {
|
||||
const branch = target.config.branch || 'main'
|
||||
if (!target.config.repoUrl) {
|
||||
return { onRemote: false }
|
||||
}
|
||||
if (!repo.remoteReady) {
|
||||
await repo.git.raw(['remote', 'update', 'origin', '--prune'])
|
||||
}
|
||||
|
||||
const branches = await repo.git.branch(['-a'])
|
||||
const onRemote = branches.all.includes(`remotes/origin/${branch}`)
|
||||
const onLocal = branches.all.includes(branch)
|
||||
|
||||
/*
|
||||
A remote that does not have the branch yet is the ordinary state of a repository somebody has just
|
||||
created, and the first push is what creates it — 2.x refused to start at all in that case, which
|
||||
made an empty remote something an administrator had to go and fix by hand before the wiki would
|
||||
talk to it.
|
||||
|
||||
What is still worth refusing is a branch that exists nowhere on a remote that has other branches,
|
||||
because that is a typo rather than a beginning.
|
||||
*/
|
||||
if (!onRemote && !onLocal) {
|
||||
const remoteBranches = branches.all.filter((b) => b.startsWith('remotes/origin/'))
|
||||
if (remoteBranches.length > 0) {
|
||||
throw new Error(
|
||||
`The branch "${branch}" does not exist locally or on the remote, which has ${remoteBranches
|
||||
.map((b) => b.replace('remotes/origin/', ''))
|
||||
.join(', ')}. Check the branch name, or create it on the remote first.`
|
||||
)
|
||||
}
|
||||
} else if (onRemote && branches.current !== branch) {
|
||||
WIKI.logger.info(`(STORAGE/GIT) Checking out branch ${branch}...`)
|
||||
await repo.git.checkout(branch)
|
||||
}
|
||||
|
||||
repo.remoteReady = true
|
||||
return { onRemote }
|
||||
}
|
||||
|
||||
/** Whether the repository's own ignore rules exclude this path. */
|
||||
async function isIgnored(repo: Repo, relPath: string): Promise<boolean> {
|
||||
try {
|
||||
return (await repo.git.checkIgnore([relPath])).length > 0
|
||||
} catch {
|
||||
// -> `check-ignore` exits non-zero when nothing matches, which simple-git raises
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Who to attribute a commit to: whoever made the change, or the target's configured stand-in.
|
||||
*
|
||||
* `alwaysUseDefaultAuthor` is the stand-in for everything, for a repository whose history should not
|
||||
* carry the wiki's accounts — a public mirror, or an instance whose users did not agree to have their
|
||||
* name and address published with every edit they make. The actor is not looked up at all in that
|
||||
* case rather than looked up and discarded, so there is nothing to leak by mistake.
|
||||
*
|
||||
* The committer is the default author either way: it comes from the repository's own `user.name` and
|
||||
* `user.email`, which `prepareRepo` sets from these same two settings. This decides the *author*,
|
||||
* which is the half of a commit that git shows and that would otherwise name the person.
|
||||
*/
|
||||
async function commitAuthor(target: StorageTarget, actorId?: string): Promise<string> {
|
||||
const name = target.config.defaultName || FALLBACK_AUTHOR.name
|
||||
const email = target.config.defaultEmail || FALLBACK_AUTHOR.email
|
||||
if (target.config.alwaysUseDefaultAuthor) {
|
||||
return `${name} <${email}>`
|
||||
}
|
||||
const actor = await WIKI.models.storage.actorFor(actorId)
|
||||
return `${actor?.name || name} <${actor?.email || email}>`
|
||||
}
|
||||
|
||||
/**
|
||||
* Commit whatever is staged at these paths, and nothing if that is nothing.
|
||||
*
|
||||
* The empty check is not an optimization. Every page save reaches this, and most of them save a page
|
||||
* whose stored form has not actually changed — a re-publish, a tag reordered into the same order — so
|
||||
* without it the repository would fill with empty commits that say a file changed when it did not.
|
||||
*/
|
||||
async function commitPaths(
|
||||
repo: Repo,
|
||||
target: StorageTarget,
|
||||
paths: string[],
|
||||
message: string,
|
||||
actorId?: string
|
||||
): Promise<boolean> {
|
||||
const staged = await repo.git.raw(['diff', '--cached', '--name-only', '--', ...paths])
|
||||
if (!staged.trim()) {
|
||||
return false
|
||||
}
|
||||
await repo.git.commit(message, paths, { '--author': await commitAuthor(target, actorId) })
|
||||
return true
|
||||
}
|
||||
|
||||
/** Stage a written file and commit it, unless the repository is told to ignore it. */
|
||||
async function stageAndCommit(
|
||||
repo: Repo,
|
||||
target: StorageTarget,
|
||||
relPath: string,
|
||||
message: string,
|
||||
actorId?: string
|
||||
): Promise<void> {
|
||||
if (await isIgnored(repo, relPath)) {
|
||||
return
|
||||
}
|
||||
await repo.git.add(relPath)
|
||||
await commitPaths(repo, target, [relPath], message, actorId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage a deletion and commit it.
|
||||
*
|
||||
* `git rm` fails on a path git has never heard of, which is an ordinary situation here — a file
|
||||
* excluded by `.gitignore`, or one deleted before this target was enabled — so the removal is staged
|
||||
* from the index instead and a path that was not in it simply leaves nothing to commit.
|
||||
*/
|
||||
async function removeAndCommit(
|
||||
repo: Repo,
|
||||
target: StorageTarget,
|
||||
relPath: string,
|
||||
message: string,
|
||||
actorId?: string
|
||||
): Promise<void> {
|
||||
await fs.rm(absPathIn(repo.root, relPath), { force: true })
|
||||
try {
|
||||
await repo.git.raw(['rm', '--cached', '--ignore-unmatch', '--', relPath])
|
||||
} catch (err: any) {
|
||||
WIKI.logger.warn(`(STORAGE/GIT) Could not unstage ${relPath}: ${err.message}`)
|
||||
return
|
||||
}
|
||||
await commitPaths(repo, target, [relPath], message, actorId)
|
||||
}
|
||||
|
||||
/** A page's path as a commit message names it: its locale and where it sits. */
|
||||
function pageLabel(ref: StoragePageRef): string {
|
||||
return `[${ref.locale}] ${ref.path || '/'}`
|
||||
}
|
||||
|
||||
/** An asset's path as a commit message names it. */
|
||||
function assetLabel(ref: { locale: string; folderPath: string; fileName: string }): string {
|
||||
return `[${ref.locale}] ${ref.folderPath ? `${ref.folderPath}/` : ''}${ref.fileName}`
|
||||
}
|
||||
|
||||
/** One entry of a `--name-status` diff. */
|
||||
interface DiffEntry {
|
||||
status: string
|
||||
segments: string[]
|
||||
previousSegments?: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* What changed between two commits, as paths this module can act on.
|
||||
*
|
||||
* Read with `--name-status -M` rather than through a diff summary because the three things that
|
||||
* matter here are exactly what that reports: whether a path arrived, went, or moved. A summary of
|
||||
* insertions and deletions cannot tell a deleted file from one emptied to nothing, and 2.x guessed at
|
||||
* that from the line counts.
|
||||
*/
|
||||
async function changedPaths(repo: Repo, from: string, to: string): Promise<DiffEntry[]> {
|
||||
const raw = await repo.git.raw(['diff', '--name-status', '-M', '-z', from, to])
|
||||
// -> `-z` because a path may contain anything at all, newlines included, and the fields are then
|
||||
// NUL-separated: `status NUL path` for most, `Rxxx NUL old NUL new` for a rename
|
||||
const fields = raw.split('\0').filter((f) => f !== '')
|
||||
const entries: DiffEntry[] = []
|
||||
for (let i = 0; i < fields.length;) {
|
||||
const status = fields[i++]
|
||||
if (status.startsWith('R') || status.startsWith('C')) {
|
||||
const previous = fields[i++]
|
||||
const current = fields[i++]
|
||||
if (!current) {
|
||||
break
|
||||
}
|
||||
entries.push({
|
||||
status: 'R',
|
||||
segments: current.split('/'),
|
||||
previousSegments: previous.split('/')
|
||||
})
|
||||
continue
|
||||
}
|
||||
const file = fields[i++]
|
||||
if (!file) {
|
||||
break
|
||||
}
|
||||
entries.push({ status: status[0], segments: file.split('/') })
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
/**
|
||||
* Take a path the repository no longer has out of the wiki.
|
||||
*
|
||||
* The half of a pull that `importTree` cannot do, and the one that makes the remote authoritative
|
||||
* rather than merely a source: a commit somebody pushed that deletes a file deletes the page or the
|
||||
* asset here too.
|
||||
*
|
||||
* Which of the two it was has to be worked out from the tree rather than from the file, since the
|
||||
* file is exactly what is no longer there. A page is filed under its editor's extension and addressed
|
||||
* without one, so the stem is looked up first and only counts as the page if the page's own stored
|
||||
* file name is the one that went — `readme.pdf` disappearing is not the markdown page `readme`.
|
||||
*
|
||||
* @returns What was deleted, for the report
|
||||
*/
|
||||
async function removeFromWiki(
|
||||
target: StorageTarget,
|
||||
segments: string[],
|
||||
actorId: string
|
||||
): Promise<'page' | 'asset' | null> {
|
||||
const stored = WIKI.models.storage.parseStoredPath(target.siteId, segments)
|
||||
if (!stored) {
|
||||
return null
|
||||
}
|
||||
const rest = [...stored.segments]
|
||||
const fileName = rest.pop()!
|
||||
const folderPath = rest.join('/')
|
||||
const ext = path.extname(fileName).replace(/^\./, '').toLowerCase()
|
||||
const stem = fileName.slice(0, fileName.length - (ext ? ext.length + 1 : 0))
|
||||
|
||||
if (stem) {
|
||||
const asPage = await WIKI.models.tree.getEntryAt({
|
||||
siteId: target.siteId,
|
||||
locale: stored.locale,
|
||||
parentPath: folderPath || null,
|
||||
fileName: stem
|
||||
})
|
||||
if (
|
||||
asPage?.type === 'page' &&
|
||||
(await WIKI.models.pages.storageFileNameOf(asPage.id)) === fileName
|
||||
) {
|
||||
await WIKI.models.pages.deletePage(target.siteId, asPage.id, {
|
||||
id: actorId,
|
||||
permissions: ['manage:system']
|
||||
})
|
||||
return 'page'
|
||||
}
|
||||
}
|
||||
|
||||
const asAsset = await WIKI.models.tree.getEntryAt({
|
||||
siteId: target.siteId,
|
||||
locale: stored.locale,
|
||||
parentPath: folderPath || null,
|
||||
fileName
|
||||
})
|
||||
if (asAsset?.type === 'asset') {
|
||||
await WIKI.models.assets.deleteAsset(target.siteId, asAsset.id, actorId)
|
||||
return 'asset'
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/** Every file this target should be holding, written into the working copy. */
|
||||
async function writeEverything(
|
||||
repo: Repo,
|
||||
target: StorageTarget
|
||||
): Promise<{ pages: number; assets: number; unstored: number; unreadable: number }> {
|
||||
const counts = { pages: 0, assets: 0, unstored: 0, unreadable: 0 }
|
||||
|
||||
for (const asset of await WIKI.models.assets.listStoredAssets(target.siteId)) {
|
||||
const contentType = WIKI.models.storage.contentTypeFor(
|
||||
target.siteId,
|
||||
asset.kind,
|
||||
asset.fileSize
|
||||
)
|
||||
if (!target.contentTypes.activeTypes.includes(contentType)) {
|
||||
continue
|
||||
}
|
||||
const relPath = assetRelPath(target, asset)
|
||||
if (!relPath) {
|
||||
counts.unstored++
|
||||
continue
|
||||
}
|
||||
const data = await WIKI.models.storage.getAsset(asset)
|
||||
if (!data) {
|
||||
counts.unreadable++
|
||||
continue
|
||||
}
|
||||
const filePath = absPathIn(repo.root, relPath)
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true })
|
||||
await fs.writeFile(filePath, data)
|
||||
counts.assets++
|
||||
}
|
||||
|
||||
if (target.contentTypes.activeTypes.includes('pages')) {
|
||||
for (const { ref, content } of await WIKI.models.pages.listForStorage(target.siteId)) {
|
||||
const relPath = pageRelPath(target, ref)
|
||||
if (!relPath) {
|
||||
counts.unstored++
|
||||
continue
|
||||
}
|
||||
const filePath = absPathIn(repo.root, relPath)
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true })
|
||||
await fs.writeFile(filePath, serializePage(ref, content))
|
||||
counts.pages++
|
||||
}
|
||||
}
|
||||
|
||||
return counts
|
||||
}
|
||||
|
||||
/** What an import run did, in words. */
|
||||
function describeImport(summary: ImportSummary | null): string {
|
||||
if (!summary) {
|
||||
return 'There is nothing in the local repository yet. Run a Force Sync to fetch from the remote first.'
|
||||
}
|
||||
const parts = []
|
||||
if (summary.pages > 0 || summary.assets > 0) {
|
||||
parts.push(`Imported ${summary.pages} page(s) and ${summary.assets} asset(s).`)
|
||||
} else {
|
||||
parts.push('There was nothing to import.')
|
||||
}
|
||||
if (summary.skipped > 0) {
|
||||
parts.push(`${summary.skipped} could not replace what is at their path and were left alone.`)
|
||||
}
|
||||
if (summary.failed > 0) {
|
||||
parts.push(`${summary.failed} could not be imported - see the server log.`)
|
||||
}
|
||||
return parts.join(' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* Git storage module
|
||||
*
|
||||
* Keeps the site's content as a git repository: the same tree the local disk target writes, committed
|
||||
* as it changes and synchronized with a remote. What that buys over the disk target is history — every
|
||||
* edit is a commit by the person who made it, so the repository is a record of the wiki and not only a
|
||||
* copy of it — and a second place the content lives that is not this machine.
|
||||
*
|
||||
* The layout, the front matter and what makes a file a page are all `helpers/storageFiles.ts`, shared
|
||||
* with the disk target. This module is what git adds on top: a commit per change, and a sync.
|
||||
*
|
||||
* **Local writes, batched network.** A page save commits and returns; nothing waits on a remote. The
|
||||
* push and the pull happen in `sync`, which the scheduler runs every few minutes and an administrator
|
||||
* can run on demand. That is why `prepareRepo` and `ensureRemote` are separate: an unreachable remote
|
||||
* must not be able to make the wiki slow to edit, or fail an upload.
|
||||
*
|
||||
* **A pull is authoritative.** What it brings in is applied to the wiki, replacing what is there — and
|
||||
* a commit that deleted a file deletes the page or the asset here too, which is the whole point of
|
||||
* pointing a wiki at a repository other people push to. It also means push access to the remote is
|
||||
* effectively write access to the wiki, which is worth knowing before configuring one.
|
||||
*
|
||||
* Everything runs through `withRepo`, one operation at a time per target: git locks its index for the
|
||||
* length of a write, so two concurrent uploads would otherwise have one of them fail outright.
|
||||
*/
|
||||
const gitStorage: StorageModule = {
|
||||
canStore(target, ref) {
|
||||
return WIKI.models.storage.pathPrefixFor(target.siteId, ref.locale) !== null
|
||||
},
|
||||
|
||||
async putAsset(target, ref, data) {
|
||||
const relPath = assetRelPath(target, ref)
|
||||
if (!relPath) {
|
||||
throw new Error(
|
||||
`${target.title} has no path for ${ref.locale} content, so ${ref.fileName} cannot be stored there.`
|
||||
)
|
||||
}
|
||||
await withRepo(target, async (repo) => {
|
||||
const filePath = absPathIn(repo.root, relPath)
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true })
|
||||
await fs.writeFile(filePath, data)
|
||||
await stageAndCommit(repo, target, relPath, `docs: upload ${assetLabel(ref)}`, ref.actorId)
|
||||
})
|
||||
},
|
||||
|
||||
async getAsset(target, ref) {
|
||||
const relPath = assetRelPath(target, ref)
|
||||
if (!relPath) {
|
||||
return null
|
||||
}
|
||||
// -> Read straight off the working copy rather than out of git: what the wiki serves is the
|
||||
// current state of the branch, which is exactly what is checked out
|
||||
try {
|
||||
return await fs.readFile(absPathIn(repoDir(target), relPath))
|
||||
} catch (err: any) {
|
||||
if (err.code !== 'ENOENT') {
|
||||
throw err
|
||||
}
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
async deleteAsset(target, ref) {
|
||||
const relPath = assetRelPath(target, ref)
|
||||
if (!relPath) {
|
||||
return
|
||||
}
|
||||
await withRepo(target, (repo) =>
|
||||
removeAndCommit(repo, target, relPath, `docs: delete ${assetLabel(ref)}`, ref.actorId)
|
||||
)
|
||||
},
|
||||
|
||||
async moveAsset(target, ref, previous) {
|
||||
const from = assetRelPath(target, { ...ref, ...previous })
|
||||
const to = assetRelPath(target, ref)
|
||||
await withRepo(target, async (repo) => {
|
||||
const outcome = await moveStored(repo.root, from, to)
|
||||
if (outcome === 'nothing') {
|
||||
return
|
||||
}
|
||||
const paths = outcome === 'moved' ? [from!, to!] : [from!]
|
||||
for (const relPath of paths) {
|
||||
await repo.git.add(['-A', '--', relPath])
|
||||
}
|
||||
await commitPaths(
|
||||
repo,
|
||||
target,
|
||||
paths,
|
||||
outcome === 'moved'
|
||||
? `docs: rename ${assetLabel({ ...ref, ...previous })} to ${assetLabel(ref)}`
|
||||
: `docs: delete ${assetLabel({ ...ref, ...previous })}`,
|
||||
ref.actorId
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
async putPage(target, ref, page) {
|
||||
const relPath = pageRelPath(target, ref)
|
||||
if (!relPath) {
|
||||
return
|
||||
}
|
||||
await withRepo(target, async (repo) => {
|
||||
const filePath = absPathIn(repo.root, relPath)
|
||||
// -> Which of the two verbs the commit gets. Read before the write, since afterwards every page
|
||||
// looks like one that was already there.
|
||||
const existed = await fs
|
||||
.access(filePath)
|
||||
.then(() => true)
|
||||
.catch(() => false)
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true })
|
||||
await fs.writeFile(filePath, serializePage(ref, page), 'utf8')
|
||||
await stageAndCommit(
|
||||
repo,
|
||||
target,
|
||||
relPath,
|
||||
`docs: ${existed ? 'update' : 'create'} ${pageLabel(ref)}`,
|
||||
ref.actorId
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
async deletePage(target, ref) {
|
||||
const relPath = pageRelPath(target, ref)
|
||||
if (!relPath) {
|
||||
return
|
||||
}
|
||||
await withRepo(target, (repo) =>
|
||||
removeAndCommit(repo, target, relPath, `docs: delete ${pageLabel(ref)}`, ref.actorId)
|
||||
)
|
||||
},
|
||||
|
||||
async movePage(target, ref, previousPath) {
|
||||
const from = pageRelPath(target, { ...ref, path: previousPath })
|
||||
const to = pageRelPath(target, ref)
|
||||
await withRepo(target, async (repo) => {
|
||||
const outcome = await moveStored(repo.root, from, to)
|
||||
if (outcome === 'nothing') {
|
||||
return
|
||||
}
|
||||
const paths = outcome === 'moved' ? [from!, to!] : [from!]
|
||||
for (const relPath of paths) {
|
||||
await repo.git.add(['-A', '--', relPath])
|
||||
}
|
||||
await commitPaths(
|
||||
repo,
|
||||
target,
|
||||
paths,
|
||||
outcome === 'moved'
|
||||
? `docs: rename ${pageLabel({ ...ref, path: previousPath })} to ${pageLabel(ref)}`
|
||||
: `docs: delete ${pageLabel({ ...ref, path: previousPath })}`,
|
||||
ref.actorId
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
/**
|
||||
* Pull from the remote, push to it, and apply what came back.
|
||||
*
|
||||
* The direction is the target's `syncMode`, and it decides which half runs: `push` never takes
|
||||
* anything in and force-pushes, so the wiki wins; `pull` never sends anything, so the remote does;
|
||||
* `sync` does both, rebasing the wiki's commits on top of what it pulled.
|
||||
*
|
||||
* Whatever a pull brought in is then applied to the wiki — created, replaced, or deleted. That is
|
||||
* done from a `--name-status` diff between the commit the branch was on before and the one it is on
|
||||
* now, rather than by walking the tree: a sync runs every few minutes, and reading every file in the
|
||||
* repository each time to find the two that changed would be absurd.
|
||||
*/
|
||||
async sync(target: StorageTarget, actorId: string): Promise<string> {
|
||||
const mode = target.config.syncMode || 'sync'
|
||||
const branch = target.config.branch || 'main'
|
||||
return withRepo(target, async (repo) => {
|
||||
if (!target.config.repoUrl) {
|
||||
return 'No repository URI is configured, so there is nothing to sync with. Commits are being made locally.'
|
||||
}
|
||||
const { onRemote } = await ensureRemote(repo, target)
|
||||
|
||||
const before = await repo.git.revparse(['HEAD']).catch(() => null)
|
||||
const parts: string[] = []
|
||||
|
||||
// -> Nothing to pull from a branch the remote does not have yet; the push below creates it
|
||||
if (mode !== 'push' && onRemote) {
|
||||
WIKI.logger.info(`(STORAGE/GIT) Pulling from origin/${branch}...`)
|
||||
await repo.git.pull('origin', branch, ['--rebase'])
|
||||
}
|
||||
if (mode !== 'pull') {
|
||||
WIKI.logger.info(`(STORAGE/GIT) Pushing to origin/${branch}...`)
|
||||
// -> `--force` only in push mode, which is the mode that says the wiki is the authority
|
||||
await repo.git.push(
|
||||
'origin',
|
||||
branch,
|
||||
mode === 'push' ? ['--signed=if-asked', '--force'] : ['--signed=if-asked']
|
||||
)
|
||||
}
|
||||
|
||||
if (mode !== 'push' && onRemote) {
|
||||
const after = await repo.git.revparse(['HEAD']).catch(() => null)
|
||||
if (!after) {
|
||||
return 'Synced. The repository has no commits yet.'
|
||||
}
|
||||
parts.push(await applyIncoming(repo, target, before, after, actorId))
|
||||
} else if (mode === 'pull') {
|
||||
parts.push(
|
||||
'Synced. The remote does not have this branch yet, so there was nothing to pull.'
|
||||
)
|
||||
} else {
|
||||
parts.push('Pushed to the remote.')
|
||||
}
|
||||
return parts.join(' ')
|
||||
})
|
||||
},
|
||||
|
||||
/**
|
||||
* Write and commit everything this target should be holding but has never been given.
|
||||
*
|
||||
* The way in for content that predates the target: a wiki that ran for a year before git was
|
||||
* enabled has a repository with nothing in it, and nothing in the ordinary course of things ever
|
||||
* goes back for those pages. One commit, by the administrator who asked for it, since it is their
|
||||
* action and not a hundred authors' edits.
|
||||
*/
|
||||
async syncUntracked(target: StorageTarget, actorId: string): Promise<string> {
|
||||
return withRepo(target, async (repo) => {
|
||||
const counts = await writeEverything(repo, target)
|
||||
await repo.git.add(['-A', '--', '.'])
|
||||
const committed = await commitPaths(
|
||||
repo,
|
||||
target,
|
||||
['.'],
|
||||
'docs: add all untracked content',
|
||||
actorId
|
||||
)
|
||||
WIKI.logger.info(
|
||||
`(STORAGE/GIT) Wrote ${counts.pages} page(s) and ${counts.assets} asset(s) to ${repo.root} [ OK ]`
|
||||
)
|
||||
const parts = [
|
||||
committed
|
||||
? `Committed the untracked part of ${counts.pages} page(s) and ${counts.assets} asset(s).`
|
||||
: `Wrote ${counts.pages} page(s) and ${counts.assets} asset(s); all of it was already tracked.`
|
||||
]
|
||||
if (counts.unreadable > 0) {
|
||||
parts.push(`${counts.unreadable} asset(s) could not be read and were skipped.`)
|
||||
}
|
||||
if (counts.unstored > 0) {
|
||||
const { primaryLocale } = WIKI.models.storage.pathLayoutFor(target.siteId)
|
||||
parts.push(
|
||||
`${counts.unstored} item(s) are not in the ${primaryLocale} locale, which is the only one this site stores.`
|
||||
)
|
||||
}
|
||||
return parts.join(' ')
|
||||
})
|
||||
},
|
||||
|
||||
/**
|
||||
* Take everything in the working copy into the wiki, whatever the last commit did.
|
||||
*
|
||||
* For a repository that already had content before this target existed: the sync only ever looks at
|
||||
* what changed between two commits, so a repository cloned with a thousand files in it has none of
|
||||
* them in the wiki. The repository wins every collision, consistent with what a pull does — this is
|
||||
* the same direction, applied to everything at once instead of to a diff.
|
||||
*/
|
||||
async importAll(target: StorageTarget, actorId: string): Promise<string> {
|
||||
return withRepo(target, async (repo) =>
|
||||
describeImport(await importTree({ target, root: repo.root, actorId, overwrite: true }))
|
||||
)
|
||||
},
|
||||
|
||||
/**
|
||||
* Throw the working copy away and take it again from the remote.
|
||||
*
|
||||
* The answer to a working copy git can no longer make sense of — unrelated histories, a rebase that
|
||||
* cannot be finished, an index that will not unlock. Nothing about the remote changes and nothing is
|
||||
* committed, so the cost is any commit that only existed here.
|
||||
*/
|
||||
async purge(target: StorageTarget): Promise<string> {
|
||||
const root = repoDir(target)
|
||||
return withRepo(target, async () => {
|
||||
WIKI.logger.info(`(STORAGE/GIT) Purging the local repository at ${root}...`)
|
||||
await fs.rm(root, { recursive: true, force: true })
|
||||
// -> Dropped rather than reused: it holds a SimpleGit bound to a directory that no longer
|
||||
// exists, and the next operation is what sets the replacement up
|
||||
repos.delete(target.id)
|
||||
if (!target.config.repoUrl) {
|
||||
return 'The local repository has been emptied. It will be initialized again on the next change.'
|
||||
}
|
||||
const repo = await prepareRepo(target)
|
||||
repos.set(target.id, repo)
|
||||
await ensureRemote(repo, target)
|
||||
return 'The local repository has been emptied and taken again from the remote. Run Import Everything if the wiki should now say what it holds.'
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply what a pull brought in to the wiki.
|
||||
*
|
||||
* Split out of `sync` because it is the interesting half and reads as its own thing: a diff, and then
|
||||
* two lists — what to take in and what to take out. A file that arrived or changed is imported with
|
||||
* the repository winning; one that went is deleted. A rename is both, in that order, which is what
|
||||
* moves a page rather than losing its history to a delete and a create.
|
||||
*/
|
||||
async function applyIncoming(
|
||||
repo: Repo,
|
||||
target: StorageTarget,
|
||||
before: string | null,
|
||||
after: string,
|
||||
actorId: string
|
||||
): Promise<string> {
|
||||
// -> Nothing came back, which is the ordinary outcome of a sync and worth saying plainly
|
||||
if (before === after) {
|
||||
return 'Synced. Nothing had changed on the remote.'
|
||||
}
|
||||
|
||||
let files: StoredFile[] | null
|
||||
const removals: string[][] = []
|
||||
if (!before) {
|
||||
// -> Nothing to diff against: the branch had no commits here at all, so everything in it is new
|
||||
files = await walkStored(repo.root)
|
||||
} else {
|
||||
const changes = await changedPaths(repo, before, after)
|
||||
files = []
|
||||
for (const change of changes) {
|
||||
if (change.status === 'D') {
|
||||
removals.push(change.segments)
|
||||
continue
|
||||
}
|
||||
if (change.status === 'R' && change.previousSegments) {
|
||||
removals.push(change.previousSegments)
|
||||
}
|
||||
files.push({
|
||||
filePath: absPathIn(repo.root, change.segments.join('/')),
|
||||
segments: change.segments
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const summary = await importTree({
|
||||
target,
|
||||
root: repo.root,
|
||||
actorId,
|
||||
overwrite: true,
|
||||
files
|
||||
})
|
||||
|
||||
let deletedPages = 0
|
||||
let deletedAssets = 0
|
||||
for (const segments of removals) {
|
||||
try {
|
||||
const removed = await removeFromWiki(target, segments, actorId)
|
||||
if (removed === 'page') {
|
||||
deletedPages++
|
||||
} else if (removed === 'asset') {
|
||||
deletedAssets++
|
||||
}
|
||||
} catch (err: any) {
|
||||
// -> One entry the wiki could not let go of must not stop the rest of the commit being applied
|
||||
WIKI.logger.warn(`(STORAGE/GIT) Could not delete ${segments.join('/')} [ SKIPPED ]`)
|
||||
WIKI.logger.warn(err.message)
|
||||
}
|
||||
}
|
||||
|
||||
const parts = ['Synced.']
|
||||
if (summary && (summary.pages > 0 || summary.assets > 0)) {
|
||||
parts.push(`Took in ${summary.pages} page(s) and ${summary.assets} asset(s).`)
|
||||
}
|
||||
if (deletedPages > 0 || deletedAssets > 0) {
|
||||
parts.push(`Deleted ${deletedPages} page(s) and ${deletedAssets} asset(s) the remote removed.`)
|
||||
}
|
||||
if (summary && summary.failed > 0) {
|
||||
parts.push(`${summary.failed} could not be imported - see the server log.`)
|
||||
}
|
||||
if (parts.length === 1) {
|
||||
parts.push('Nothing the remote changed affected this wiki.')
|
||||
}
|
||||
return parts.join(' ')
|
||||
}
|
||||
|
||||
export default gitStorage
|
||||
@ -0,0 +1,74 @@
|
||||
key: s3
|
||||
title: S3 Object Storage
|
||||
icon: '/_assets/icons/ultraviolet-amazon-web-services.svg'
|
||||
banner: '/_assets/storage/s3.jpg'
|
||||
description: Amazon S3 and any store that speaks its API - Cloudflare R2, DigitalOcean Spaces, Backblaze B2, Wasabi, MinIO and the rest. Leave the endpoint empty for AWS itself, or point it at whichever service you use.
|
||||
assetDelivery:
|
||||
isDirectAccessSupported: true
|
||||
contentTypes:
|
||||
defaultTypesEnabled: ['images', 'documents', 'others', 'large']
|
||||
props:
|
||||
endpoint:
|
||||
type: String
|
||||
title: Endpoint
|
||||
default: ''
|
||||
hint: Leave empty for AWS S3. Otherwise the full URL of the service, e.g. https://<account>.r2.cloudflarestorage.com for Cloudflare R2, https://nyc3.digitaloceanspaces.com for DigitalOcean Spaces.
|
||||
icon: dns
|
||||
order: 1
|
||||
region:
|
||||
type: String
|
||||
title: Region
|
||||
default: us-east-1
|
||||
hint: The region the bucket lives in. Stores that do not have regions usually want "auto" (Cloudflare R2) or accept anything (MinIO).
|
||||
icon: geography
|
||||
order: 2
|
||||
bucket:
|
||||
type: String
|
||||
title: Bucket Name
|
||||
default: ''
|
||||
hint: The bucket to store content in. It must already exist - this target will not create it.
|
||||
icon: open-box
|
||||
order: 3
|
||||
accessKeyId:
|
||||
type: String
|
||||
title: Access Key ID
|
||||
default: ''
|
||||
hint: Leave both this and the secret empty to use the credentials the machine already has - an IAM role, or the standard AWS environment variables.
|
||||
icon: 3d-touch
|
||||
order: 4
|
||||
secretAccessKey:
|
||||
type: String
|
||||
title: Secret Access Key
|
||||
default: ''
|
||||
hint: The secret for the access key above.
|
||||
icon: key
|
||||
sensitive: true
|
||||
order: 5
|
||||
storageClass:
|
||||
type: String
|
||||
title: Storage Class
|
||||
default: STANDARD
|
||||
hint: What new objects are stored as. An AWS concept - most compatible stores ignore it, and leaving it at Standard is always safe.
|
||||
icon: scan-stock
|
||||
order: 6
|
||||
enum:
|
||||
- STANDARD|Standard
|
||||
- STANDARD_IA|Standard Infrequent Access
|
||||
- INTELLIGENT_TIERING|Intelligent Tiering
|
||||
- ONEZONE_IA|One Zone Infrequent Access
|
||||
- REDUCED_REDUNDANCY|Reduced Redundancy
|
||||
- GLACIER_IR|Glacier Instant Retrieval
|
||||
- GLACIER|Glacier Flexible Retrieval
|
||||
- DEEP_ARCHIVE|Glacier Deep Archive
|
||||
forcePathStyle:
|
||||
type: Boolean
|
||||
title: Force Path Style
|
||||
default: false
|
||||
hint: Address the bucket as a path (endpoint/bucket/key) rather than as a subdomain. Needed by MinIO and some self-hosted stores; leave off for AWS, R2 and Spaces.
|
||||
icon: filtration
|
||||
order: 10
|
||||
actions:
|
||||
exportAll:
|
||||
label: Export Everything
|
||||
hint: Write a copy of every page and asset this target is configured to hold to the bucket, overwriting whatever is already there. Nothing in the database is changed and nothing is moved, so this is how content created before the target was enabled gets onto it.
|
||||
icon: this-way-up
|
||||
@ -0,0 +1,208 @@
|
||||
import {
|
||||
CopyObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
PutObjectCommand,
|
||||
S3Client
|
||||
} from '@aws-sdk/client-s3'
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
|
||||
import { objectStorageModule, signingBaseUrl } from '../../../helpers/storageObjects.ts'
|
||||
import type { ObjectStoreClient } from '../../../helpers/storageObjects.ts'
|
||||
import type { StorageTarget } from '../../../models/storage.ts'
|
||||
|
||||
/** Live clients, keyed by target. See `clientFor`. */
|
||||
const clients = new Map<string, { client: S3Client; fingerprint: string }>()
|
||||
|
||||
/** The settings a client is built from — a change to any of them needs a new one. */
|
||||
function configFingerprint(target: StorageTarget): string {
|
||||
const c = target.config
|
||||
return JSON.stringify([c.endpoint, c.region, c.accessKeyId, c.secretAccessKey, c.forcePathStyle])
|
||||
}
|
||||
|
||||
/**
|
||||
* The S3 client for this target, built once and kept.
|
||||
*
|
||||
* Rebuilt when the configuration changes, so a rotated key takes effect on the next operation rather
|
||||
* than at the next restart.
|
||||
*
|
||||
* **Credentials are optional.** Left empty, the SDK falls back to its own chain — an IAM role on the
|
||||
* instance, the standard `AWS_*` environment variables, a shared credentials file — which is how a
|
||||
* deployment avoids putting a long-lived secret in the database at all.
|
||||
*/
|
||||
function clientFor(target: StorageTarget): S3Client {
|
||||
const fingerprint = configFingerprint(target)
|
||||
const cached = clients.get(target.id)
|
||||
if (cached && cached.fingerprint === fingerprint) {
|
||||
return cached.client
|
||||
}
|
||||
const { endpoint, region, accessKeyId, secretAccessKey, forcePathStyle } = target.config
|
||||
const client = new S3Client({
|
||||
region: region || 'us-east-1',
|
||||
...(endpoint ? { endpoint } : {}),
|
||||
...(forcePathStyle ? { forcePathStyle: true } : {}),
|
||||
...(accessKeyId && secretAccessKey ? { credentials: { accessKeyId, secretAccessKey } } : {})
|
||||
})
|
||||
clients.set(target.id, { client, fingerprint })
|
||||
return client
|
||||
}
|
||||
|
||||
/**
|
||||
* The client and bucket a signature should be made against.
|
||||
*
|
||||
* SigV4 covers the `Host` header, so a URL signed for the bucket's own address and then rewritten
|
||||
* onto a CDN domain carries a signature for the wrong host and is rejected. The domain has to be
|
||||
* signed *for*, which means a client pointed at it rather than the ordinary client with its output
|
||||
* edited afterwards.
|
||||
*
|
||||
* Which of the two forms that takes depends on what sits at the domain, and `forcePathStyle` is the
|
||||
* target's existing answer to exactly that question for the store itself:
|
||||
*
|
||||
* - **off** — the domain *is* the bucket, which is what a Cloudflare R2 custom domain or a Spaces CDN
|
||||
* endpoint is. The SDK spells this `bucketEndpoint`, and it means the `Bucket` *parameter* carries
|
||||
* the URL: passing the bucket's name alongside it fails outright.
|
||||
* - **on** — the bucket is the first path segment, which is what a reverse proxy onto MinIO looks
|
||||
* like, and the endpoint and the bucket name are then both ordinary.
|
||||
*
|
||||
* Not cached: signing is per request, and an administrator changing the base URL must not have to
|
||||
* wait for anything to expire before seeing it.
|
||||
*/
|
||||
function signingTargetFor(
|
||||
target: StorageTarget,
|
||||
baseUrl: string | null
|
||||
): { client: S3Client; bucket: string } {
|
||||
if (!baseUrl) {
|
||||
return { client: clientFor(target), bucket: target.config.bucket }
|
||||
}
|
||||
const { region, accessKeyId, secretAccessKey, forcePathStyle } = target.config
|
||||
const credentials =
|
||||
accessKeyId && secretAccessKey ? { credentials: { accessKeyId, secretAccessKey } } : {}
|
||||
if (forcePathStyle) {
|
||||
return {
|
||||
client: new S3Client({
|
||||
region: region || 'us-east-1',
|
||||
endpoint: baseUrl,
|
||||
forcePathStyle: true,
|
||||
...credentials
|
||||
}),
|
||||
bucket: target.config.bucket
|
||||
}
|
||||
}
|
||||
return {
|
||||
client: new S3Client({ region: region || 'us-east-1', bucketEndpoint: true, ...credentials }),
|
||||
bucket: baseUrl
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether the store is telling us the key simply is not there. */
|
||||
function isNotFound(err: any): boolean {
|
||||
return (
|
||||
err?.name === 'NoSuchKey' || err?.name === 'NotFound' || err?.$metadata?.httpStatusCode === 404
|
||||
)
|
||||
}
|
||||
|
||||
const s3Client: ObjectStoreClient = {
|
||||
async put(target, key, data, contentType) {
|
||||
await clientFor(target).send(
|
||||
new PutObjectCommand({
|
||||
Bucket: target.config.bucket,
|
||||
Key: key,
|
||||
Body: data,
|
||||
ContentType: contentType,
|
||||
// -> Omitted rather than sent as Standard: a compatible store that does not implement
|
||||
// storage classes will reject the header outright rather than ignore it
|
||||
...(target.config.storageClass && target.config.storageClass !== 'STANDARD'
|
||||
? { StorageClass: target.config.storageClass }
|
||||
: {})
|
||||
})
|
||||
)
|
||||
},
|
||||
|
||||
async get(target, key) {
|
||||
try {
|
||||
const resp = await clientFor(target).send(
|
||||
new GetObjectCommand({ Bucket: target.config.bucket, Key: key })
|
||||
)
|
||||
const bytes = await resp.Body?.transformToByteArray()
|
||||
return bytes ? Buffer.from(bytes) : null
|
||||
} catch (err: any) {
|
||||
if (isNotFound(err)) {
|
||||
// -> This target does not have the file: enabled after the upload, or removed from outside
|
||||
// the wiki. Not a fault — the caller asks the next target.
|
||||
return null
|
||||
}
|
||||
throw err
|
||||
}
|
||||
},
|
||||
|
||||
async remove(target, key) {
|
||||
try {
|
||||
await clientFor(target).send(
|
||||
new DeleteObjectCommand({ Bucket: target.config.bucket, Key: key })
|
||||
)
|
||||
} catch (err: any) {
|
||||
// -> S3 itself answers a delete of a missing key with success; not every compatible store does
|
||||
if (!isNotFound(err)) {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
async copy(target, fromKey, toKey) {
|
||||
try {
|
||||
await clientFor(target).send(
|
||||
new CopyObjectCommand({
|
||||
Bucket: target.config.bucket,
|
||||
// -> The source is bucket-qualified and URI-encoded, which is the one part of this API that
|
||||
// does not take a plain key: a `#` or a `+` in a file name would otherwise be read as
|
||||
// part of the URL rather than as part of the name
|
||||
CopySource: encodeURI(`${target.config.bucket}/${fromKey}`),
|
||||
Key: toKey
|
||||
})
|
||||
)
|
||||
return true
|
||||
} catch (err: any) {
|
||||
if (isNotFound(err)) {
|
||||
return false
|
||||
}
|
||||
throw err
|
||||
}
|
||||
},
|
||||
|
||||
async presign(target, { key, expiresInSeconds, contentType, downloadAs }) {
|
||||
const { client, bucket } = signingTargetFor(target, signingBaseUrl(target))
|
||||
/*
|
||||
The response headers travel in the signature rather than being left to the object's own
|
||||
metadata: the wiki knows what it thinks the file is and whether this request was a download,
|
||||
and neither of those is necessarily what was stored — an object written by an older instance,
|
||||
or one uploaded straight into the bucket, carries whatever it carries.
|
||||
*/
|
||||
return getSignedUrl(
|
||||
client,
|
||||
new GetObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
ResponseContentType: contentType,
|
||||
...(downloadAs
|
||||
? {
|
||||
ResponseContentDisposition: `attachment; filename="${encodeURIComponent(downloadAs)}"`
|
||||
}
|
||||
: {})
|
||||
}),
|
||||
{ expiresIn: expiresInSeconds }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* S3 object storage module
|
||||
*
|
||||
* Amazon S3 and everything that speaks its API. One module rather than the three that 2.x shipped —
|
||||
* an AWS one, a DigitalOcean one and a custom one — because the difference between them is an
|
||||
* endpoint and a region, and treating that as a preset meant a new module for every service that
|
||||
* appeared. Empty endpoint is AWS; anything else is whichever store the URL points at.
|
||||
*
|
||||
* The keys are the same paths the disk target writes, so a bucket and a folder hold the wiki's content
|
||||
* laid out identically, and the shape of both is the site's `pathPrefixFor` to decide. See
|
||||
* `helpers/storageObjects.ts` for everything above the four calls below.
|
||||
*/
|
||||
export default objectStorageModule(s3Client)
|
||||
@ -0,0 +1,100 @@
|
||||
key: sftp
|
||||
title: SFTP
|
||||
icon: '/_assets/icons/ultraviolet-nas.svg'
|
||||
banner: '/_assets/storage/ssh.jpg'
|
||||
description: Store the wiki's content as ordinary files on a remote server over SSH. The same tree the local disk target writes, on a machine that is not this one. Meant as a copy rather than a source, so it cannot be chosen under Content Delivery.
|
||||
vendor: 'Wiki.js'
|
||||
website: 'https://js.wiki'
|
||||
assetDelivery:
|
||||
isDirectAccessSupported: false
|
||||
# -> A place to keep a copy of the site's content, not one to serve it from: every image on every
|
||||
# page would be an SSH round trip. Written to, exported to and imported from as normal; simply
|
||||
# never offered under Content Delivery.
|
||||
isDeliverySupported: false
|
||||
contentTypes:
|
||||
defaultTypesEnabled: ['pages', 'images', 'documents', 'others', 'large']
|
||||
props:
|
||||
host:
|
||||
type: String
|
||||
title: Host
|
||||
default: ''
|
||||
hint: Hostname or IP of the remote SSH server.
|
||||
icon: dns
|
||||
order: 1
|
||||
port:
|
||||
type: Number
|
||||
title: Port
|
||||
default: 22
|
||||
hint: SSH port of the remote server.
|
||||
icon: ethernet-off
|
||||
order: 2
|
||||
authMode:
|
||||
type: String
|
||||
title: Authentication Method
|
||||
default: 'privateKey'
|
||||
hint: Whether to use Private Key or Password-based authentication. A private key is highly recommended for best security.
|
||||
icon: grand-master-key
|
||||
enum:
|
||||
- privateKey|Private Key
|
||||
- password|Password
|
||||
enumDisplay: buttons
|
||||
order: 3
|
||||
username:
|
||||
type: String
|
||||
title: Username
|
||||
default: ''
|
||||
hint: Username for authentication.
|
||||
icon: test-account
|
||||
order: 4
|
||||
privateKey:
|
||||
type: String
|
||||
title: Private Key Contents
|
||||
default: ''
|
||||
hint: Contents of the private key, including the BEGIN and END lines.
|
||||
icon: key
|
||||
multiline: true
|
||||
sensitive: true
|
||||
order: 5
|
||||
if:
|
||||
- { key: 'authMode', eq: 'privateKey' }
|
||||
passphrase:
|
||||
type: String
|
||||
title: Private Key Passphrase
|
||||
default: ''
|
||||
hint: Passphrase if the private key is encrypted, leave empty otherwise.
|
||||
icon: password
|
||||
sensitive: true
|
||||
order: 6
|
||||
if:
|
||||
- { key: 'authMode', eq: 'privateKey' }
|
||||
password:
|
||||
type: String
|
||||
title: Password
|
||||
default: ''
|
||||
hint: Password for authentication.
|
||||
icon: password
|
||||
sensitive: true
|
||||
order: 6
|
||||
if:
|
||||
- { key: 'authMode', eq: 'password' }
|
||||
basePath:
|
||||
type: String
|
||||
title: Base Directory Path
|
||||
default: '/var/wiki'
|
||||
hint: Where this site's folder tree is written on the remote server. It must already exist and be writable by the user above. Give each site its own path unless you turn on Add Site ID Prefix under Configuration.
|
||||
icon: symlink-directory
|
||||
order: 7
|
||||
actions:
|
||||
exportAll:
|
||||
label: Export Everything
|
||||
hint: Write a copy of every page and asset this target is configured to hold to the remote server, overwriting whatever is already there. Nothing in the database is changed and nothing is moved, so this is how content created before the target was enabled gets onto it.
|
||||
icon: this-way-up
|
||||
importAll:
|
||||
label: Import Everything
|
||||
hint: Take every page and asset on the remote server that the wiki does not have yet into the wiki. A file is imported as a page if its extension is one of the site's Page Extensions, or if it declares an editor in its front matter; everything else is imported as an asset. Anything already at the same path is left alone on both sides, so this is safe to run again.
|
||||
icon: database-daily-import
|
||||
importAllOverwrite:
|
||||
label: Import Everything and Overwrite
|
||||
hint: The same walk, with the remote server winning every collision. For a restore, or a tree edited on the server that is meant to be taken as the new truth.
|
||||
warn: This replaces what the wiki currently has wherever a remote file lands on it. An overwritten page keeps its previous version in its history, but a file has none - the bytes it replaces are gone from every storage target holding them.
|
||||
icon: database-restore
|
||||
@ -0,0 +1,515 @@
|
||||
import path from 'node:path'
|
||||
import SftpClient from 'ssh2-sftp-client'
|
||||
import {
|
||||
assetRelPath,
|
||||
importTree,
|
||||
pageRelPath,
|
||||
serializePage
|
||||
} from '../../../helpers/storageFiles.ts'
|
||||
import type { ImportSummary, StoredFile } from '../../../helpers/storageFiles.ts'
|
||||
import type { StorageModule, StorageTarget } from '../../../models/storage.ts'
|
||||
|
||||
/** Where files go when the target has no base path configured, matching the definition default. */
|
||||
const DEFAULT_BASE_PATH = '/var/wiki'
|
||||
|
||||
/** Names never walked by an import, as on the local disk. */
|
||||
const IGNORED_NAME = /^\./
|
||||
|
||||
/** One live connection, plus the queue that keeps it to one operation at a time. */
|
||||
interface Connection {
|
||||
client: SftpClient
|
||||
fingerprint: string
|
||||
queue: Promise<unknown>
|
||||
}
|
||||
|
||||
const connections = new Map<string, Connection>()
|
||||
|
||||
/** The remote root this target writes under, without a trailing slash. */
|
||||
function baseDir(target: StorageTarget): string {
|
||||
return (target.config.basePath || DEFAULT_BASE_PATH).replace(/\/+$/, '') || '/'
|
||||
}
|
||||
|
||||
/** Everything a connection is made from — a change to any of it needs a new one. */
|
||||
function configFingerprint(target: StorageTarget): string {
|
||||
const c = target.config
|
||||
return JSON.stringify([
|
||||
c.host,
|
||||
c.port,
|
||||
c.authMode,
|
||||
c.username,
|
||||
c.privateKey,
|
||||
c.passphrase,
|
||||
c.password
|
||||
])
|
||||
}
|
||||
|
||||
/**
|
||||
* The absolute remote path of a stored file, refusing anything that would land outside the root.
|
||||
*
|
||||
* `path.posix` throughout rather than `path`, because the shape of the remote file system has nothing
|
||||
* to do with the shape of this one: a wiki running on Windows still talks to an SSH server in
|
||||
* slashes, and `path.win32.resolve` would turn every one of these into a backslash path the server
|
||||
* has never heard of.
|
||||
*/
|
||||
function remotePath(target: StorageTarget, relPath: string): string {
|
||||
const base = baseDir(target)
|
||||
const resolved = path.posix.resolve(base, relPath)
|
||||
if (resolved !== base && !resolved.startsWith(base === '/' ? '/' : `${base}/`)) {
|
||||
throw new Error(`The stored path "${relPath}" resolves outside the base directory.`)
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
/**
|
||||
* Run something against this target's connection, one operation at a time.
|
||||
*
|
||||
* A single SFTP client multiplexes badly — `ssh2-sftp-client` is explicit that concurrent operations
|
||||
* on one instance are not supported — so the queue is what makes two simultaneous uploads safe. Per
|
||||
* target, because two targets are two servers.
|
||||
*
|
||||
* A connection that fails is dropped rather than reused: the failure may be the connection itself,
|
||||
* and the next operation is what re-establishes it. This is also the reconnect path after the server
|
||||
* has timed the session out, which for a wiki that uploads a file once a week it always will have.
|
||||
*/
|
||||
async function withClient<T>(
|
||||
target: StorageTarget,
|
||||
run: (client: SftpClient) => Promise<T>
|
||||
): Promise<T> {
|
||||
const fingerprint = configFingerprint(target)
|
||||
let connection = connections.get(target.id)
|
||||
if (connection && connection.fingerprint !== fingerprint) {
|
||||
await connection.client.end().catch(() => {})
|
||||
connections.delete(target.id)
|
||||
connection = undefined
|
||||
}
|
||||
if (!connection) {
|
||||
connection = { client: new SftpClient(), fingerprint, queue: Promise.resolve() }
|
||||
connections.set(target.id, connection)
|
||||
connection.queue = connect(target, connection.client).catch((err) => {
|
||||
connections.delete(target.id)
|
||||
throw err
|
||||
})
|
||||
}
|
||||
|
||||
const entry = connection
|
||||
const result = entry.queue.then(
|
||||
() => run(entry.client),
|
||||
// -> The previous operation failed; this one still gets its turn, on a connection that may well
|
||||
// have been replaced underneath it
|
||||
() => run(entry.client)
|
||||
)
|
||||
entry.queue = result.catch(() => {})
|
||||
try {
|
||||
return await result
|
||||
} catch (err: any) {
|
||||
if (isConnectionError(err)) {
|
||||
await entry.client.end().catch(() => {})
|
||||
connections.delete(target.id)
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
/** Open the connection and check the base directory is actually there. */
|
||||
async function connect(target: StorageTarget, client: SftpClient): Promise<void> {
|
||||
const { host, port, authMode, username, privateKey, passphrase, password } = target.config
|
||||
WIKI.logger.info(`(STORAGE/SFTP) Connecting to ${username}@${host}...`)
|
||||
await client.connect({
|
||||
host,
|
||||
port: Number(port) || 22,
|
||||
username,
|
||||
...(authMode === 'password'
|
||||
? { password }
|
||||
: { privateKey, ...(passphrase ? { passphrase } : {}) })
|
||||
})
|
||||
const base = baseDir(target)
|
||||
if (!(await client.exists(base))) {
|
||||
// -> Not created: the base path is where somebody has decided this site's content belongs, and
|
||||
// a typo in it should be a refusal rather than a new directory nobody meant
|
||||
throw new Error(
|
||||
`The base directory ${base} does not exist on the remote server, or the user cannot see it.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether this looks like the session rather than the file being the problem. */
|
||||
function isConnectionError(err: any): boolean {
|
||||
const message = String(err?.message ?? '')
|
||||
return (
|
||||
/connect|closed|ECONNRESET|ETIMEDOUT|EPIPE|not connected|handshake|authentication/i.test(
|
||||
message
|
||||
) && !isMissing(err)
|
||||
)
|
||||
}
|
||||
|
||||
/** Whether the server is telling us the file simply is not there. */
|
||||
function isMissing(err: any): boolean {
|
||||
return err?.code === 2 || /no such file|ENOENT/i.test(String(err?.message ?? ''))
|
||||
}
|
||||
|
||||
/** Read every file under a remote directory, skipping anything hidden. */
|
||||
async function walkRemote(client: SftpClient, root: string, dir: string): Promise<StoredFile[]> {
|
||||
const found: StoredFile[] = []
|
||||
let entries
|
||||
try {
|
||||
entries = await client.list(dir)
|
||||
} catch (err: any) {
|
||||
if (isMissing(err)) {
|
||||
return found
|
||||
}
|
||||
throw err
|
||||
}
|
||||
for (const entry of entries) {
|
||||
if (IGNORED_NAME.test(entry.name)) {
|
||||
continue
|
||||
}
|
||||
const full = path.posix.join(dir, entry.name)
|
||||
if (entry.type === 'd') {
|
||||
found.push(...(await walkRemote(client, root, full)))
|
||||
} else if (entry.type === '-') {
|
||||
found.push({
|
||||
filePath: full,
|
||||
segments: path.posix.relative(root, full).split('/')
|
||||
})
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
/** Write a file, creating the directories above it. */
|
||||
async function writeRemote(
|
||||
client: SftpClient,
|
||||
target: StorageTarget,
|
||||
relPath: string,
|
||||
data: Buffer
|
||||
): Promise<void> {
|
||||
const filePath = remotePath(target, relPath)
|
||||
const dir = path.posix.dirname(filePath)
|
||||
// -> `true` is recursive, and an existing directory is not an error to this client
|
||||
await client.mkdir(dir, true).catch(() => {})
|
||||
await client.put(data, filePath)
|
||||
}
|
||||
|
||||
/** Remove a file, and any directories it leaves empty, stopping at the first one still in use. */
|
||||
async function removeRemote(
|
||||
client: SftpClient,
|
||||
target: StorageTarget,
|
||||
relPath: string
|
||||
): Promise<void> {
|
||||
const base = baseDir(target)
|
||||
const filePath = remotePath(target, relPath)
|
||||
try {
|
||||
await client.delete(filePath)
|
||||
} catch (err: any) {
|
||||
if (!isMissing(err)) {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
let dir = path.posix.dirname(filePath)
|
||||
while (dir !== base && dir.startsWith(`${base}/`)) {
|
||||
try {
|
||||
await client.rmdir(dir)
|
||||
} catch {
|
||||
// -> Not empty, or another request is writing into it. Best effort, exactly as on disk.
|
||||
return
|
||||
}
|
||||
dir = path.posix.dirname(dir)
|
||||
}
|
||||
}
|
||||
|
||||
/** Follow a rename, where either end may be a locale this site does not store. */
|
||||
async function moveRemote(
|
||||
client: SftpClient,
|
||||
target: StorageTarget,
|
||||
fromRel: string | null,
|
||||
toRel: string | null
|
||||
): Promise<void> {
|
||||
if (!fromRel) {
|
||||
return
|
||||
}
|
||||
if (!toRel) {
|
||||
await removeRemote(client, target, fromRel)
|
||||
return
|
||||
}
|
||||
const from = remotePath(target, fromRel)
|
||||
const to = remotePath(target, toRel)
|
||||
await client.mkdir(path.posix.dirname(to), true).catch(() => {})
|
||||
try {
|
||||
await client.rename(from, to)
|
||||
} catch (err: any) {
|
||||
// -> Nothing there to move: this target was enabled after the file was uploaded
|
||||
if (!isMissing(err)) {
|
||||
throw err
|
||||
}
|
||||
return
|
||||
}
|
||||
let dir = path.posix.dirname(from)
|
||||
const base = baseDir(target)
|
||||
while (dir !== base && dir.startsWith(`${base}/`)) {
|
||||
try {
|
||||
await client.rmdir(dir)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
dir = path.posix.dirname(dir)
|
||||
}
|
||||
}
|
||||
|
||||
/** What an import run did, in the words the two import actions report it with. */
|
||||
function describeImport(summary: ImportSummary | null, overwrite: boolean): string {
|
||||
if (!summary) {
|
||||
return 'There is nothing in the base directory for this site yet.'
|
||||
}
|
||||
const verb = overwrite ? 'Imported or replaced' : 'Imported'
|
||||
const parts = []
|
||||
if (summary.pages > 0) {
|
||||
parts.push(`${verb} ${summary.pages} page(s).`)
|
||||
}
|
||||
if (summary.assets > 0) {
|
||||
parts.push(`${verb} ${summary.assets} asset(s).`)
|
||||
}
|
||||
if (parts.length < 1) {
|
||||
parts.push(overwrite ? 'There was nothing to import.' : 'There was nothing new to import.')
|
||||
}
|
||||
if (summary.skipped > 0) {
|
||||
parts.push(
|
||||
overwrite
|
||||
? `${summary.skipped} could not replace what is at their path and were left alone.`
|
||||
: `${summary.skipped} were already in the wiki and were left alone.`
|
||||
)
|
||||
}
|
||||
if (summary.failed > 0) {
|
||||
parts.push(`${summary.failed} could not be imported - see the server log.`)
|
||||
}
|
||||
return parts.join(' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* SFTP storage module
|
||||
*
|
||||
* The local disk target, on a machine that is not this one. The same tree, laid out the same way and
|
||||
* bracketed by whatever the site's `pathPrefixFor` says, written over SSH — so a wiki can keep its
|
||||
* content on a NAS or a backup host without that machine having to run anything but sshd.
|
||||
*
|
||||
* Everything about the shape of the tree is `helpers/storageFiles.ts`, shared with `disk` and `git`.
|
||||
* What this module adds is the connection: one at a time per target, re-established when it drops,
|
||||
* and posix paths throughout however this server spells its own.
|
||||
*
|
||||
* **It reads as well as writes**, which the 2.x module did not — it declared no streaming support and
|
||||
* had no way back out. Here `getAsset` is part of the contract, so a site can serve files from the
|
||||
* remote host, and the two import actions can take a tree on it into the wiki.
|
||||
*/
|
||||
const sftpStorage: StorageModule = {
|
||||
canStore(target, ref) {
|
||||
return WIKI.models.storage.pathPrefixFor(target.siteId, ref.locale) !== null
|
||||
},
|
||||
|
||||
async putAsset(target, ref, data) {
|
||||
const relPath = assetRelPath(target, ref)
|
||||
// -> Guarded rather than skipped: the model asks `canStore` before dispatching a write, so
|
||||
// reaching this means somebody wrote without asking, and an asset may have no other copy
|
||||
if (!relPath) {
|
||||
throw new Error(
|
||||
`${target.title} has no path for ${ref.locale} content, so ${ref.fileName} cannot be stored there.`
|
||||
)
|
||||
}
|
||||
await withClient(target, (client) => writeRemote(client, target, relPath, data))
|
||||
},
|
||||
|
||||
async getAsset(target, ref) {
|
||||
const relPath = assetRelPath(target, ref)
|
||||
if (!relPath) {
|
||||
return null
|
||||
}
|
||||
return withClient(target, async (client) => {
|
||||
try {
|
||||
return (await client.get(remotePath(target, relPath))) as Buffer
|
||||
} catch (err: any) {
|
||||
if (isMissing(err)) {
|
||||
// -> This target does not have the file: enabled after the upload, or removed from
|
||||
// outside the wiki. Not a fault — the caller asks the next target.
|
||||
return null
|
||||
}
|
||||
throw err
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
async deleteAsset(target, ref) {
|
||||
const relPath = assetRelPath(target, ref)
|
||||
if (!relPath) {
|
||||
return
|
||||
}
|
||||
await withClient(target, (client) => removeRemote(client, target, relPath))
|
||||
},
|
||||
|
||||
async moveAsset(target, ref, previous) {
|
||||
await withClient(target, (client) =>
|
||||
moveRemote(
|
||||
client,
|
||||
target,
|
||||
assetRelPath(target, { ...ref, ...previous }),
|
||||
assetRelPath(target, ref)
|
||||
)
|
||||
)
|
||||
},
|
||||
|
||||
async putPage(target, ref, page) {
|
||||
const relPath = pageRelPath(target, ref)
|
||||
// -> Unlike an asset, a page with no place here is not worth failing over: it is in the
|
||||
// database, which is where a page always is, and this copy is the thing the site declined
|
||||
if (!relPath) {
|
||||
return
|
||||
}
|
||||
await withClient(target, (client) =>
|
||||
writeRemote(client, target, relPath, Buffer.from(serializePage(ref, page), 'utf8'))
|
||||
)
|
||||
},
|
||||
|
||||
async deletePage(target, ref) {
|
||||
// -> Exactly one name, taken from the page's own content type: in a folder where pages and
|
||||
// attachments sit together, guessing at the others would delete whatever is beside it
|
||||
const relPath = pageRelPath(target, ref)
|
||||
if (!relPath) {
|
||||
return
|
||||
}
|
||||
await withClient(target, (client) => removeRemote(client, target, relPath))
|
||||
},
|
||||
|
||||
async movePage(target, ref, previousPath) {
|
||||
await withClient(target, (client) =>
|
||||
moveRemote(
|
||||
client,
|
||||
target,
|
||||
pageRelPath(target, { ...ref, path: previousPath }),
|
||||
pageRelPath(target, ref)
|
||||
)
|
||||
)
|
||||
},
|
||||
|
||||
/**
|
||||
* Write a copy of everything this target is configured to hold to the remote server.
|
||||
*
|
||||
* A plain copy: content is read from wherever it currently lives and written here, overwriting
|
||||
* whatever is at each path. Nothing in the database is touched, so this is how content that
|
||||
* predates the target being enabled gets onto it, and running it twice does the same work.
|
||||
*/
|
||||
async exportAll(target: StorageTarget): Promise<string> {
|
||||
let assets = 0
|
||||
let unreadable = 0
|
||||
let unstored = 0
|
||||
let pages = 0
|
||||
|
||||
await withClient(target, async (client) => {
|
||||
for (const asset of await WIKI.models.assets.listStoredAssets(target.siteId)) {
|
||||
const contentType = WIKI.models.storage.contentTypeFor(
|
||||
target.siteId,
|
||||
asset.kind,
|
||||
asset.fileSize
|
||||
)
|
||||
if (!target.contentTypes.activeTypes.includes(contentType)) {
|
||||
continue
|
||||
}
|
||||
const relPath = assetRelPath(target, asset)
|
||||
if (!relPath) {
|
||||
unstored++
|
||||
continue
|
||||
}
|
||||
const data = await WIKI.models.storage.getAsset(asset)
|
||||
if (!data) {
|
||||
unreadable++
|
||||
continue
|
||||
}
|
||||
await writeRemote(client, target, relPath, data)
|
||||
assets++
|
||||
}
|
||||
|
||||
if (target.contentTypes.activeTypes.includes('pages')) {
|
||||
for (const { ref, content } of await WIKI.models.pages.listForStorage(target.siteId)) {
|
||||
const relPath = pageRelPath(target, ref)
|
||||
if (!relPath) {
|
||||
unstored++
|
||||
continue
|
||||
}
|
||||
await writeRemote(
|
||||
client,
|
||||
target,
|
||||
relPath,
|
||||
Buffer.from(serializePage(ref, content), 'utf8')
|
||||
)
|
||||
pages++
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
WIKI.logger.info(
|
||||
`(STORAGE/SFTP) Exported ${assets} asset(s) and ${pages} page(s) to ${baseDir(target)} [ OK ]`
|
||||
)
|
||||
const parts = []
|
||||
if (assets > 0 || pages > 0) {
|
||||
parts.push(`Exported ${pages} page(s) and ${assets} asset(s).`)
|
||||
} else {
|
||||
parts.push('There was nothing to export.')
|
||||
}
|
||||
if (unreadable > 0) {
|
||||
parts.push(`${unreadable} asset(s) could not be read and were skipped.`)
|
||||
}
|
||||
if (unstored > 0) {
|
||||
const { primaryLocale } = WIKI.models.storage.pathLayoutFor(target.siteId)
|
||||
parts.push(
|
||||
`${unstored} item(s) are not in the ${primaryLocale} locale, which is the only one this site stores.`
|
||||
)
|
||||
}
|
||||
return parts.join(' ')
|
||||
},
|
||||
|
||||
/**
|
||||
* Take everything on the remote server that the wiki does not know about yet into the wiki.
|
||||
*
|
||||
* The direction that makes the remote host a store rather than a dumping ground: content arrives
|
||||
* there from outside — restored from a backup, dropped in over scp — and this is what turns it back
|
||||
* into pages and assets. What counts as a page is `importTree`'s to say, exactly as it is for the
|
||||
* local disk; the only difference here is where the bytes are read from.
|
||||
*/
|
||||
async importAll(target: StorageTarget, actorId: string): Promise<string> {
|
||||
return describeImport(await runImport(target, actorId, false), false)
|
||||
},
|
||||
|
||||
/**
|
||||
* The same walk, with the remote server winning every collision.
|
||||
*
|
||||
* For a restore, or a tree edited on the server that is meant to be taken as the new truth. A page
|
||||
* it replaces keeps its previous version in its history; an asset has none.
|
||||
*/
|
||||
async importAllOverwrite(target: StorageTarget, actorId: string): Promise<string> {
|
||||
return describeImport(await runImport(target, actorId, true), true)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk the remote tree and hand it to the shared adoption code.
|
||||
*
|
||||
* One connection for the whole run — the walk and every read go through the same queued client,
|
||||
* rather than a fresh operation per file.
|
||||
*/
|
||||
async function runImport(
|
||||
target: StorageTarget,
|
||||
actorId: string,
|
||||
overwrite: boolean
|
||||
): Promise<ImportSummary | null> {
|
||||
const root = baseDir(target)
|
||||
return withClient(target, async (client) => {
|
||||
const files = await walkRemote(client, root, root)
|
||||
return importTree({
|
||||
target,
|
||||
root,
|
||||
actorId,
|
||||
overwrite,
|
||||
files,
|
||||
// -> `filePath` here is already an absolute remote path, put there by the walk above
|
||||
readFile: async (filePath) => (await client.get(filePath)) as Buffer
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export default sftpStorage
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,69 @@
|
||||
/**
|
||||
* Sync every storage target that has a remote to keep in step with, and whose site is due.
|
||||
*
|
||||
* The git target's schedule, and what makes it a synchronized store rather than a local repository
|
||||
* that happens to get committed to. A change is committed the moment it is made — that happens on the
|
||||
* request, not here — and this is what pushes those commits and brings back what other people pushed.
|
||||
*
|
||||
* **This runs every minute; the site's `syncInterval` decides what actually happens.** The interval is
|
||||
* a per-site setting, so one schedule cannot express it — the tick is therefore as fine as the
|
||||
* shortest interval anybody can ask for, and each site is skipped on the ticks that are not its own.
|
||||
*
|
||||
* Due-ness is read off the clock rather than off a record of when each target last synced. A site with
|
||||
* a five-minute interval syncs on every fifth minute of the epoch, which needs nothing stored, means
|
||||
* two instances agree without coordinating, survives a restart, and cannot drift. What it gives up is
|
||||
* catching up on a missed tick: an instance that was down at the moment simply waits for the next one,
|
||||
* which for something that runs all day is the right trade.
|
||||
*
|
||||
* Every target is attempted whatever the ones before it did: a site whose credentials have expired
|
||||
* must not stop the rest of them syncing. `executeAction` records the outcome on the target itself,
|
||||
* so a failure shows up on its Status card in the admin area rather than only in this log.
|
||||
*
|
||||
* **One instance at a time.** The scheduler hands a job to a single instance, which is the one whose
|
||||
* working copy is synced. Every instance in a high-availability set keeps its own, so they each fall
|
||||
* in step at their own turn rather than fighting over one repository.
|
||||
*/
|
||||
export async function task(): Promise<void> {
|
||||
const syncable = await WIKI.models.storage.syncableTargets()
|
||||
if (syncable.length < 1) {
|
||||
return
|
||||
}
|
||||
|
||||
const minute = Math.floor(Temporal.Now.instant().epochMilliseconds / 60_000)
|
||||
const targets = syncable.filter((target) => {
|
||||
const interval = WIKI.models.storage.syncIntervalFor(target.siteId)
|
||||
// -> An interval nothing can be made of is a site that is never synced on a schedule, rather than
|
||||
// one synced every minute. `validateSiteConfig` refuses to store such a value in the first
|
||||
// place, so this is the belt to that braces.
|
||||
return interval > 0 && minute % interval === 0
|
||||
})
|
||||
if (targets.length < 1) {
|
||||
return
|
||||
}
|
||||
|
||||
// -> A pull creates content, and content records who authored it. There is nobody behind a
|
||||
// scheduled run, so it is recorded against the wiki's own administrator.
|
||||
const actorId = await WIKI.models.users.getSystemActorId()
|
||||
if (!actorId) {
|
||||
WIKI.logger.warn(
|
||||
'Syncing storage targets: no active administrator to attribute incoming content to [ SKIPPED ]'
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
WIKI.logger.info(`Syncing ${targets.length} storage target(s)...`)
|
||||
let failed = 0
|
||||
for (const target of targets) {
|
||||
try {
|
||||
const message = await WIKI.models.storage.executeAction(target, 'sync', actorId)
|
||||
WIKI.logger.info(`Synced ${target.title} for site ${target.siteId}: ${message ?? 'done'}`)
|
||||
} catch (err: any) {
|
||||
failed++
|
||||
WIKI.logger.warn(`Could not sync ${target.title} for site ${target.siteId} [ FAILED ]`)
|
||||
WIKI.logger.warn(err.message)
|
||||
}
|
||||
}
|
||||
WIKI.logger.info(
|
||||
`Syncing storage targets: ${targets.length - failed} of ${targets.length} succeeded [ COMPLETED ]`
|
||||
)
|
||||
}
|
||||
Loading…
Reference in new issue