diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 924101cb8..8c0d613c2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -89,7 +89,7 @@ jobs: strategy: matrix: - dbtype: [postgres, mysql, mariadb, mssql, sqlite] + dbtype: [postgres, mysql, mariadb, sqlite] steps: - uses: actions/checkout@v4 @@ -125,8 +125,8 @@ jobs: include: - platform: linux/arm64 docker: arm64 - - platform: linux/arm/v7 - docker: armv7 + # - platform: linux/arm/v7 + # docker: armv7 steps: - uses: actions/checkout@v4 @@ -192,7 +192,7 @@ jobs: - name: Setup Node.js environment uses: actions/setup-node@v4 with: - node-version: 18.x + node-version: 20.x - name: Download a Build Artifact uses: actions/download-artifact@v4 @@ -257,8 +257,8 @@ jobs: run: | echo "Creating the manifests..." - docker manifest create requarks/wiki:beta-$REL_VERSION_STRICT requarks/wiki:canary-$REL_VERSION_STRICT requarks/wiki:canary-arm64-$REL_VERSION_STRICT requarks/wiki:canary-armv7-$REL_VERSION_STRICT - docker manifest create ghcr.io/requarks/wiki:beta-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-arm64-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-armv7-$REL_VERSION_STRICT + docker manifest create requarks/wiki:beta-$REL_VERSION_STRICT requarks/wiki:canary-$REL_VERSION_STRICT requarks/wiki:canary-arm64-$REL_VERSION_STRICT + docker manifest create ghcr.io/requarks/wiki:beta-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-arm64-$REL_VERSION_STRICT echo "Pushing the manifests..." @@ -307,14 +307,14 @@ jobs: echo "Using major $MAJOR and minor $MINOR..." echo "Creating the manifests..." - docker manifest create requarks/wiki:$REL_VERSION_STRICT requarks/wiki:canary-$REL_VERSION_STRICT requarks/wiki:canary-arm64-$REL_VERSION_STRICT requarks/wiki:canary-armv7-$REL_VERSION_STRICT - docker manifest create requarks/wiki:$MAJOR requarks/wiki:canary-$REL_VERSION_STRICT requarks/wiki:canary-arm64-$REL_VERSION_STRICT requarks/wiki:canary-armv7-$REL_VERSION_STRICT - docker manifest create requarks/wiki:$MAJORMINOR requarks/wiki:canary-$REL_VERSION_STRICT requarks/wiki:canary-arm64-$REL_VERSION_STRICT requarks/wiki:canary-armv7-$REL_VERSION_STRICT - docker manifest create requarks/wiki:latest requarks/wiki:canary-$REL_VERSION_STRICT requarks/wiki:canary-arm64-$REL_VERSION_STRICT requarks/wiki:canary-armv7-$REL_VERSION_STRICT - docker manifest create ghcr.io/requarks/wiki:$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-arm64-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-armv7-$REL_VERSION_STRICT - docker manifest create ghcr.io/requarks/wiki:$MAJOR ghcr.io/requarks/wiki:canary-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-arm64-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-armv7-$REL_VERSION_STRICT - docker manifest create ghcr.io/requarks/wiki:$MAJORMINOR ghcr.io/requarks/wiki:canary-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-arm64-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-armv7-$REL_VERSION_STRICT - docker manifest create ghcr.io/requarks/wiki:latest ghcr.io/requarks/wiki:canary-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-arm64-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-armv7-$REL_VERSION_STRICT + docker manifest create requarks/wiki:$REL_VERSION_STRICT requarks/wiki:canary-$REL_VERSION_STRICT requarks/wiki:canary-arm64-$REL_VERSION_STRICT + docker manifest create requarks/wiki:$MAJOR requarks/wiki:canary-$REL_VERSION_STRICT requarks/wiki:canary-arm64-$REL_VERSION_STRICT + docker manifest create requarks/wiki:$MAJORMINOR requarks/wiki:canary-$REL_VERSION_STRICT requarks/wiki:canary-arm64-$REL_VERSION_STRICT + docker manifest create requarks/wiki:latest requarks/wiki:canary-$REL_VERSION_STRICT requarks/wiki:canary-arm64-$REL_VERSION_STRICT + docker manifest create ghcr.io/requarks/wiki:$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-arm64-$REL_VERSION_STRICT + docker manifest create ghcr.io/requarks/wiki:$MAJOR ghcr.io/requarks/wiki:canary-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-arm64-$REL_VERSION_STRICT + docker manifest create ghcr.io/requarks/wiki:$MAJORMINOR ghcr.io/requarks/wiki:canary-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-arm64-$REL_VERSION_STRICT + docker manifest create ghcr.io/requarks/wiki:latest ghcr.io/requarks/wiki:canary-$REL_VERSION_STRICT ghcr.io/requarks/wiki:canary-arm64-$REL_VERSION_STRICT echo "Pushing the manifests..." @@ -358,16 +358,16 @@ jobs: token: ${{ github.token }} artifacts: 'drop/wiki-js.tar.gz,drop-win/wiki-js-windows.tar.gz' - - name: Notify Slack Releases Channel - uses: slackapi/slack-github-action@v1.26.0 - with: - payload: | - { - "text": "Wiki.js ${{ github.ref_name }} has been released." - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + # - name: Notify Slack Releases Channel + # uses: slackapi/slack-github-action@v1.26.0 + # with: + # payload: | + # { + # "text": "Wiki.js ${{ github.ref_name }} has been released." + # } + # env: + # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + # SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - name: Notify Telegram Channel uses: appleboy/telegram-action@v0.1.1 diff --git a/README.md b/README.md index 468ddd8c6..5a25b9c2e 100644 --- a/README.md +++ b/README.md @@ -8,16 +8,14 @@ [![Release](https://img.shields.io/github/release/Requarks/wiki.svg?style=flat&maxAge=3600)](https://github.com/Requarks/wiki/releases) [![License](https://img.shields.io/badge/license-AGPLv3-blue.svg?style=flat)](https://github.com/requarks/wiki/blob/master/LICENSE) [![Standard - JavaScript Style Guide](https://img.shields.io/badge/code%20style-standard-green.svg?style=flat&logo=javascript&logoColor=white)](http://standardjs.com/) +[![Build + Publish](https://github.com/Requarks/wiki/actions/workflows/build.yml/badge.svg)](https://github.com/Requarks/wiki/actions/workflows/build.yml) +[![GitHub Sponsors](https://img.shields.io/github/sponsors/ngpixel?logo=github&color=ea4aaa)](https://github.com/users/NGPixel/sponsorship) +[![Open Collective backers and sponsors](https://img.shields.io/opencollective/all/wikijs?label=backers&color=218bff&logo=opencollective&logoColor=white)](https://opencollective.com/wikijs) [![Downloads](https://img.shields.io/github/downloads/Requarks/wiki/total.svg?style=flat&logo=github)](https://github.com/Requarks/wiki/releases) [![Docker Pulls](https://img.shields.io/docker/pulls/requarks/wiki.svg?logo=docker&logoColor=white)](https://hub.docker.com/r/requarks/wiki/) -[![Build + Publish](https://github.com/Requarks/wiki/actions/workflows/build.yml/badge.svg)](https://github.com/Requarks/wiki/actions/workflows/build.yml) -[![Huntr](https://img.shields.io/badge/security%20bounty-disclose-brightgreen.svg?style=flat&logo=cachet&logoColor=white)](https://huntr.dev/bounties/disclose) -[![GitHub Sponsors](https://img.shields.io/github/sponsors/ngpixel?logo=github&color=ea4aaa)](https://github.com/users/NGPixel/sponsorship) -[![Open Collective backers and sponsors](https://img.shields.io/opencollective/all/wikijs?label=backers&color=218bff&logo=opencollective&logoColor=white)](https://opencollective.com/wikijs) -[![Chat on Slack](https://img.shields.io/badge/slack-requarks-CC2B5E.svg?style=flat&logo=slack)](https://wiki.requarks.io/slack) -[![Follow on Twitter](https://img.shields.io/badge/twitter-%40requarks-blue.svg?style=flat&logo=twitter&logoColor=white)](https://twitter.com/requarks) -[![Follow on Telegram](https://img.shields.io/badge/telegram-%40wiki__js-blue.svg?style=flat&logo=telegram)](https://t.me/wiki_js) [![Chat on Discord](https://img.shields.io/badge/discord-join-8D96F6.svg?style=flat&logo=discord&logoColor=white)](https://discord.gg/rcxt9QS2jd) +[![Follow on Bluesky](https://img.shields.io/badge/bluesky-%40js.wiki-blue.svg?style=flat&logo=bluesky&logoColor=white)](https://bsky.app/profile/js.wiki) +[![Follow on Telegram](https://img.shields.io/badge/telegram-%40wiki__js-blue.svg?style=flat&logo=telegram)](https://t.me/wiki_js) [![Reddit](https://img.shields.io/badge/reddit-%2Fr%2Fwikijs-orange?logo=reddit&logoColor=white)](https://www.reddit.com/r/wikijs/) ##### A modern, lightweight and powerful wiki app built on NodeJS @@ -31,7 +29,7 @@ - [Demo](https://docs.requarks.io/demo) - [Changelog](https://github.com/requarks/wiki/releases) - [Feature Requests](https://feedback.js.wiki/wiki) -- [Chat with us on Slack](https://wiki.requarks.io/slack) +- Chat with us on [Discord](https://discord.gg/rcxt9QS2jd) - [Translations](https://docs.requarks.io/dev/translations) *(We need your help!)* - [E2E Testing Results](https://dashboard.cypress.io/projects/r7qxah/runs) - [Special Thanks](#special-thanks) @@ -396,6 +394,23 @@ Support this project by becoming a sponsor. Your logo will show up in the Contri + + + + + + + + + + + + + + + + + diff --git a/SECURITY.md b/SECURITY.md index a68b8b9f3..0f6a1e66e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -15,13 +15,7 @@ If you find such vulnerability, it's important to disclose it in a quick and sec **DO NOT CREATE A GITHUB ISSUE / DISCUSSION** to report a potential vulnerability / security problem. Instead, choose one of these options: -### A) Submit a Vulnerability Report *(recommended)* - -Fill in the form on https://github.com/requarks/wiki/security/advisories/new - -### B) Send an email - -Send an email to security@requarks.io. +Submit a Vulnerability Report by filling in the form on https://github.com/requarks/wiki/security/advisories/new Include as much details as possible, such as: - The version(s) of Wiki.js that are impacted diff --git a/dev/build-arm/Dockerfile b/dev/build-arm/Dockerfile index 01138056b..19342ee35 100644 --- a/dev/build-arm/Dockerfile +++ b/dev/build-arm/Dockerfile @@ -1,7 +1,7 @@ # ========================= # --- BUILD NPM MODULES --- # ========================= -FROM node:18-alpine AS build +FROM node:20-alpine AS build RUN apk add yarn g++ make cmake python3 --no-cache @@ -16,7 +16,7 @@ RUN yarn patch-package # =============== # --- Release --- # =============== -FROM node:18-alpine +FROM node:20-alpine LABEL maintainer="requarks.io" RUN apk add bash curl git openssh gnupg sqlite --no-cache && \ diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index adc5bfc75..fa95ac991 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,7 +1,7 @@ # ==================== # --- Build Assets --- # ==================== -FROM node:18-alpine AS assets +FROM node:20-alpine AS assets RUN apk add yarn g++ make cmake python3 --no-cache @@ -25,7 +25,7 @@ RUN yarn patch-package # =============== # --- Release --- # =============== -FROM node:18-alpine +FROM node:20-alpine LABEL maintainer="requarks.io" RUN apk add bash curl git openssh gnupg sqlite --no-cache && \ diff --git a/dev/helm/Chart.yaml b/dev/helm/Chart.yaml index 0b8d593a1..458defa31 100644 --- a/dev/helm/Chart.yaml +++ b/dev/helm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: wiki # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 2.2.0 +version: 2.3.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. AppVersion: latest diff --git a/dev/helm/README.md b/dev/helm/README.md index 87404d767..c1b73e227 100644 --- a/dev/helm/README.md +++ b/dev/helm/README.md @@ -97,16 +97,16 @@ The following table lists the configurable parameters of the Wiki.js chart and t | `image.repository` | Wiki.js image | `requarks/wiki` | | `image.tag` | Wiki.js image tag | `latest` | | `imagePullPolicy` | Image pull policy | `IfNotPresent` | -| `replicacount` | Amount of wiki.js service pods to run | `1` | -| `revisionHistoryLimit` | Total amount of revision history points | `10` | -| `resources.limits` | wiki.js service resource limits | `nil` | -| `resources.requests` | wiki.js service resource requests | `nil` | -| `nodeSelector` | Node labels for wiki.js pod assignment | `{}` | -| `affinity` | Affinity settings for wiki.js pod assignment | `{}` | -| `schedulerName` | Name of an alternate scheduler for wiki.js pod | `nil` | -| `tolerations` | Toleration labels for wiki.jsk pod assignment | `[]` | -| `volumeMounts` | Volume mounts for Wiki.js container | `[]` | -| `volumes` | Volumes for Wiki.js Pod | `[]` | +| `replicacount` | Number of Wiki.js pods to run | `1` | +| `revisionHistoryLimit` | Total number of revision history points | `10` | +| `resources.limits` | Wiki.js service resource limits | `nil` | +| `resources.requests` | Wiki.js service resource requests | `nil` | +| `nodeSelector` | Node labels for the Wiki.js pod assignment | `{}` | +| `affinity` | Affinity settings for the Wiki.js pod assignment | `{}` | +| `schedulerName` | Name of an alternate scheduler for the Wiki.js pod | `nil` | +| `tolerations` | Toleration labels for the Wiki.js pod assignment | `[]` | +| `volumeMounts` | Volume mounts for the Wiki.js container | `[]` | +| `volumes` | Volumes for the Wiki.js pod | `[]` | | `ingress.enabled` | Enable ingress controller resource | `false` | | `ingress.className` | Ingress class name | `""` | | `ingress.annotations` | Ingress annotations | `{}` | @@ -114,7 +114,10 @@ The following table lists the configurable parameters of the Wiki.js chart and t | `ingress.tls` | Ingress TLS configuration | `[]` | | `sideload.enabled` | Enable sideloading of locale files from git | `false` | | `sideload.repoURL` | Git repository URL containing locale files | `https://github.com/Requarks/wiki-localization` | -| `sideload.env` | Environment variables for sideload Container | `{}` | +| `sideload.env` | Environment variables for the sideload container | `{}` | +| `sideload.securityContext` | Security context for the sideload container | `nil` | +| `sideload.resources.limits` | Resource limits for the sideload container | `nil` | +| `sideload.resources.requests` | Resource requests for the sideload container | `nil` | | `nodeExtraCaCerts` | Trusted certificates path | `nil` | | `postgresql.enabled` | Deploy postgres server (see below) | `true` | | `postgresql.postgresqlDatabase` | Postgres database name | `wiki` | @@ -122,7 +125,7 @@ The following table lists the configurable parameters of the Wiki.js chart and t | `postgresql.postgresqlHost` | External postgres host | `nil` | | `postgresql.postgresqlPassword` | External postgres password | `nil` | | `postgresql.existingSecret` | Provide an existing `Secret` for postgres | `nil` | -| `postgresql.existingSecretKey` | The postgres password key in the existing `Secret` | `postgresql-password` | +| `postgresql.existingSecretKey` | The postgres password key in the existing `Secret` | `postgresql-password` | | `postgresql.postgresqlPort` | External postgres port | `5432` | | `postgresql.ssl` | Enable external postgres SSL connection | `false` | | `postgresql.ca` | Certificate of Authority content for postgres | `nil` | diff --git a/dev/helm/templates/deployment.yaml b/dev/helm/templates/deployment.yaml index 21db6a826..4a7e3b8a6 100644 --- a/dev/helm/templates/deployment.yaml +++ b/dev/helm/templates/deployment.yaml @@ -27,12 +27,16 @@ spec: {{- if .Values.sideload.enabled }} initContainers: - name: {{ .Chart.Name }}-sideload + securityContext: + {{- toYaml .Values.sideload.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} env: {{- toYaml .Values.sideload.env | nindent 12 }} command: [ "sh", "-c" ] args: [ "mkdir -p /wiki/data/sideload && git clone --depth=1 {{ .Values.sideload.repoURL }} /wiki/data/sideload/" ] + resources: + {{- toYaml .Values.sideload.resources | nindent 12 }} {{- end }} containers: - name: {{ .Chart.Name }} @@ -47,7 +51,7 @@ spec: {{- end }} - name: DB_TYPE value: postgres - {{- if (.Values.externalPostgresql).databaseURL }} + {{- if and .Values.externalPostgresql .Values.externalPostgresql.databaseURL }} - name: DATABASE_URL value: {{ .Values.externalPostgresql.databaseURL }} - name: NODE_TLS_REJECT_UNAUTHORIZED diff --git a/dev/helm/values.yaml b/dev/helm/values.yaml index 099a694c1..8e5e36670 100644 --- a/dev/helm/values.yaml +++ b/dev/helm/values.yaml @@ -115,6 +115,26 @@ sideload: # - name: HTTPS_PROXY # value: http://my.proxy.com:3128 + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + ## Append extra trusted certificates for node process from extra volume via NODE_EXTRA_CA_CERTS variable # nodeExtraCaCerts: "/path/to/certs.pem" diff --git a/package.json b/package.json index d6723534f..64c1c4d51 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,7 @@ "dotize": "0.3.0", "elasticsearch6": "npm:@elastic/elasticsearch@6", "elasticsearch7": "npm:@elastic/elasticsearch@7", + "elasticsearch8": "npm:@elastic/elasticsearch@8", "emoji-regex": "10.2.1", "eventemitter2": "6.4.9", "express": "4.18.2", @@ -336,7 +337,8 @@ }, "resolutions": { "apollo-server-express/**/graphql-tools": "4.0.8", - "graphql": "15.3.0" + "graphql": "15.3.0", + "passport-saml/**/xml-crypto": "2.1.6" }, "browserslist": [ "> 1%", diff --git a/server/core/db.js b/server/core/db.js index 25f21ad34..b729282a4 100644 --- a/server/core/db.js +++ b/server/core/db.js @@ -222,7 +222,7 @@ module.exports = { * Subscribe to database LISTEN / NOTIFY for multi-instances events */ async subscribeToNotifications () { - const useHA = (WIKI.config.ha === true || WIKI.config.ha === 'true' || WIKI.config.ha === 1 || WIKI.config.ha === '1') + const useHA = (WIKI.config.ha === true || (typeof WIKI.config.ha === 'string' && WIKI.config.ha.toLowerCase() === 'true') || WIKI.config.ha === 1 || WIKI.config.ha === '1') if (!useHA) { return } else if (WIKI.config.db.type !== 'postgres') { diff --git a/server/models/users.js b/server/models/users.js index 33f6b24fd..8191621db 100644 --- a/server/models/users.js +++ b/server/models/users.js @@ -499,6 +499,10 @@ module.exports = class User extends Model { }) if (usr) { + if (!usr.isActive) { + throw new WIKI.Error.AuthAccountBanned() + } + await WIKI.models.users.query().patch({ password: newPassword, mustChangePwd: false @@ -527,6 +531,9 @@ module.exports = class User extends Model { if (!usr) { WIKI.logger.debug(`Password reset attempt on nonexistant local account ${email}: [DISCARDED]`) return + } else if (!usr.isActive) { + WIKI.logger.debug(`Password reset attempt on disabled local account ${email}: [DISCARDED]`) + return } const resetToken = await WIKI.models.userKeys.generateToken({ userId: usr.id, diff --git a/server/modules/search/elasticsearch/definition.yml b/server/modules/search/elasticsearch/definition.yml index 8475c0b5f..514d48310 100644 --- a/server/modules/search/elasticsearch/definition.yml +++ b/server/modules/search/elasticsearch/definition.yml @@ -12,9 +12,10 @@ props: hint: Should match the version of the Elasticsearch nodes you are connecting to order: 1 enum: + - '8.x' - '7.x' - '6.x' - default: '6.x' + default: '7.x' hosts: type: String title: Host(s) diff --git a/server/modules/search/elasticsearch/engine.js b/server/modules/search/elasticsearch/engine.js index 4a96b2bbf..0daf00374 100644 --- a/server/modules/search/elasticsearch/engine.js +++ b/server/modules/search/elasticsearch/engine.js @@ -19,6 +19,16 @@ module.exports = { async init() { WIKI.logger.info(`(SEARCH/ELASTICSEARCH) Initializing...`) switch (this.config.apiVersion) { + case '8.x': + const { Client: Client8 } = require('elasticsearch8') + this.client = new Client8({ + nodes: this.config.hosts.split(',').map(_.trim), + sniffOnStart: this.config.sniffOnStart, + sniffInterval: (this.config.sniffInterval > 0) ? this.config.sniffInterval : false, + tls: getTlsOptions(this.config), + name: 'wiki-js' + }) + break case '7.x': const { Client: Client7 } = require('elasticsearch7') this.client = new Client7({ @@ -54,7 +64,8 @@ module.exports = { async createIndex() { try { const indexExists = await this.client.indices.exists({ index: this.config.indexName }) - if (!indexExists.body) { + // Elasticsearch 6.x / 7.x + if (this.config.apiVersion !== '8.x' && !indexExists.body) { WIKI.logger.info(`(SEARCH/ELASTICSEARCH) Creating index...`) try { const idxBody = { @@ -68,6 +79,7 @@ module.exports = { tags: { type: 'text', boost: 8.0 } } } + await this.client.indices.create({ index: this.config.indexName, body: { @@ -88,7 +100,42 @@ module.exports = { } catch (err) { WIKI.logger.error(`(SEARCH/ELASTICSEARCH) Create Index Error: `, _.get(err, 'meta.body.error', err)) } - } + // Elasticsearch 8.x + } else if (this.config.apiVersion === '8.x' && !indexExists) { + WIKI.logger.info(`(SEARCH/ELASTICSEARCH) Creating index...`) + try { + // 8.x Doesn't support boost in mappings, so we will need to boost at query time. + const idxBody = { + properties: { + suggest: { type: 'completion' }, + title: { type: 'text' }, + description: { type: 'text' }, + content: { type: 'text' }, + locale: { type: 'keyword' }, + path: { type: 'text' }, + tags: { type: 'text' } + } + } + + await this.client.indices.create({ + index: this.config.indexName, + body: { + mappings: idxBody, + settings: { + analysis: { + analyzer: { + default: { + type: this.config.analyzer + } + } + } + } + } + }) + } catch (err) { + WIKI.logger.error(`(SEARCH/ELASTICSEARCH) Create Index Error: `, _.get(err, 'meta.body.error', err)) + } + } } catch (err) { WIKI.logger.error(`(SEARCH/ELASTICSEARCH) Index Check Error: `, _.get(err, 'meta.body.error', err)) } @@ -129,7 +176,7 @@ module.exports = { } }) return { - results: _.get(results, 'body.hits.hits', []).map(r => ({ + results: _.get(results, this.config.apiVersion === '8.x' ? 'hits.hits' : 'body.hits.hits', []).map(r => ({ id: r._id, locale: r._source.locale, path: r._source.path, @@ -137,7 +184,7 @@ module.exports = { description: r._source.description })), suggestions: _.reject(_.get(results, 'suggest.suggestions', []).map(s => _.get(s, 'options[0].text', false)), s => !s), - totalHits: _.get(results, 'body.hits.total.value', _.get(results, 'body.hits.total', 0)) + totalHits: _.get(results, this.config.apiVersion === '8.x' ? 'hits.total.value' : 'body.hits.total.value', _.get(results, this.config.apiVersion === '8.x' ? 'hits.total' : 'body.hits.total', 0)) } } catch (err) { WIKI.logger.warn('Search Engine Error: ', _.get(err, 'meta.body.error', err)) @@ -182,7 +229,7 @@ module.exports = { async created(page) { await this.client.index({ index: this.config.indexName, - type: '_doc', + ...(this.config.apiVersion !== '8.x' && { type: '_doc' }), id: page.hash, body: { suggest: this.buildSuggest(page), @@ -204,7 +251,7 @@ module.exports = { async updated(page) { await this.client.index({ index: this.config.indexName, - type: '_doc', + ...(this.config.apiVersion !== '8.x' && { type: '_doc' }), id: page.hash, body: { suggest: this.buildSuggest(page), @@ -226,7 +273,7 @@ module.exports = { async deleted(page) { await this.client.delete({ index: this.config.indexName, - type: '_doc', + ...(this.config.apiVersion !== '8.x' && { type: '_doc' }), id: page.hash, refresh: true }) @@ -239,13 +286,13 @@ module.exports = { async renamed(page) { await this.client.delete({ index: this.config.indexName, - type: '_doc', + ...(this.config.apiVersion !== '8.x' && { type: '_doc' }), id: page.hash, refresh: true }) await this.client.index({ index: this.config.indexName, - type: '_doc', + ...(this.config.apiVersion !== '8.x' && { type: '_doc' }), id: page.destinationHash, body: { suggest: this.buildSuggest(page), @@ -314,8 +361,8 @@ module.exports = { result.push({ index: { _index: this.config.indexName, - _type: '_doc', - _id: doc.id + _id: doc.id, + ...(this.config.apiVersion !== '8.x' && { _type: '_doc' }) } }) doc.safeContent = WIKI.models.pages.cleanHTML(doc.render) diff --git a/server/modules/storage/git/definition.yml b/server/modules/storage/git/definition.yml index 63c095a22..bea279ed1 100644 --- a/server/modules/storage/git/definition.yml +++ b/server/modules/storage/git/definition.yml @@ -87,6 +87,12 @@ props: default: './data/repo' hint: 'Path where the local git repository will be created.' order: 30 + alwaysNamespace: + type: Boolean + title: Always Locale Namespace + default: false + hint: 'Whether to put content from the primary language into a subfolder.' + order: 40 gitBinaryPath: type: String title: Git Binary Path diff --git a/server/modules/storage/git/storage.js b/server/modules/storage/git/storage.js index 62401b0df..fcb197bc0 100644 --- a/server/modules/storage/git/storage.js +++ b/server/modules/storage/git/storage.js @@ -298,7 +298,7 @@ module.exports = { async created(page) { WIKI.logger.info(`(STORAGE/GIT) Committing new file [${page.localeCode}] ${page.path}...`) let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}` - if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { + if (this.config.alwaysNamespace || (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode)) { fileName = `${page.localeCode}/${fileName}` } const filePath = path.join(this.repoPath, fileName) @@ -320,7 +320,7 @@ module.exports = { async updated(page) { WIKI.logger.info(`(STORAGE/GIT) Committing updated file [${page.localeCode}] ${page.path}...`) let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}` - if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { + if (this.config.alwaysNamespace || (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode)) { fileName = `${page.localeCode}/${fileName}` } const filePath = path.join(this.repoPath, fileName) @@ -342,7 +342,7 @@ module.exports = { async deleted(page) { WIKI.logger.info(`(STORAGE/GIT) Committing removed file [${page.localeCode}] ${page.path}...`) let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}` - if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { + if (this.config.alwaysNamespace || (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode)) { fileName = `${page.localeCode}/${fileName}` } @@ -364,11 +364,11 @@ module.exports = { let sourceFileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}` let destinationFileName = `${page.destinationPath}.${pageHelper.getFileExtension(page.contentType)}` - if (WIKI.config.lang.namespacing) { - if (WIKI.config.lang.code !== page.localeCode) { + if (this.config.alwaysNamespace || WIKI.config.lang.namespacing) { + if (this.config.alwaysNamespace || WIKI.config.lang.code !== page.localeCode) { sourceFileName = `${page.localeCode}/${sourceFileName}` } - if (WIKI.config.lang.code !== page.destinationLocaleCode) { + if (this.config.alwaysNamespace || WIKI.config.lang.code !== page.destinationLocaleCode) { destinationFileName = `${page.destinationLocaleCode}/${destinationFileName}` } } @@ -483,7 +483,7 @@ module.exports = { page.tags = await pageObject.$relatedQuery('tags') let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}` - if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { + if (this.config.alwaysNamespace || (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode)) { fileName = `${page.localeCode}/${fileName}` } WIKI.logger.info(`(STORAGE/GIT) Adding page ${fileName}...`) diff --git a/yarn.lock b/yarn.lock index 6ac6a6f4f..17e8fbd59 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2782,6 +2782,19 @@ enabled "2.0.x" kuler "^2.0.0" +"@elastic/transport@^8.7.0": + version "8.8.1" + resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.8.1.tgz#d64244907bccdad5626c860b492faeef12194b1f" + integrity sha512-4RQIiChwNIx3B0O+2JdmTq/Qobj6+1g2RQnSv1gt4V2SVfAYjGwOKu0ZMKEHQOXYNG6+j/Chero2G9k3/wXLEw== + dependencies: + "@opentelemetry/api" "1.x" + debug "^4.3.4" + hpagent "^1.0.0" + ms "^2.1.3" + secure-json-parse "^2.4.0" + tslib "^2.4.0" + undici "^6.12.0" + "@eslint/eslintrc@^0.2.0": version "0.2.0" resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.2.0.tgz#bc7e3c4304d4c8720968ccaee793087dfb5fe6b4" @@ -3339,6 +3352,11 @@ dependencies: mkdirp "^1.0.4" +"@opentelemetry/api@1.x": + version "1.9.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" + integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== + "@opentelemetry/api@^1.0.1": version "1.1.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.1.0.tgz#563539048255bbe1a5f4f586a4a10a1bb737f44a" @@ -4342,6 +4360,11 @@ resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.7.7.tgz#16bd8a4e5c953018b8168e5d0a7d26b117cd7fa9" integrity sha512-RwEdIYho2kjbSZ7fpvhkHy5wk1Y3x0O6e/EHL3/SoiAfFWH+yhV2/XZQvsBoAeGRNFwgScJS/gRZv+uIwoj7yA== +"@xmldom/xmldom@^0.7.9": + version "0.7.13" + resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.7.13.tgz#ff34942667a4e19a9f4a0996a76814daac364cf3" + integrity sha512-lm2GW5PkosIzccsaZIz7tp8cPADSIlIHWDFTR1N0SzfinhhYgeIQjFMz4rYzanCScr3DqQLeomUDArp6MWKm+g== + "@xtuc/ieee754@^1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790" @@ -8753,6 +8776,14 @@ ejs@^2.3.4, ejs@^2.6.1: pump "^3.0.0" secure-json-parse "^2.1.0" +"elasticsearch8@npm:@elastic/elasticsearch@8": + version "8.15.0" + resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-8.15.0.tgz#cb29b3ae33203c545d435cf3dc4b557c8b4961d5" + integrity sha512-mG90EMdTDoT6GFSdqpUAhWK9LGuiJo6tOWqs0Usd/t15mPQDj7ZqHXfCBqNkASZpwPZpbAYVjd57S6nbUBINCg== + dependencies: + "@elastic/transport" "^8.7.0" + tslib "^2.4.0" + electron-to-chromium@^1.3.247, electron-to-chromium@^1.3.30: version "1.3.255" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.255.tgz#9f4b4f9ffd559dd7a81fef962186fc918d118b66" @@ -10842,6 +10873,11 @@ hosted-git-info@^2.1.4: resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.4.tgz#44119abaf4bc64692a16ace34700fed9c03e2546" integrity sha512-pzXIvANXEFrc5oFFXRMkbLPQ2rXRoDERwDLyrcUxGhaZhgP54BBSl9Oheh7Vv0T090cszWBxPjkQQ5Sq1PbBRQ== +hpagent@^1.0.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-1.2.0.tgz#0ae417895430eb3770c03443456b8d90ca464903" + integrity sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA== + hsl-regex@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/hsl-regex/-/hsl-regex-1.0.0.tgz#d49330c789ed819e276a4c0d272dffa30b18fe6e" @@ -13753,7 +13789,7 @@ ms@2.1.2, ms@^2.1.1: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3, ms@^2.0.0: +ms@2.1.3, ms@^2.0.0, ms@^2.1.3: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -17907,6 +17943,11 @@ secure-json-parse@^2.1.0: resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.1.0.tgz#ae76f5624256b5c497af887090a5d9e156c9fb20" integrity sha512-GckO+MS/wT4UogDyoI/H/S1L0MCcKS1XX/vp48wfmU7Nw4woBmb8mIpu4zPBQjKlRT88/bt9xdoV4111jPpNJA== +secure-json-parse@^2.4.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.7.0.tgz#5a5f9cd6ae47df23dba3151edd06855d47e09862" + integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw== + select@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/select/-/select-1.1.2.tgz#0e7350acdec80b1108528786ec1d4418d11b396d" @@ -19382,6 +19423,11 @@ tslib@^2.2.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== +tslib@^2.4.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.7.0.tgz#d9b40c5c40ab59e8738f297df3087bf1a2690c01" + integrity sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA== + tslib@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.1.tgz#410eb0d113e5b6356490eec749603725b021b43e" @@ -19550,6 +19596,11 @@ underscore@~1.8.3: resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022" integrity sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI= +undici@^6.12.0: + version "6.20.0" + resolved "https://registry.yarnpkg.com/undici/-/undici-6.20.0.tgz#3b94d967693759ea625a3b78b2097213f30405a1" + integrity sha512-AITZfPuxubm31Sx0vr8bteSalEbs9wQb/BOBi9FPlD9Qpd6HxZ4Q0+hI742jBhkPb4RT2v5MQzaW5VhRVyj+9A== + unicode-canonical-property-names-ecmascript@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818" @@ -20589,12 +20640,12 @@ ws@^7.2.3: resolved "https://registry.yarnpkg.com/ws/-/ws-7.2.5.tgz#abb1370d4626a5a9cd79d8de404aa18b3465d10d" integrity sha512-C34cIU4+DB2vMyAbmEKossWq2ZQDr6QEyuuCzWrM9zfw1sGc0mYiJ0UnG9zzNykt49C2Fi34hvr2vssFQRS6EA== -xml-crypto@^2.1.3: - version "2.1.3" - resolved "https://registry.yarnpkg.com/xml-crypto/-/xml-crypto-2.1.3.tgz#6a7272b610ea3e4ea7f13e9e4876f1b20cbc32c8" - integrity sha512-MpXZwnn9JK0mNPZ5mnFIbNnQa+8lMGK4NtnX2FlJMfMWR60sJdFO9X72yO6ji068pxixzk53O7x0/iSKh6IhyQ== +xml-crypto@2.1.6, xml-crypto@^2.1.3: + version "2.1.6" + resolved "https://registry.yarnpkg.com/xml-crypto/-/xml-crypto-2.1.6.tgz#c51a016cc8391fc1d9ebd9abc589e4c08b62d652" + integrity sha512-jjvpO8vHNV8QFhW5bMypP+k4BjBqHe/HrpIwpPcdUnUTIJakSIuN96o3Sdah4tKu2z64kM/JHEH8iEHGCc6Gyw== dependencies: - "@xmldom/xmldom" "^0.7.0" + "@xmldom/xmldom" "^0.7.9" xpath "0.0.32" xml-encryption@^2.0.0: