diff --git a/package.json b/package.json index eb31aa5a..fa13a660 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "build:prepare": "rimraf dist && node scripts/copyShared", "build:client": "vue-tsc --noEmit -p src/client && tsc -p src/client && node scripts/copyClient", "build:node": "tsc -p src/node --noEmit && rollup --config rollup.config.ts --configPlugin esbuild", - "test": "run-p --aggregate-output test:unit test:e2e test:init", + "test": "run-s test:unit test:e2e test:init", "test:unit": "vitest run -r __tests__/unit", "test:unit:watch": "vitest -r __tests__/unit", "test:e2e": "run-s test:e2e-dev test:e2e-build", @@ -100,6 +100,7 @@ "@vueuse/core": "^10.7.1", "@vueuse/integrations": "^10.7.1", "focus-trap": "^7.5.4", + "jsdom": "^23.0.1", "mark.js": "8.11.1", "minisearch": "^6.3.0", "shikiji": "^0.9.19", @@ -139,6 +140,7 @@ "@types/debug": "^4.1.12", "@types/escape-html": "^1.0.4", "@types/fs-extra": "^11.0.4", + "@types/jsdom": "^21.1.6", "@types/lodash.template": "^4.5.3", "@types/mark.js": "^8.11.12", "@types/markdown-it-attrs": "^4.1.3", @@ -155,6 +157,7 @@ "conventional-changelog-cli": "^4.1.0", "cross-spawn": "^7.0.3", "debug": "^4.3.4", + "dom-traverse": "^0.0.1", "esbuild": "^0.19.11", "escape-html": "^1.0.3", "execa": "^8.0.1", @@ -190,6 +193,7 @@ "rollup": "^4.9.5", "rollup-plugin-dts": "^6.1.0", "rollup-plugin-esbuild": "^6.1.0", + "rpc-magic-proxy": "^2.0.6", "semver": "^7.5.4", "simple-git-hooks": "^2.9.0", "sirv": "^2.0.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 019471a9..a2d2dec1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -35,6 +35,9 @@ importers: focus-trap: specifier: ^7.5.4 version: 7.5.4 + jsdom: + specifier: ^23.0.1 + version: 23.0.1(supports-color@9.4.0) mark.js: specifier: 8.11.1 version: 8.11.1 @@ -111,6 +114,9 @@ importers: '@types/fs-extra': specifier: ^11.0.4 version: 11.0.4 + '@types/jsdom': + specifier: ^21.1.6 + version: 21.1.6 '@types/lodash.template': specifier: ^4.5.3 version: 4.5.3 @@ -159,6 +165,9 @@ importers: debug: specifier: ^4.3.4 version: 4.3.4(supports-color@9.4.0) + dom-traverse: + specifier: ^0.0.1 + version: 0.0.1 esbuild: specifier: ^0.19.11 version: 0.19.11 @@ -264,6 +273,9 @@ importers: rollup-plugin-esbuild: specifier: ^6.1.0 version: 6.1.0(esbuild@0.19.11)(rollup@4.9.5)(supports-color@9.4.0) + rpc-magic-proxy: + specifier: ^2.0.6 + version: 2.0.6 semver: specifier: ^7.5.4 version: 7.5.4 @@ -284,7 +296,7 @@ importers: version: 5.3.3 vitest: specifier: ^1.2.0 - version: 1.2.0(@types/node@20.11.0)(supports-color@9.4.0) + version: 1.2.0(@types/node@20.11.0)(jsdom@23.0.1)(supports-color@9.4.0) vue-tsc: specifier: ^1.8.27 version: 1.8.27(typescript@5.3.3) @@ -1241,6 +1253,14 @@ packages: '@types/sizzle': 2.3.8 dev: true + /@types/jsdom@21.1.6: + resolution: {integrity: sha512-/7kkMsC+/kMs7gAYmmBR9P0vGTnOoLhQhyhQJSlXGI5bzTHp6xdo0TtKWQAsz6pmSAeVqKSbqeyP6hytqr9FDw==} + dependencies: + '@types/node': 20.11.0 + '@types/tough-cookie': 4.0.5 + parse5: 7.1.2 + dev: true + /@types/jsonfile@6.1.4: resolution: {integrity: sha512-D5qGUYwjvnNNextdU59/+fI+spnwtTFmyQP0h+PfIOSkNfpU6AOICUOkm4i0OnSk+NyjdPJrxCDro0sJsWlRpQ==} dependencies: @@ -1388,6 +1408,10 @@ packages: resolution: {integrity: sha512-0vWLNK2D5MT9dg0iOo8GlKguPAU02QjmZitPEsXRuJXU/OGIOt9vT9Fc26wtYuavLxtO45v9PGleoL9Z0k1LHg==} dev: true + /@types/tough-cookie@4.0.5: + resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==} + dev: true + /@types/trusted-types@2.0.7: resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} dev: true @@ -1673,6 +1697,14 @@ packages: resolution: {integrity: sha512-qQLMr+8o0WC4FZGQTcJiKBVC59JylcPSrTtk6usvmIDFUOCKegapy1VHQwRbFMOFyb/inzUVqHs+eMYKDM1YeQ==} dev: true + /agent-base@7.1.0(supports-color@9.4.0): + resolution: {integrity: sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==} + engines: {node: '>= 14'} + dependencies: + debug: 4.3.4(supports-color@9.4.0) + transitivePeerDependencies: + - supports-color + /algoliasearch@4.22.1: resolution: {integrity: sha512-jwydKFQJKIx9kIZ8Jm44SdpigFwRGPESaxZBaHSV0XWN2yBJAOT4mT7ppvlrpA4UGzz92pqFnVKr/kaZXrcreg==} dependencies: @@ -1790,7 +1822,6 @@ packages: /asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - dev: true /available-typed-arrays@1.0.5: resolution: {integrity: sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==} @@ -2007,7 +2038,6 @@ packages: engines: {node: '>= 0.8'} dependencies: delayed-stream: 1.0.0 - dev: true /comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} @@ -2240,6 +2270,12 @@ packages: engines: {node: '>= 6'} dev: true + /cssstyle@3.0.0: + resolution: {integrity: sha512-N4u2ABATi3Qplzf0hWbVCdjenim8F3ojEXpBDF5hBpjzW182MjNGLqfmQ0SkSPeQ+V86ZXgeH8aXj6kayd4jgg==} + engines: {node: '>=14'} + dependencies: + rrweb-cssom: 0.6.0 + /csstype@3.1.3: resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} dev: false @@ -2254,6 +2290,13 @@ packages: engines: {node: '>= 12'} dev: true + /data-urls@5.0.0: + resolution: {integrity: sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==} + engines: {node: '>=18'} + dependencies: + whatwg-mimetype: 4.0.0 + whatwg-url: 14.0.0 + /de-indent@1.0.2: resolution: {integrity: sha512-e/1zu3xH5MQryN2zdVaF0OrdNLUbvWxzMbi+iNA6Bky7l1RoP8a2fIbRocyHclXt/arDrrR6lL3TqFD9pMQTsg==} dev: true @@ -2281,7 +2324,9 @@ packages: dependencies: ms: 2.1.2 supports-color: 9.4.0 - dev: true + + /decimal.js@10.4.3: + resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==} /deep-eql@4.1.3: resolution: {integrity: sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==} @@ -2334,7 +2379,6 @@ packages: /delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} - dev: true /dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} @@ -2360,6 +2404,10 @@ packages: entities: 2.2.0 dev: true + /dom-traverse@0.0.1: + resolution: {integrity: sha512-NH2OTY2d0oma7Xpf+iVgF1CJVMMi2sgrF80tqOovW62lgrl+5/C2+UyshS2crg7bJdn1bkY6iZnRwNS4WVVQTw==} + dev: true + /domelementtype@2.3.0: resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} dev: true @@ -2685,7 +2733,6 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 - dev: true /formdata-polyfill@4.0.10: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} @@ -3051,6 +3098,12 @@ packages: lru-cache: 10.1.0 dev: true + /html-encoding-sniffer@4.0.0: + resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==} + engines: {node: '>=18'} + dependencies: + whatwg-encoding: 3.1.1 + /html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} dev: true @@ -3077,11 +3130,35 @@ packages: entities: 2.2.0 dev: true + /http-proxy-agent@7.0.0(supports-color@9.4.0): + resolution: {integrity: sha512-+ZT+iBxVUQ1asugqnD6oWoRiS25AkjNfG085dKJGtGxkdwLQrMKU5wJr2bOOFAXzKcTuqq+7fZlTMgG3SRfIYQ==} + engines: {node: '>= 14'} + dependencies: + agent-base: 7.1.0(supports-color@9.4.0) + debug: 4.3.4(supports-color@9.4.0) + transitivePeerDependencies: + - supports-color + + /https-proxy-agent@7.0.2(supports-color@9.4.0): + resolution: {integrity: sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA==} + engines: {node: '>= 14'} + dependencies: + agent-base: 7.1.0(supports-color@9.4.0) + debug: 4.3.4(supports-color@9.4.0) + transitivePeerDependencies: + - supports-color + /human-signals@5.0.0: resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} engines: {node: '>=16.17.0'} dev: true + /iconv-lite@0.6.3: + resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} + engines: {node: '>=0.10.0'} + dependencies: + safer-buffer: 2.1.2 + /ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} dev: true @@ -3253,6 +3330,9 @@ packages: engines: {node: '>=12'} dev: true + /is-potential-custom-element-name@1.0.1: + resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==} + /is-reference@1.2.1: resolution: {integrity: sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ==} dependencies: @@ -3374,6 +3454,41 @@ packages: esprima: 4.0.1 dev: true + /jsdom@23.0.1(supports-color@9.4.0): + resolution: {integrity: sha512-2i27vgvlUsGEBO9+/kJQRbtqtm+191b5zAZrU/UezVmnC2dlDAFLgDYJvAEi94T4kjsRKkezEtLQTgsNEsW2lQ==} + engines: {node: '>=18'} + peerDependencies: + canvas: ^2.11.2 + peerDependenciesMeta: + canvas: + optional: true + dependencies: + cssstyle: 3.0.0 + data-urls: 5.0.0 + decimal.js: 10.4.3 + form-data: 4.0.0 + html-encoding-sniffer: 4.0.0 + http-proxy-agent: 7.0.0(supports-color@9.4.0) + https-proxy-agent: 7.0.2(supports-color@9.4.0) + is-potential-custom-element-name: 1.0.1 + nwsapi: 2.2.7 + parse5: 7.1.2 + rrweb-cssom: 0.6.0 + saxes: 6.0.0 + symbol-tree: 3.2.4 + tough-cookie: 4.1.3 + w3c-xmlserializer: 5.0.0 + webidl-conversions: 7.0.0 + whatwg-encoding: 3.1.1 + whatwg-mimetype: 4.0.0 + whatwg-url: 14.0.0 + ws: 8.16.0 + xml-name-validator: 5.0.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + /json-parse-better-errors@1.0.2: resolution: {integrity: sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==} dev: true @@ -3730,14 +3845,12 @@ packages: /mime-db@1.52.0: resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} engines: {node: '>= 0.6'} - dev: true /mime-types@2.1.35: resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} engines: {node: '>= 0.6'} dependencies: mime-db: 1.52.0 - dev: true /mime@2.6.0: resolution: {integrity: sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==} @@ -3817,7 +3930,6 @@ packages: /ms@2.1.2: resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} - dev: true /muggle-string@0.3.1: resolution: {integrity: sha512-ckmWDJjphvd/FvZawgygcUeQCxzvohjFO5RxTjj4eq8kw359gFF3E1brjfI+viLMxss5JrHTDRHZvu2/tuy0Qg==} @@ -3926,6 +4038,9 @@ packages: boolbase: 1.0.0 dev: true + /nwsapi@2.2.7: + resolution: {integrity: sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ==} + /object-inspect@1.13.1: resolution: {integrity: sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==} dev: true @@ -4066,7 +4181,6 @@ packages: resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} dependencies: entities: 4.5.0 - dev: true /path-browserify@1.0.1: resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==} @@ -4243,6 +4357,9 @@ packages: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} dev: true + /psl@1.9.0: + resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} + /punycode.js@2.3.1: resolution: {integrity: sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==} engines: {node: '>=6'} @@ -4251,7 +4368,9 @@ packages: /punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} - dev: true + + /querystringify@2.2.0: + resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==} /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -4382,6 +4501,9 @@ packages: unified: 11.0.4 dev: true + /requires-port@1.0.0: + resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} + /resolve-pkg-maps@1.0.0: resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} dev: true @@ -4473,6 +4595,13 @@ packages: '@rollup/rollup-win32-x64-msvc': 4.9.5 fsevents: 2.3.3 + /rpc-magic-proxy@2.0.6: + resolution: {integrity: sha512-PtJfEIwXiFg64FUXUGv6cot6X3RssVeQ6Xb7t8lHaAPRDEwvuZf6zCNtHEyxgFw3qnBf+LyoLAX9EOl0ipNvig==} + dev: true + + /rrweb-cssom@0.6.0: + resolution: {integrity: sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw==} + /run-applescript@7.0.0: resolution: {integrity: sha512-9by4Ij99JUr/MCFBUkDKLWK3G9HVXmabKz9U5MlIAIuvuzkiOicRYs8XJLxX+xahD+mLiiCYDqF9dKAgtzKP1A==} engines: {node: '>=18'} @@ -4517,10 +4646,19 @@ packages: is-regex: 1.1.4 dev: true + /safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + /sax@1.3.0: resolution: {integrity: sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==} dev: true + /saxes@6.0.0: + resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==} + engines: {node: '>=v12.22.7'} + dependencies: + xmlchars: 2.2.0 + /section-matter@1.0.0: resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==} engines: {node: '>=4'} @@ -4896,13 +5034,15 @@ packages: /supports-color@9.4.0: resolution: {integrity: sha512-VL+lNrEoIXww1coLPOmiEmK/0sGigko5COxI09KzHc2VJXJsQ37UaQ+8quuxjDeA7+KnLGTWRyOXSLLR2Wb4jw==} engines: {node: '>=12'} - dev: true /supports-preserve-symlinks-flag@1.0.0: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} engines: {node: '>= 0.4'} dev: true + /symbol-tree@3.2.4: + resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==} + /tabbable@6.2.0: resolution: {integrity: sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew==} dev: false @@ -4981,10 +5121,25 @@ packages: engines: {node: '>=6'} dev: true + /tough-cookie@4.1.3: + resolution: {integrity: sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==} + engines: {node: '>=6'} + dependencies: + psl: 1.9.0 + punycode: 2.3.1 + universalify: 0.2.0 + url-parse: 1.5.10 + /tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} dev: true + /tr46@5.0.0: + resolution: {integrity: sha512-tk2G5R2KRwBd+ZN0zaEXpmzdKyOYksXwywulIX95MBODjSzMIuQnQ3m8JxgbhnL1LeVo7lqQKsYa1O3Htl7K5g==} + engines: {node: '>=18'} + dependencies: + punycode: 2.3.1 + /trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} dev: true @@ -5156,11 +5311,21 @@ packages: unist-util-visit-parents: 6.0.1 dev: true + /universalify@0.2.0: + resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==} + engines: {node: '>= 4.0.0'} + /universalify@2.0.1: resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} engines: {node: '>= 10.0.0'} dev: true + /url-parse@1.5.10: + resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==} + dependencies: + querystringify: 2.2.0 + requires-port: 1.0.0 + /util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} dev: true @@ -5260,7 +5425,7 @@ packages: optionalDependencies: fsevents: 2.3.3 - /vitest@1.2.0(@types/node@20.11.0)(supports-color@9.4.0): + /vitest@1.2.0(@types/node@20.11.0)(jsdom@23.0.1)(supports-color@9.4.0): resolution: {integrity: sha512-Ixs5m7BjqvLHXcibkzKRQUvD/XLw0E3rvqaCMlrm/0LMsA0309ZqYvTlPzkhh81VlEyVZXFlwWnkhb6/UMtcaQ==} engines: {node: ^18.0.0 || >=20.0.0} hasBin: true @@ -5296,6 +5461,7 @@ packages: chai: 4.4.1 debug: 4.3.4(supports-color@9.4.0) execa: 8.0.1 + jsdom: 23.0.1(supports-color@9.4.0) local-pkg: 0.5.0 magic-string: 0.30.5 pathe: 1.1.2 @@ -5367,6 +5533,12 @@ packages: typescript: 5.3.3 dev: false + /w3c-xmlserializer@5.0.0: + resolution: {integrity: sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==} + engines: {node: '>=18'} + dependencies: + xml-name-validator: 5.0.0 + /wait-on@7.2.0(debug@4.3.4): resolution: {integrity: sha512-wCQcHkRazgjG5XoAq9jbTMLpNIjoSlZslrJ2+N9MxDsGEv1HnFoVjOCexL0ESva7Y9cu350j+DWADdk54s4AFQ==} engines: {node: '>=12.0.0'} @@ -5408,6 +5580,27 @@ packages: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} dev: true + /webidl-conversions@7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + + /whatwg-encoding@3.1.1: + resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==} + engines: {node: '>=18'} + dependencies: + iconv-lite: 0.6.3 + + /whatwg-mimetype@4.0.0: + resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==} + engines: {node: '>=18'} + + /whatwg-url@14.0.0: + resolution: {integrity: sha512-1lfMEm2IEr7RIV+f4lUNPOqfFL+pO+Xw3fJSqmjX9AbXcXcYOkCe1P6+9VBZB6n94af16NfZf+sSk0JCBZC9aw==} + engines: {node: '>=18'} + dependencies: + tr46: 5.0.0 + webidl-conversions: 7.0.0 + /whatwg-url@5.0.0: resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} dependencies: @@ -5499,6 +5692,25 @@ packages: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} dev: true + /ws@8.16.0: + resolution: {integrity: sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + /xml-name-validator@5.0.0: + resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==} + engines: {node: '>=18'} + + /xmlchars@2.2.0: + resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} + /xmldom-sre@0.1.31: resolution: {integrity: sha512-f9s+fUkX04BxQf+7mMWAp5zk61pciie+fFLC9hX9UVvCeJQfNHRHXpeo5MPcR0EUf57PYLdt+ZO4f3Ipk2oZUw==} engines: {node: '>=0.1'} diff --git a/src/node/build/build.ts b/src/node/build/build.ts index d6177864..09a9ae09 100644 --- a/src/node/build/build.ts +++ b/src/node/build/build.ts @@ -9,12 +9,36 @@ import { pathToFileURL } from 'url' import type { BuildOptions, Rollup } from 'vite' import { resolveConfig, type SiteConfig } from '../config' import { clearCache } from '../markdownToVue' -import { slash, type HeadConfig } from '../shared' +import { slash, type HeadConfig, type SSGContext } from '../shared' import { deserializeFunctions, serializeFunctions } from '../utils/fnSerialize' import { task } from '../utils/task' import { bundle } from './bundle' import { generateSitemap } from './generateSitemap' -import { renderPage } from './render' +import { renderPage, type RenderPageContext } from './render' +import { launchWorkers, shouldUseParallel, stopWorkers } from '../worker' +import { registerWorkload, updateContext } from '../worker' + +type RenderFn = (path: string) => Promise + +// Worker: workload functions will be called with `this` context +export interface WorkerContext { + config: SiteConfig + options: BuildOptions +} + +// Worker proxy (worker thread) +const dispatchRenderPageWork = registerWorkload( + 'build:render-page', + function (page: string) { + return renderPage(this.render, page, this) + }, + async function init( + this: WorkerContext & + RenderPageContext & { render: RenderFn; renderEntry: string } + ) { + this.render = (await import(this.renderEntry)).render as RenderFn + } +) export async function build( root?: string, @@ -26,6 +50,13 @@ export async function build( const siteConfig = await resolveConfig(root, 'build', 'production') const unlinkVue = linkVue() + if (shouldUseParallel(siteConfig)) { + launchWorkers(siteConfig.concurrency, { + config: siteConfig, + options: buildOptions + }) + } + if (buildOptions.base) { siteConfig.site.base = buildOptions.base delete buildOptions.base @@ -51,12 +82,12 @@ export async function build( return } - const entryPath = path.join(siteConfig.tempDir, 'app.js') - const { render } = await import( - pathToFileURL(entryPath).toString() + '?t=' + Date.now() - ) - await task('rendering pages', async () => { + const renderEntry = + pathToFileURL(path.join(siteConfig.tempDir, 'app.js')).toString() + + '?t=' + + Date.now() + const appChunk = clientResult && (clientResult.output.find( @@ -110,24 +141,32 @@ export async function build( } } - await pMap( - ['404.md', ...siteConfig.pages], - async (page) => { - await renderPage( - render, - siteConfig, - siteConfig.rewrites.map[page] || page, - clientResult, - appChunk, - cssChunk, - assets, - pageToHashMap, - metadataScript, - additionalHeadTags - ) - }, - { concurrency: siteConfig.buildConcurrency } - ) + const context: RenderPageContext = { + config: siteConfig, + result: clientResult, + appChunk, + cssChunk, + assets, + pageToHashMap, + metadataScript, + additionalHeadTags + } + + let task: (page: string) => Promise + + if (shouldUseParallel(siteConfig, 'render')) { + const { config, ...additionalContext } = context + await updateContext({ renderEntry, ...additionalContext }) + task = (page) => dispatchRenderPageWork(page) + } else { + const { render } = await import(renderEntry) + task = (page) => renderPage(render, page, context) + } + + const pages = ['404.md', ...siteConfig.pages] + await pMap(pages, task, { + concurrency: siteConfig.concurrency + }) }) // emit page hash map for the case where a user session is open @@ -144,7 +183,7 @@ export async function build( await generateSitemap(siteConfig) await siteConfig.buildEnd?.(siteConfig) clearCache() - + stopWorkers('build complete') siteConfig.logger.info( `build complete in ${((Date.now() - start) / 1000).toFixed(2)}s.` ) diff --git a/src/node/build/render.ts b/src/node/build/render.ts index 7aacff68..0182a8d8 100644 --- a/src/node/build/render.ts +++ b/src/node/build/render.ts @@ -19,18 +19,34 @@ import { } from '../shared' import { version } from '../../../package.json' +export interface RenderPageContext { + config: SiteConfig + result: Rollup.RollupOutput | null + appChunk: Rollup.OutputChunk | null + cssChunk: Rollup.OutputAsset | null + assets: string[] + pageToHashMap: Record + metadataScript: { html: string; inHead: boolean } + additionalHeadTags: HeadConfig[] +} + export async function renderPage( render: (path: string) => Promise, - config: SiteConfig, - page: string, // foo.md - result: Rollup.RollupOutput | null, - appChunk: Rollup.OutputChunk | null, - cssChunk: Rollup.OutputAsset | null, - assets: string[], - pageToHashMap: Record, - metadataScript: { html: string; inHead: boolean }, - additionalHeadTags: HeadConfig[] + page: string, + renderContext: RenderPageContext ) { + const { + config, + result, + appChunk, + cssChunk, + assets, + pageToHashMap, + metadataScript, + additionalHeadTags + } = renderContext + + page = config.rewrites.inv[page] ?? page const routePath = `/${page.replace(/\.md$/, '')}` const siteData = resolveSiteDataByRoute(config.site, routePath) diff --git a/src/node/config.ts b/src/node/config.ts index 6d1bbd49..8d71aeb8 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -19,6 +19,7 @@ import { type SiteData } from './shared' import type { RawConfigExports, SiteConfig, UserConfig } from './siteConfig' +import { cpus } from 'os' export { resolvePages } from './plugins/dynamicRoutesPlugin' export * from './siteConfig' @@ -143,7 +144,11 @@ export async function resolveConfig( rewrites, userConfig, sitemap: userConfig.sitemap, - buildConcurrency: userConfig.buildConcurrency ?? 64 + concurrency: Math.max( + userConfig.concurrency ?? Math.round(cpus().length / 1.5), + 1 // At least one thread required + ), + parallel: userConfig.parallel ?? ['render', 'local-search'] } // to be shared with content loaders diff --git a/src/node/plugins/localSearchPlugin.ts b/src/node/plugins/localSearchPlugin.ts index e94eafb7..5a62fb49 100644 --- a/src/node/plugins/localSearchPlugin.ts +++ b/src/node/plugins/localSearchPlugin.ts @@ -5,14 +5,19 @@ import pMap from 'p-map' import path from 'path' import type { Plugin, ViteDevServer } from 'vite' import type { SiteConfig } from '../config' -import { createMarkdownRenderer } from '../markdown/markdown' +import { + createMarkdownRenderer, + type MarkdownRenderer +} from '../markdown/markdown' import { resolveSiteDataByRoute, slash, type DefaultTheme, - type MarkdownEnv + type MarkdownEnv, + type Awaitable } from '../shared' import { processIncludes } from '../utils/processIncludes' +import type { PageSplitSection } from '../../../types/local-search' const debug = _debug('vitepress:local-search') @@ -26,6 +31,28 @@ interface IndexObject { titles: string[] } +// SSR being `true` or `false` dose not affect the config related to markdown +// renderer, so it can be reused. +let md: MarkdownRenderer + +let flagScanned = false +const taskByLocale = new Map>() +const filesByLocale = new Map>() +const indexByLocale = new Map>() + +function getIndexByLocale(locale: string, options?: any) { + let index = indexByLocale.get(locale) + if (!index) { + index = new MiniSearch({ + fields: ['title', 'titles', 'text'], + storeFields: ['title', 'titles'], + ...options + }) + indexByLocale.set(locale, index) + } + return index +} + export async function localSearchPlugin( siteConfig: SiteConfig ): Promise { @@ -45,7 +72,7 @@ export async function localSearchPlugin( } } - const md = await createMarkdownRenderer( + md ??= await createMarkdownRenderer( siteConfig.srcDir, siteConfig.markdown, siteConfig.site.base, @@ -68,21 +95,6 @@ export async function localSearchPlugin( } } - const indexByLocales = new Map>() - - function getIndexByLocale(locale: string) { - let index = indexByLocales.get(locale) - if (!index) { - index = new MiniSearch({ - fields: ['title', 'titles', 'text'], - storeFields: ['title', 'titles'], - ...options.miniSearch?.options - }) - indexByLocales.set(locale, index) - } - return index - } - function getLocaleForPath(file: string) { const relativePath = slash(path.relative(siteConfig.srcDir, file)) const siteData = resolveSiteDataByRoute(siteConfig.site, relativePath) @@ -122,19 +134,19 @@ export async function localSearchPlugin( return id } - async function indexFile(page: string) { - const file = path.join(siteConfig.srcDir, page) - // get file metadata + async function indexFile(file: string, parallel: boolean = false) { const fileId = getDocId(file) const locale = getLocaleForPath(file) - const index = getIndexByLocale(locale) + const index = getIndexByLocale(locale, options.miniSearch?.options) // retrieve file and split into "sections" const html = await render(file) const sections = // user provided generator (await options.miniSearch?._splitIntoSections?.(file, html)) ?? + // default implementation (parallel) + (parallel ? parallelSplitter(html, fileId) : undefined) ?? // default implementation - splitPageIntoSections(html) + splitPageIntoSections(html, fileId) // add sections to the locale index for await (const section of sections) { if (!section || !(section.text || section.titles)) break @@ -149,14 +161,28 @@ export async function localSearchPlugin( } } - async function scanForBuild() { - debug('🔍️ Indexing files for search...') - await pMap(siteConfig.pages, indexFile, { - concurrency: siteConfig.buildConcurrency + // scan all pages, group by locale and create empty indexes accordingly. + function scanForIndex() { + if (flagScanned) return + flagScanned = true + + for (const page of siteConfig.pages) { + const file = path.join(siteConfig.srcDir, page) + const locale = getLocaleForPath(file) + if (!filesByLocale.has(locale)) filesByLocale.set(locale, new Set()) + filesByLocale.get(locale)!.add(file) + } + } + + async function indexLocale(locale: string) { + const files = [...filesByLocale.get(locale)!] + await pMap(files, (f) => indexFile(f, parallel), { + concurrency: siteConfig.concurrency }) - debug('✅ Indexing finished...') } + const parallel = shouldUseParallel(siteConfig, 'local-search') + return { name: 'vitepress:local-search', @@ -172,7 +198,6 @@ export async function localSearchPlugin( async configureServer(_server) { server = _server - await scanForBuild() onIndexUpdated() }, @@ -184,32 +209,28 @@ export async function localSearchPlugin( async load(id) { if (id === LOCAL_SEARCH_INDEX_REQUEST_PATH) { - if (process.env.NODE_ENV === 'production') { - await scanForBuild() - } + scanForIndex() let records: string[] = [] - for (const [locale] of indexByLocales) { + for (const [locale] of filesByLocale) { records.push( `${JSON.stringify( locale - )}: () => import('@localSearchIndex${locale}')` + )}: () => import('${LOCAL_SEARCH_INDEX_ID}:${locale}')` ) } return `export default {${records.join(',')}}` } else if (id.startsWith(LOCAL_SEARCH_INDEX_REQUEST_PATH)) { + const locale = id.slice(LOCAL_SEARCH_INDEX_REQUEST_PATH.length + 1) + await ((taskByLocale as any)[locale] ??= indexLocale(locale)) return `export default ${JSON.stringify( - JSON.stringify( - indexByLocales.get( - id.replace(LOCAL_SEARCH_INDEX_REQUEST_PATH, '') - ) ?? {} - ) + JSON.stringify(indexByLocale.get(locale) ?? {}) )}` } }, async handleHotUpdate({ file }) { if (file.endsWith('.md')) { - await indexFile(file) + await indexFile(file, parallel) debug('🔍️ Updated', file) onIndexUpdated() } @@ -217,40 +238,87 @@ export async function localSearchPlugin( } } -const headingRegex = /(.*?.*?<\/a>)<\/h\1>/gi -const headingContentRegex = /(.*?).*?<\/a>/i - -/** - * Splits HTML into sections based on headings - */ -function* splitPageIntoSections(html: string) { - const result = html.split(headingRegex) - result.shift() - let parentTitles: string[] = [] - for (let i = 0; i < result.length; i += 3) { - const level = parseInt(result[i]) - 1 - const heading = result[i + 1] - const headingResult = headingContentRegex.exec(heading) - const title = clearHtmlTags(headingResult?.[1] ?? '').trim() - const anchor = headingResult?.[2] ?? '' - const content = result[i + 2] - if (!title || !content) continue - const titles = parentTitles.slice(0, level) - titles[level] = title - yield { anchor, titles, text: getSearchableText(content) } - if (level === 0) { - parentTitles = [title] - } else { - parentTitles[level] = title +async function* splitPageIntoSections( + html: string, + pageName: string = 'Unknown Document' +) { + const { JSDOM } = await import('jsdom') + const { default: traverse, Node } = await import('dom-traverse') + const dom = JSDOM.fragment(html) + // Stack of title hierarchy for current working section + const titleStack: Array<{ level: number; text: string }> = [] + // Set of all used ids (for duplicate id detection) + const existingIdSet = new Set() + // Current working section + let section: PageSplitSection = { text: '', titles: [''] } + // Traverse the DOM + for (const [node, skipChildren] of traverse.skippable(dom)) { + if (node.nodeType === Node.ELEMENT_NODE) { + const el = node as Element + if (!/^H\d+$/i.test(el.tagName)) continue + if (!el.hasAttribute('id')) continue + const id = el.getAttribute('id')! + // Skip duplicate id, content will be treated as normal text + if (existingIdSet.has(id)) { + console.error( + `\x1b[2K\r ⚠️ Duplicate heading id "${id}" in ${pageName}` + ) + continue + } + existingIdSet.add(id) + // Submit previous section + if (section.text || section.anchor) yield section + // Pop adjacent titles depending on level + const level = parseInt(el.tagName.slice(1)) + while (titleStack.length > 0) { + if (titleStack.at(-1)!.level >= level) titleStack.pop() + else break + } + titleStack.push({ level, text: el.textContent ?? '' }) + // Create new section + section = { + text: '', + anchor: id, + titles: titleStack.map((_) => _.text) + } + skipChildren() + } else if (node.nodeType === Node.TEXT_NODE) { + // Collect text content + section.text += node.textContent } } + // Submit last section + yield section } -function getSearchableText(content: string) { - content = clearHtmlTags(content) - return content -} +/*=============================== Worker API ===============================*/ +import { registerWorkload, shouldUseParallel } from '../worker' +import Queue from '../utils/queue' -function clearHtmlTags(str: string) { - return str.replace(/<[^>]*>/g, '') +// Worker proxy (worker thread) +const dispatchPageSplitWork = registerWorkload( + 'local-search:split', + async ( + html: string, + fileId: string, + _yield: (section: PageSplitSection) => Awaitable, + _end: () => Awaitable + ) => { + for await (const section of splitPageIntoSections(html, fileId)) { + await _yield(section) + } + await _end() + } +) + +// Worker proxy (main thread) +function parallelSplitter(html: string, fileId: string) { + const queue = new Queue() + dispatchPageSplitWork( + html, + fileId, + queue.enqueue.bind(queue), + queue.close.bind(queue) + ) + return queue.items() } diff --git a/src/node/server.ts b/src/node/server.ts index 4105edfe..953d7daf 100644 --- a/src/node/server.ts +++ b/src/node/server.ts @@ -1,6 +1,7 @@ import { createServer as createViteServer, type ServerOptions } from 'vite' import { resolveConfig } from './config' import { createVitePressPlugin } from './plugin' +import { launchWorkers, stopWorkers, shouldUseParallel } from './worker' export async function createServer( root: string = process.cwd(), @@ -9,6 +10,9 @@ export async function createServer( ) { const config = await resolveConfig(root) + if (shouldUseParallel(config)) + launchWorkers(config.concurrency, { config: config }) + if (serverOptions.base) { config.site.base = serverOptions.base delete serverOptions.base @@ -22,5 +26,12 @@ export async function createServer( server: serverOptions, customLogger: config.logger, configFile: config.vite?.configFile - }) + }).then((server) => + Object.assign({}, server, { + close() { + stopWorkers('server.close()') + return server.close() + } + }) + ) } diff --git a/src/node/siteConfig.ts b/src/node/siteConfig.ts index 74174280..518d399e 100644 --- a/src/node/siteConfig.ts +++ b/src/node/siteConfig.ts @@ -13,6 +13,7 @@ import type { SSGContext, SiteData } from './shared' +import type { SupportsParallel } from './worker' export type RawConfigExports = | Awaitable> @@ -150,11 +151,24 @@ export interface UserConfig /** * This option allows you to configure the concurrency of the build. * A lower number will reduce the memory usage but will increase the build time. + * When parallel is enabled, this option indicates the number of threads. * * @experimental - * @default 64 + * @default "Number of CPU cores available / 150%" */ - buildConcurrency?: number + concurrency?: number + + /** + * This option is the general switch for enabling parallel computing. When + * enabled, vitepress will create worker threads and distribute workload to + * them. Currently, the following features are supported: + * 1. Parallel SPA Bundling + * 2. Parallel SSR Rendering + * 3. Parallel Local Search Indexing (when using default splitter) + * @experimental + * @default ['render', 'local-search'] + */ + parallel?: boolean | SupportsParallel[] /** * @experimental @@ -249,5 +263,6 @@ export interface SiteConfig } logger: Logger userConfig: UserConfig - buildConcurrency: number + concurrency: number + parallel: boolean | SupportsParallel[] } diff --git a/src/node/utils/queue.ts b/src/node/utils/queue.ts new file mode 100644 index 00000000..dff9753e --- /dev/null +++ b/src/node/utils/queue.ts @@ -0,0 +1,37 @@ +// Asynchronous queue with a close method +export default class Queue { + private queue: Array = [] + private pending: Array<(data: T | null) => void> = [] + #closed: boolean = false + + get closed() { + return this.#closed + } + + async *items() { + while (true) { + const item = await this.dequeue() + if (item === null) break + yield item + } + } + + enqueue(data: T) { + if (this.closed) + throw new Error(`Failed to enqueue ${data}, queue already closed`) + if (data === null) return this.close() + if (this.pending.length) this.pending.shift()!(data) + else this.queue.push(data) + } + + async dequeue(): Promise { + if (this.closed) return null + if (this.queue.length) return this.queue.shift()! + return new Promise((res) => this.pending.push(res)) + } + + close() { + this.#closed = true + for (const res of this.pending) res(null) + } +} diff --git a/src/node/worker.ts b/src/node/worker.ts new file mode 100644 index 00000000..8ef29528 --- /dev/null +++ b/src/node/worker.ts @@ -0,0 +1,220 @@ +import { Worker, isMainThread, parentPort, workerData } from 'worker_threads' +import RPCContext, { + deferPromise, + type RPCContextOptions +} from 'rpc-magic-proxy' +import c from 'picocolors' +import Queue from './utils/queue' +import _debug from 'debug' +import type { SiteConfig } from 'siteConfig' + +export type SupportsParallel = 'render' | 'local-search' + +const options: RPCContextOptions = { + carryThis: false, + carrySideEffect: false +} +/** + * Checks if the given task should be run in parallel. + * If task is omitted, checks if any task should be run in parallel. + */ +export function shouldUseParallel(config: SiteConfig, task?: SupportsParallel) { + const { parallel = false } = config + if (task === undefined) + return parallel === true || (Array.isArray(parallel) && parallel.length > 0) + if (typeof parallel === 'boolean') return parallel + if (Array.isArray(parallel)) return parallel.includes(task) + throw new TypeError(`Invalid value for config.parallel: ${parallel}`) +} + +let debug = _debug('vitepress:worker:main') +const WORKER_MAGIC = 'vitepress:worker' +/*=============================== Main Thread ===============================*/ +interface WorkerTask { + name: string + argv: any[] + resolve: (retVal: any) => void + reject: (error?: any) => void +} + +// Owned by main thread, will be distributed to workers +let taskQueue: Queue | null = null + +function dispatchWork(name: string, ...argv: any[]): Promise { + if (workerMeta) { + return workerMeta.dispatchWork(name, ...argv) + } else if (taskQueue) { + const { promise, resolve, reject } = deferPromise() + taskQueue.enqueue({ name, argv, resolve, reject }) + return promise + } else { + throw new Error(`trying to dispatch ${name} before launching workers.`) + } +} + +type WorkerInstance = Worker & { + workerId: string + hooks: { + // Update worker's context + updateContext: (ctx: Object | null) => void + } +} + +const workers: Array = [] + +export async function launchWorkers(numWorkers: number, context: Object) { + debug(`launching ${numWorkers} workers`) + taskQueue = new Queue() + const allInitialized: Array> = [] + const ctx = new RPCContext(options) + const getNextTask = () => taskQueue?.dequeue() ?? null + for (let i = 0; i < numWorkers; i++) { + const workerId = (i + 1).toString().padStart(2, '0') + const { promise, resolve } = deferPromise() + const initWorkerHooks = (hooks: WorkerInstance['hooks']) => { + Object.assign(worker, { workerId, hooks }) + resolve() + } + const debug = _debug(`vitepress:worker:${workerId.padEnd(4)}`) + const payload = await ctx.serialize({ + workerMeta: { + workerId, + dispatchWork, + // Save some RPC overhead when debugger is not active + debug: debug.enabled ? debug : null + }, + initWorkerHooks, + getNextTask, + context + }) + const worker = new Worker(new URL(import.meta.url), { + workerData: { [WORKER_MAGIC]: payload } + }) as WorkerInstance + ctx.bind(worker as any) + workers.push(worker) + allInitialized.push(promise) + worker.on('error', console.error) + } + await Promise.all(allInitialized) +} + +export function updateContext(context: Object) { + return Promise.all(workers.map(({ hooks }) => hooks.updateContext(context))) +} + +// Wait for workers to finish and exit. +// Will return immediately if no worker exists. +export async function stopWorkers(reason: string = 'exit') { + debug('stopping workers:', reason) + const allClosed = workers.map((w) => + new Promise((res) => w.once('exit', () => res())).then(() => + debug(`worker:${w.workerId} confirmed exit`) + ) + ) + taskQueue?.close() + taskQueue = null + const success = await Promise.any([ + Promise.all(allClosed).then(() => true), + new Promise((res) => setTimeout(() => res(false), 1500)) + ]) + if (!success) { + debug('forcefully terminating workers') + for (const w of workers) { + try { + w.terminate() + } catch (e) {} + } + } +} + +/*============================== Worker Thread ==============================*/ + +export let workerMeta: { + workerId: string + dispatchWork: typeof dispatchWork + debug: typeof debug +} | null = null + +const registry: Map = new Map() + +export function registerWorkload( + name: string, + main: (this: T, ...args: K) => V, + init?: (this: T, ...args: void[]) => void +) { + // Only register workload in worker threads + if (!isMainThread) { + if (registry.has(name)) + throw new Error(`Workload "${name}" already registered.`) + registry.set(name, { main, init }) + } + return (...args: Parameters) => + dispatchWork(name, ...args) as Promise>> +} + +// Will keep querying next workload from main thread +async function workerMainLoop() { + const ctx = new RPCContext(options).bind(parentPort! as any) + const { + workerMeta: _workerMeta, + initWorkerHooks, + getNextTask, + context + }: { + workerMeta: typeof workerMeta + getNextTask: () => Promise + initWorkerHooks: (hooks: Object) => Promise + context: Object + } = ctx.deserialize(workerData[WORKER_MAGIC]) as any + // Set up magic proxy to main thread dispatchWork + workerMeta = _workerMeta! + if (workerMeta.debug) debug = workerMeta.debug + else debug = (() => {}) as any as typeof debug + debug(`started`) + // Upon worker initialization, report back the hooks that main thread can use + // to reach this worker. + await initWorkerHooks({ + updateContext(ctx: Object | null) { + if (ctx === null) for (const k in context) delete (context as any)[k] + else Object.assign(context, ctx) + } + }) + + let workTime = 0 + while (true) { + const task = await getNextTask() + if (task === null) break + const { name, argv, resolve, reject } = task + if (!registry.has(name)) throw new Error(`No task "${name}" registered.`) + const el = registry.get(name)! + const { main, init } = el + const timeStart = performance.now() + if (init) { + try { + await init.apply(context) + } catch (e) { + console.error(c.red(`worker: failed to init workload "${name}":`), e) + reject(e) + } finally { + el.init = undefined + } + } + try { + resolve(await main.apply(context, argv)) + } catch (e) { + console.error( + c.red(`worker:${workerMeta.workerId} error running task "${name}":`), + e + ) + reject(e) + } finally { + workTime += performance.now() - timeStart + } + } + ctx.reset() + const duration = (workTime / 1000).toFixed(2) + await debug(`stopped - total workload: ${duration}s`) +} + +if (!isMainThread && workerData?.[WORKER_MAGIC]) + workerMainLoop().then(() => process.exit())