diff --git a/.gitignore b/.gitignore index fd6e6c12f0..d016351234 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ # build output dist/ +# reference-only build stash (npm run build:reference-only) +.reference-only-stash/ # generated types .astro/ diff --git a/astro.config.mjs b/astro.config.mjs index c4b04d47c3..528114d728 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -18,6 +18,10 @@ if (shouldSkipCompress) { const isA11yTest = Boolean(process.env.A11Y_TEST); +// Reference-only preview builds (processing/p5.js#9303) skip the service +// worker: it globs and precaches the whole dist, which a preview doesn't need. +const isReferenceOnly = Boolean(process.env.REFERENCE_ONLY); + // https://astro.build/config export default defineConfig({ site: 'https://p5js.org', @@ -32,27 +36,31 @@ export default defineConfig({ }), mdx(), fast(), - serviceWorker({ - workbox: { - globPatterns: [ - "**/*.{css,js,jpg,json,png,svg,ico,woff,woff2}", // Cache all assets accept HTML - ], - clientsClaim: true, - runtimeCaching: [ - { - urlPattern: ({ url }) => url.pathname.endsWith(".html"), // Caches HTML pages - handler: "CacheFirst", // Tries the cache first, then falls back to network if offline - options: { - cacheName: "html-pages-cache", - expiration: { - maxEntries: 50, // Limits the number of HTML pages cached - maxAgeSeconds: 24 * 60 * 60 * 7, // Cache for 1 week - }, + ...(isReferenceOnly + ? [] + : [ + serviceWorker({ + workbox: { + globPatterns: [ + "**/*.{css,js,jpg,json,png,svg,ico,woff,woff2}", // Cache all assets accept HTML + ], + clientsClaim: true, + runtimeCaching: [ + { + urlPattern: ({ url }) => url.pathname.endsWith(".html"), // Caches HTML pages + handler: "CacheFirst", // Tries the cache first, then falls back to network if offline + options: { + cacheName: "html-pages-cache", + expiration: { + maxEntries: 50, // Limits the number of HTML pages cached + maxAgeSeconds: 24 * 60 * 60 * 7, // Cache for 1 week + }, + }, + }, + ], }, - }, - ], - }, - }), + }), + ]), ], prefetch: { defaultStrategy: "viewport", diff --git a/docs/scripts.md b/docs/scripts.md index c119ed2c36..55325d3328 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -56,6 +56,20 @@ When you're done, you can run this command to reset your changes: npm run custom:cleanup ``` +## Building the reference from a pre-built data.json + +`npm run build:reference` normally clones p5.js and runs its docs build to produce the YUIDoc `data.json` it works from. If you already have that file — for example from running `npm run docs` in a p5.js checkout, or from a CI artifact — you can hand it over directly with the `P5_DATA_JSON_PATH` environment variable, which skips the clone and the docs build entirely: + +```sh +P5_DATA_JSON_PATH=../p5.js/docs/reference/data.json npm run build:reference +``` + +This is mainly useful in CI, where the p5.js build that produced the `data.json` has already run. + +## Reference-only site build + +`npm run build:reference-only` builds only the English reference pages (`/reference/**`); every other page 404s. It works by temporarily moving the non-reference routes out of `src/pages`, running `astro build`, and restoring them afterwards. Setting `REFERENCE_SKIP_LOWERCASE=1` additionally skips the lowercase alias of each reference route (for example `/reference/p5/loadimage/` alongside `/reference/p5/loadImage/`), which nearly halves the page count. This build is what the p5.js PR preview uses; combined with `P5_DATA_JSON_PATH` above it produces a reference preview without cloning or building p5.js. + ## After a p5.js release To update the content on the p5.js website following a new release of p5.js, you should run all of these commands to pull in the latest content. **Be sure to run the search indices script last as it depends on the results of the other scripts to be accurate.** diff --git a/package.json b/package.json index fe0122165a..736712e1fa 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "build:contributor-docs": "tsx ./src/scripts/builders/contribute.ts", "build:contributors": "tsx ./src/scripts/builders/people.ts", "build:reference": "tsx ./src/scripts/builders/reference.ts", + "build:reference-only": "tsx ./src/scripts/referenceOnlyBuild.ts", "build:search": "tsx ./src/scripts/builders/search.ts", "build:p5-version": "tsx ./src/scripts/p5-version.ts", "custom:dev": "tsx ./src/scripts/branchTest.ts", diff --git a/src/pages/reference/[...id].astro b/src/pages/reference/[...id].astro index 7f7d55eb46..55cca57988 100644 --- a/src/pages/reference/[...id].astro +++ b/src/pages/reference/[...id].astro @@ -9,7 +9,7 @@ import { export async function getStaticPaths() { const referenceEntries = await getCollectionInDefaultLocale("reference"); - return referenceEntries + const paths = referenceEntries // @ts-expect-error - Casting to the reference item schema .filter((entry) => entry.module !== 'Constants') .map((entry) => ({ @@ -22,7 +22,16 @@ export async function getStaticPaths() { return relatedEntry.data.submodule === entry.data.submodule; }), }, - })) + })); + + // The lowercase alias of every route nearly doubles the reference page + // count; preview builds (processing/p5.js#9303) skip them to keep the + // output small. + if (process.env.REFERENCE_SKIP_LOWERCASE) { + return paths; + } + + return paths .reduce((acc, entry) => { acc.push(entry); acc.push(Object.assign({}, entry, { diff --git a/src/scripts/parsers/reference.ts b/src/scripts/parsers/reference.ts index ed086a6e36..4b95283923 100644 --- a/src/scripts/parsers/reference.ts +++ b/src/scripts/parsers/reference.ts @@ -30,28 +30,7 @@ export const parseLibraryReference = const useExternalP5Sound = !!process.env.P5_REPO_BRANCH || latestRelease.startsWith('v2'); - // Clone p5.js - await cloneLibraryRepo( - localPath, - process.env.P5_REPO_URL || p5RepoUrl, - process.env.P5_BRANCH || latestRelease, - { shouldFixAbsolutePathInPreprocessor: false }, - ); - - // Install dependencies and build docs in the p5 repo - await createP5Docs('p5.js', 'data-p5'); - - // If we're using a custom build of p5 instead of a public release, create - // a build and copy it to the specified path - if (process.env.PUBLIC_P5_LIBRARY_PATH) { - await createP5Build('p5.js', `../../../public${process.env.PUBLIC_P5_LIBRARY_PATH}`); - } - if (process.env.PUBLIC_P5_WEBGPU_LIBRARY_PATH) { - await fs.cp( - path.join(__dirname, 'in', 'p5.js', 'lib', 'p5.webgpu.js'), - path.join(__dirname, `../../../public${process.env.PUBLIC_P5_WEBGPU_LIBRARY_PATH}`), - ); - } + await prepareP5Data(latestRelease); // Copy the reference output so we can process it const p5Data = await getYuidocOutput('data-p5'); @@ -145,6 +124,66 @@ export const parseLibraryReference = return result; }; +/** + * Puts the p5.js YUIDoc data where getYuidocOutput('data-p5') reads it. + * + * If P5_DATA_JSON_PATH is set, that file is used directly and the p5.js + * clone, `npm install`, and `npm run docs` are all skipped. This lets a CI + * job that already built p5.js docs (e.g. a p5.js PR build) hand its + * data.json over instead of this script rebuilding it from a fresh clone. + * + * Otherwise p5.js is cloned (honoring P5_REPO_URL / P5_BRANCH) and its docs + * are built in the clone, exactly as before. + */ +export const prepareP5Data = async (latestRelease: string): Promise => { + const prebuiltDataPath = process.env.P5_DATA_JSON_PATH; + if (prebuiltDataPath) { + let stats; + try { + stats = await fs.stat(prebuiltDataPath); + } catch { + throw new Error( + `P5_DATA_JSON_PATH is set to "${prebuiltDataPath}" but no file exists there`, + ); + } + if (!stats.isFile()) { + throw new Error( + `P5_DATA_JSON_PATH is set to "${prebuiltDataPath}" but that path is not a file`, + ); + } + console.log( + `Skipping p5.js clone and docs build; using pre-built data.json from ${prebuiltDataPath}`, + ); + const dataP5Dir = path.join(yuidocOutputPath, 'data-p5'); + await fs.mkdir(dataP5Dir, { recursive: true }); + await fs.copyFile(prebuiltDataPath, path.join(dataP5Dir, 'data.json')); + return; + } + + // Clone p5.js + await cloneLibraryRepo( + localPath, + process.env.P5_REPO_URL || p5RepoUrl, + process.env.P5_BRANCH || latestRelease, + { shouldFixAbsolutePathInPreprocessor: false }, + ); + + // Install dependencies and build docs in the p5 repo + await createP5Docs('p5.js', 'data-p5'); + + // If we're using a custom build of p5 instead of a public release, create + // a build and copy it to the specified path + if (process.env.PUBLIC_P5_LIBRARY_PATH) { + await createP5Build('p5.js', `../../../public${process.env.PUBLIC_P5_LIBRARY_PATH}`); + } + if (process.env.PUBLIC_P5_WEBGPU_LIBRARY_PATH) { + await fs.cp( + path.join(__dirname, 'in', 'p5.js', 'lib', 'p5.webgpu.js'), + path.join(__dirname, `../../../public${process.env.PUBLIC_P5_WEBGPU_LIBRARY_PATH}`), + ); + } +}; + /** * Gets the parsed YUIDoc output from the saved file and parses it as JSON * returns the parsed YUIDoc output diff --git a/src/scripts/referenceOnlyBuild.ts b/src/scripts/referenceOnlyBuild.ts new file mode 100644 index 0000000000..b7401559f0 --- /dev/null +++ b/src/scripts/referenceOnlyBuild.ts @@ -0,0 +1,131 @@ +/** + * Builds only the English reference pages (/reference/**), for PR previews + * (processing/p5.js#9303). Every other route 404s in the output. + * + * Astro has no config option to exclude routes from a static build, so this + * wrapper temporarily moves every non-reference entry out of src/pages before + * running `astro build`, and restores them afterwards — also on failure and + * on SIGINT/SIGTERM. Only src/pages/reference/** plus the shared _utils + * modules (imported by layouts and components) stay in place. + * + * Run with: npm run build:reference-only + * Set REFERENCE_SKIP_LOWERCASE=1 to also skip the lowercase alias routes. + */ +import fs from "node:fs"; +import path from "node:path"; +import { spawn } from "node:child_process"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const root = path.join(__dirname, "..", ".."); +const pagesDir = path.join(root, "src", "pages"); +export const stashDir = path.join(root, ".reference-only-stash"); + +// Entries in src/pages that the reference build needs +const keep = new Set(["reference", "_utils.ts", "_utils-node.ts"]); + +/** + * A non-empty stash dir means a previous run died before restoring + * src/pages (e.g. kill -9; normal failures and Ctrl-C restore on their own). + * Refuse to run rather than mix two runs' stashes, and say how to recover. + */ +export const assertStashDirUsable = (dir: string) => { + let stashed: string[]; + try { + stashed = fs.readdirSync(dir); + } catch { + return; // no stash dir: nothing to recover + } + if (stashed.length === 0) return; + throw new Error( + `Refusing to run: ${dir} already exists and is not empty.\n` + + `A previous run was probably interrupted before it could restore ` + + `src/pages. Recover with:\n\n` + + ` mv ${path.relative(root, dir)}/* src/pages/ && rmdir ${path.relative(root, dir)}\n`, + ); +}; + +// Sync so it can also run inside a signal handler. +const restoreStash = () => { + let stashed: string[]; + try { + stashed = fs.readdirSync(stashDir); + } catch { + return; + } + for (const entry of stashed) { + fs.renameSync(path.join(stashDir, entry), path.join(pagesDir, entry)); + } + fs.rmdirSync(stashDir); + console.log("Restored non-reference pages."); +}; + +let restored = false; +const restoreOnce = () => { + if (restored) return; + restored = true; + restoreStash(); +}; + +let currentChild: ReturnType | undefined; + +const runAstroBuild = () => + new Promise((resolve) => { + // `astro build` directly rather than `npm run build`, which would also + // run `astro check` — previews don't need type checking and it costs time. + // detached puts npx and the astro process it spawns in their own process + // group, so the signal handler below can kill both; signalling only the + // npx wrapper leaves the actual build running. + const child = spawn("npx", ["astro", "build"], { + cwd: root, + stdio: "inherit", + detached: true, + env: { ...process.env, REFERENCE_ONLY: "1" }, + }); + currentChild = child; + child.on("exit", (code, signal) => resolve(signal ? 1 : (code ?? 1))); + child.on("error", (err) => { + console.error(`Failed to start astro build: ${err}`); + resolve(1); + }); + }); + +const main = async () => { + assertStashDirUsable(stashDir); + fs.mkdirSync(stashDir, { recursive: true }); + + // Kill the build's process group, restore src/pages, and exit with the + // conventional signal exit code. The child is detached, so this group-kill + // is what stops the build on Ctrl-C too. + for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.on(signal, () => { + if (currentChild?.pid) { + try { + process.kill(-currentChild.pid, signal); + } catch { + // the group already exited + } + } + restoreOnce(); + process.exit(signal === "SIGINT" ? 130 : 143); + }); + } + + try { + let moved = 0; + for (const entry of fs.readdirSync(pagesDir)) { + if (keep.has(entry)) continue; + fs.renameSync(path.join(pagesDir, entry), path.join(stashDir, entry)); + moved++; + } + console.log(`Moved ${moved} non-reference page entries aside.`); + process.exitCode = await runAstroBuild(); + } finally { + restoreOnce(); + } +}; + +// Only run when executed as a script, so tests can import the helpers. +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(); +} diff --git a/test/scripts/reference-only-build.test.ts b/test/scripts/reference-only-build.test.ts new file mode 100644 index 0000000000..0faaf6b8ec --- /dev/null +++ b/test/scripts/reference-only-build.test.ts @@ -0,0 +1,34 @@ +/** Covers the stash-dir safety check in src/scripts/referenceOnlyBuild.ts. */ +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { assertStashDirUsable } from "../../src/scripts/referenceOnlyBuild"; + +let tempDir: string; + +describe("assertStashDirUsable", () => { + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "ref-only-stash-")); + }); + + afterEach(async () => { + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + test("refuses a non-empty stash dir and says how to recover", async () => { + // The leftovers of a run that was killed before restoring src/pages; + // starting a new run over them would mix two stashes. + await fs.writeFile(path.join(tempDir, "about.astro"), ""); + + expect(() => assertStashDirUsable(tempDir)).toThrow(/Refusing to run/); + expect(() => assertStashDirUsable(tempDir)).toThrow(/mv .*src\/pages/); + }); + + test("accepts a missing or empty stash dir", async () => { + expect(() => + assertStashDirUsable(path.join(tempDir, "does-not-exist")), + ).not.toThrow(); + expect(() => assertStashDirUsable(tempDir)).not.toThrow(); + }); +}); diff --git a/test/scripts/reference-parser.test.ts b/test/scripts/reference-parser.test.ts new file mode 100644 index 0000000000..6111f4f07f --- /dev/null +++ b/test/scripts/reference-parser.test.ts @@ -0,0 +1,81 @@ +/** Covers the P5_DATA_JSON_PATH handoff in src/scripts/parsers/reference.ts. */ +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +vi.mock("../../src/scripts/utils", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + cloneLibraryRepo: vi.fn(), + }; +}); + +vi.mock("node:child_process", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + execSync: vi.fn(), + }; +}); + +import { execSync } from "node:child_process"; +import { prepareP5Data } from "../../src/scripts/parsers/reference"; +import { cloneLibraryRepo } from "../../src/scripts/utils"; + +const dataP5OutputPath = path.join( + __dirname, + "../../src/scripts/parsers/out/data-p5/data.json", +); + +let tempDir: string; +const originalEnv = process.env.P5_DATA_JSON_PATH; + +describe("prepareP5Data with P5_DATA_JSON_PATH", () => { + beforeEach(async () => { + vi.clearAllMocks(); + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "p5-data-json-")); + }); + + afterEach(async () => { + process.env.P5_DATA_JSON_PATH = originalEnv; + if (originalEnv === undefined) delete process.env.P5_DATA_JSON_PATH; + await fs.rm(tempDir, { recursive: true, force: true }); + await fs.rm(path.dirname(dataP5OutputPath), { + recursive: true, + force: true, + }); + }); + + test("uses the given file without cloning p5.js", async () => { + const fixture = path.join(tempDir, "data.json"); + const fixtureContent = JSON.stringify({ classes: { p5: { name: "p5" } } }); + await fs.writeFile(fixture, fixtureContent); + process.env.P5_DATA_JSON_PATH = fixture; + + await prepareP5Data("v2.3.3"); + + expect(cloneLibraryRepo).not.toHaveBeenCalled(); + expect(execSync).not.toHaveBeenCalled(); + // getYuidocOutput('data-p5') must find the handed-over file here + expect(await fs.readFile(dataP5OutputPath, "utf8")).toBe(fixtureContent); + }); + + test("fails instead of falling back to a clone when the path is wrong", async () => { + process.env.P5_DATA_JSON_PATH = path.join(tempDir, "does-not-exist.json"); + + await expect(prepareP5Data("v2.3.3")).rejects.toThrow( + /P5_DATA_JSON_PATH.*no file exists/, + ); + expect(cloneLibraryRepo).not.toHaveBeenCalled(); + }); + + test("rejects a directory path", async () => { + process.env.P5_DATA_JSON_PATH = tempDir; + + await expect(prepareP5Data("v2.3.3")).rejects.toThrow(/not a file/); + expect(cloneLibraryRepo).not.toHaveBeenCalled(); + }); +});