From 720f3e3ac74d57b456d5861dd3edc9515682946e Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 4 Nov 2026 14:41:42 +0100 Subject: [PATCH 01/11] Add --skip-archive to quarto-bld update-pandoc Regenerating the Pandoc reference templates and format-extension.ts needed the full update-pandoc run, which archives the binary to S3 first. Anyone without S3 credentials could not prepare or check a Pandoc bump, so the template diff was always reviewed late, by whoever did the archival. With --skip-archive the command only regenerates templates and format-extension.ts, from the binary QUARTO_PANDOC points at, and leaves configuration untouched. QUARTO_PANDOC is required and its reported version must match the requested one: pandocBinaryPath() falls back to the bundled Pandoc when the override is missing or invalid, and format-extension.ts would then be silently regenerated from the old version. --- package/src/common/update-pandoc.ts | 167 ++++++++++++++++++++++------ 1 file changed, 134 insertions(+), 33 deletions(-) diff --git a/package/src/common/update-pandoc.ts b/package/src/common/update-pandoc.ts index 9b99c17ca6a..9efcc351f83 100644 --- a/package/src/common/update-pandoc.ts +++ b/package/src/common/update-pandoc.ts @@ -7,7 +7,7 @@ import { Command } from "cliffy/command/mod.ts"; import { join } from "../../../src/deno_ral/path.ts"; import { ensureDirSync } from "../../../src/deno_ral/fs.ts"; -import { info } from "../../../src/deno_ral/log.ts"; +import { error, info } from "../../../src/deno_ral/log.ts"; import { Configuration, @@ -33,61 +33,162 @@ import { import * as ld from "../../../src/core/lodash.ts"; +import { execProcess } from "../../../src/core/process.ts"; +import { pandocBinaryPath } from "../../../src/core/resources.ts"; + export function updatePandoc() { return new Command() .name("update-pandoc") .arguments("") .description("Updates Pandoc to the specified version") - .action(async (_args, version: string) => { + .option( + "--skip-archive", + "Phase 1 prep only: regenerate templates and format-extension.ts from a QUARTO_PANDOC-provided binary without archiving to S3, configuring the binary, or bumping the configuration file. Requires QUARTO_PANDOC to point at a Pandoc binary.", + ) + // deno-lint-ignore no-explicit-any + .action(async (options: any, version: string) => { info(`Updating Pandoc to ${version}`); const configuration = readConfiguration(); + const skipArchive = !!options.skipArchive; - // Update the configuration file - info(" updating configuration file."); - const configFilePath = join( - configuration.directoryInfo.root, - "configuration", - ); - const configText = Deno.readTextFileSync(configFilePath); - const configLines = lines(configText); - const outputLines: string[] = []; - for (const line of configLines) { - if (line.startsWith("export PANDOC=")) { - outputLines.push(`export PANDOC=${version}`); - } else { - outputLines.push(line); + if (skipArchive) { + // Phase 1 (prep): no S3, no configuration rewrite. Regenerate the + // template files and format-extension.ts against a locally supplied + // binary. This requires a real + // Pandoc binary resolvable via QUARTO_PANDOC, because + // writeVariants() shells out to it to enumerate formats/extensions. + const overridePath = Deno.env.get("QUARTO_PANDOC"); + if (!overridePath) { + error( + "--skip-archive requires the QUARTO_PANDOC environment variable to " + + `point at a Pandoc ${version} binary. Without it, writeVariants() ` + + "would regenerate src/core/pandoc/format-extension.ts from whatever " + + "Pandoc is currently configured (likely the old version), silently " + + "corrupting it. Download the target release and set QUARTO_PANDOC, e.g.\n" + + ` QUARTO_PANDOC=/path/to/pandoc-${version}/bin/pandoc \\\n` + + ` ./package/src/quarto-bld update-pandoc ${version} --skip-archive`, + ); + throw new Error("QUARTO_PANDOC is not set"); } - } - Deno.writeTextFileSync(configFilePath, outputLines.join("\n")); - const pandocDependency = pandoc(version); + // Fatal check: pandocBinaryPath() silently falls back to the bundled/ + // configured Pandoc (with only a warnOnce) if QUARTO_PANDOC is unset, + // nonexistent, or otherwise fails to resolve (src/core/resources.ts + // toolsPath()). If the resolved binary doesn't report the requested + // version, continuing would regenerate format-extension.ts from the + // wrong (likely old, bundled) Pandoc - the exact silent corruption + // this flag exists to prevent - so this must fail, not warn. + let reported: string | undefined; + try { + reported = lines( + (await execProcess({ + cmd: pandocBinaryPath(), + args: ["--version"], + stdout: "piped", + })).stdout!, + )[0]?.split(" ")[1]; + } catch (_e) { + error( + "Could not run the QUARTO_PANDOC binary to verify its version. " + + `QUARTO_PANDOC is set to "${overridePath}" - confirm it points at a ` + + "runnable Pandoc executable.", + ); + throw new Error("QUARTO_PANDOC binary is not runnable"); + } + if ( + normalizePandocVersion(reported) !== normalizePandocVersion(version) + ) { + error( + `The Pandoc binary resolved via QUARTO_PANDOC reports version ` + + `${reported}, not the requested ${version}. QUARTO_PANDOC is set to ` + + `"${overridePath}" - if that path doesn't exist or isn't a file, ` + + "pandocBinaryPath() silently fell back to the bundled/configured " + + "Pandoc, which would corrupt format-extension.ts if this continued. " + + "Fix QUARTO_PANDOC and retry.", + ); + throw new Error("QUARTO_PANDOC does not resolve to the requested version"); + } + } else { + // Update the configuration file + info(" updating configuration file."); + const configFilePath = join( + configuration.directoryInfo.root, + "configuration", + ); + const configText = Deno.readTextFileSync(configFilePath); + const configLines = lines(configText); + const outputLines: string[] = []; + for (const line of configLines) { + if (line.startsWith("export PANDOC=")) { + outputLines.push(`export PANDOC=${version}`); + } else { + outputLines.push(line); + } + } + Deno.writeTextFileSync(configFilePath, outputLines.join("\n")); + } - // Call archive-bin-deps for this file await withWorkingDir(async (workingDir) => { - await archiveBinaryDependency(pandocDependency, workingDir); + if (!skipArchive) { + const pandocDependency = pandoc(version); - // Configure this version of pandoc - await configureDependency( - pandocDependency, - join(configuration.directoryInfo.bin, "tools"), - configuration, - ); + // Archive to S3 + await archiveBinaryDependency(pandocDependency, workingDir); + + // Configure this version of pandoc + await configureDependency( + pandocDependency, + join(configuration.directoryInfo.bin, "tools"), + configuration, + ); + } - // Generate templates + // Generate templates (always: downloads Pandoc's source zip from + // GitHub, never touches S3) await writePandocTemplates(configuration, version, workingDir); - // Generate variants + // Generate variants (always: uses the resolved Pandoc binary, which + // honors QUARTO_PANDOC) await writeVariants(configuration); }); - // print the warning to complete the checklist - console.log(bgBlack(brightWhite(bold( - "\n** Remember to complete the checklist in /dev-docs/update-pandoc-checklist.md! **", - )))); + // print the completion message + if (skipArchive) { + console.log(bgBlack(brightWhite(bold( + "\n** Phase 1 (prep) complete. **" + + "\n** Templates and src/core/pandoc/format-extension.ts were regenerated " + + "as an UNCOMMITTED working-tree diff - review with `git diff` and commit " + + "yourself. **" + + "\n** Only Pandoc-named copies were rewritten. Port each upstream change " + + "they show into Quarto's own templates by hand; this run cannot tell " + + "whether that was done. **" + + "\n** The `configuration` file is still pinned. Phase 2 (bump " + + "`configuration` + src/command/check/check.ts, archive the binary to S3, " + + "run the create-release.yml dry-run) must be done by an S3 credential " + + "holder before merge. See dev-docs/update-pandoc-checklist.md. **", + )))); + } else { + console.log(bgBlack(brightWhite(bold( + "\n** Remember to complete the checklist in /dev-docs/update-pandoc-checklist.md! **", + )))); + } }); } +// Pad a Pandoc version to at least 3 parts so "3.10" matches "3.10.0". +// Longer versions keep every part: 3.1.6.1 and 3.1.6.2 are distinct releases. +function normalizePandocVersion(version: string | undefined): string { + if (!version) { + return ""; + } + const parts = version.split("."); + if (parts.length < 3) { + return parts.concat(Array(3 - parts.length).fill("0")).join("."); + } + return parts.join("."); +} + // Starting in Pandoc 3, we saw a number of variants that appear to be supported // disappear from the --list-extensions command, so for the time being we're just // hard adding them here From 1f64698f9df081bfa63b978c88f7fb2a5199f3b2 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 4 Nov 2026 14:42:07 +0100 Subject: [PATCH 02/11] Allow smoke tests against a Pandoc release not yet archived A Pandoc bump could only be smoke-tested in CI once the new binary was archived to S3 and configuration pointed at it. The new pandoc-override-version dispatch input installs that release from GitHub and points quarto at it through QUARTO_PANDOC for the run only, so rendering changes can be checked before archival. The steps live in a composite action, placed after quarto is installed so the final `quarto pandoc --version` check works in every quarto-install mode. README indexes list the composite actions in both action directories. --- .github/actions/README.md | 10 +++++ .github/workflows/actions/README.md | 13 +++++++ .../actions/pandoc-override/action.yml | 38 +++++++++++++++++++ .github/workflows/test-smokes.yml | 11 ++++++ 4 files changed, 72 insertions(+) create mode 100644 .github/actions/README.md create mode 100644 .github/workflows/actions/README.md create mode 100644 .github/workflows/actions/pandoc-override/action.yml diff --git a/.github/actions/README.md b/.github/actions/README.md new file mode 100644 index 00000000000..2503b1431cb --- /dev/null +++ b/.github/actions/README.md @@ -0,0 +1,10 @@ +# Composite Actions + +Reusable steps shared across workflows in this repo. See also `.github/workflows/actions/` for another set of composite actions (historical split, no functional difference). + +| Action | Purpose | +|--------|---------| +| [`build-dist-tarball`](build-dist-tarball/action.yml) | Builds and uploads a versioned Quarto distribution tarball from the current checkout. | +| [`cache-typst`](cache-typst/action.yml) | Configures caching for Typst packages. | +| [`docker`](docker/action.yml) | Builds and pushes a Quarto Docker container to the GitHub Container Registry. | +| [`merge-extension-tests`](merge-extension-tests/action.yml) | Copies test files from extension subtrees into the main test directories. | diff --git a/.github/workflows/actions/README.md b/.github/workflows/actions/README.md new file mode 100644 index 00000000000..b0665d2cec1 --- /dev/null +++ b/.github/workflows/actions/README.md @@ -0,0 +1,13 @@ +# Composite Actions + +Reusable steps shared across workflows in this repo. See also `.github/actions/` for another set of composite actions (historical split, no functional difference). + +| Action | Purpose | +|--------|---------| +| [`amplitude`](amplitude/action.yml) | Sends an event to Amplitude. | +| [`archive`](archive/action.yml) | Archives a dependency binary to S3. | +| [`keychain`](keychain/action.yml) | Configures a P12 certificate into the keychain for signing. | +| [`pandoc-override`](pandoc-override/action.yml) | Installs an unarchived Pandoc release and points quarto at it via `QUARTO_PANDOC`, for smoke-testing a version before it has been archived to S3. | +| [`prevent-rerun`](prevent-rerun/action.yml) | Fails if the workflow is re-run (`github.run_attempt > 1`). | +| [`quarto-dev`](quarto-dev/action.yml) | Configures the image for Quarto development (runs `configure.sh`/`.cmd`). | +| [`sign-files`](sign-files/action.yml) | Installs and configures the environment to sign files. | diff --git a/.github/workflows/actions/pandoc-override/action.yml b/.github/workflows/actions/pandoc-override/action.yml new file mode 100644 index 00000000000..f94dd2dd901 --- /dev/null +++ b/.github/workflows/actions/pandoc-override/action.yml @@ -0,0 +1,38 @@ +name: "Pandoc Override" +description: "Installs an unarchived Pandoc release and points quarto at it via QUARTO_PANDOC, for smoke-testing a version before it has been archived to S3. No-op unless pandoc-version is set - the archived/configured binary is used as normal otherwise." + +inputs: + pandoc-version: + description: "Pandoc release tag to install and use instead of the archived/configured version (e.g. 3.10.1). Leave empty to skip." + required: false + default: "" + +runs: + using: "composite" + steps: + - name: Install Pandoc override + if: inputs.pandoc-version != '' + uses: r-lib/actions/setup-pandoc@v2 + with: + pandoc-version: ${{ inputs.pandoc-version }} + + - name: Point QUARTO_PANDOC at override (Linux) + if: runner.os == 'Linux' && inputs.pandoc-version != '' + shell: bash + run: echo "QUARTO_PANDOC=$(command -v pandoc)" >> "$GITHUB_ENV" + + - name: Point QUARTO_PANDOC at override (Windows) + if: runner.os == 'Windows' && inputs.pandoc-version != '' + shell: pwsh + run: | + "QUARTO_PANDOC=$((Get-Command pandoc).Source)" >> $env:GITHUB_ENV + + - name: Confirm Pandoc override is live (Linux) + if: runner.os == 'Linux' && inputs.pandoc-version != '' + shell: bash + run: quarto pandoc --version + + - name: Confirm Pandoc override is live (Windows) + if: runner.os == 'Windows' && inputs.pandoc-version != '' + shell: pwsh + run: quarto pandoc --version diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index b7a76360212..5cbabbc1339 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -71,6 +71,11 @@ on: required: false type: string default: "" + pandoc-override-version: + description: "Pandoc release tag to download from GitHub and use via QUARTO_PANDOC instead of the archived/configured version (e.g. 3.10.1) - for smoke-testing a version before it has been archived to S3. Leave empty for normal behavior." + required: false + type: string + default: "" # this is required to cleanup julia cache permissions: @@ -504,6 +509,12 @@ jobs: echo "QUARTO_TEST_EXPECTED_VERSION=$QUARTO_VERSION_INPUT" >> "$GITHUB_ENV" fi + # Guarded at the step so runs on older refs, which lack this action, never load it. + - if: inputs.pandoc-override-version != '' + uses: ./.github/workflows/actions/pandoc-override + with: + pandoc-version: ${{ inputs.pandoc-override-version }} + - name: Install Tinytex env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From afbfc86bb507730468adc9bd11cfa2af4b6d65f3 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 4 Nov 2026 14:42:30 +0100 Subject: [PATCH 03/11] Document the two-phase Pandoc update and which templates are wired Each format's pandoc/ directory holds both Quarto's real templates (template.tex, common.latex, styles.html, ...) and verbatim copies of Pandoc's own (latex.common, html.styles, typst.template, ...) that no code reads. Their names differ only by segment order, and the docs described some reference copies as if they were rendered. Template fixes have landed in the reference copy instead of the wired file because of this. The checklist now splits the update into a prep phase anyone can do (--skip-archive, the smoke-test override) and an archival phase for the S3 credential holder, and lists the wired and reference files per format. The llm-docs and path-scoped rules get the same correction, and a rule points configuration bumps to the dependency docs. --- .claude/rules/dev-tools/dependency-updates.md | 10 +++ .claude/rules/formats/pandoc-templates.md | 9 ++- dev-docs/update-pandoc-checklist.md | 60 +++++++++++++-- dev-docs/upgrade-dependencies.md | 4 +- llm-docs/localization-architecture.md | 19 ++--- llm-docs/pandoc-quarto-latex-templates.md | 76 ++++++++++++------- llm-docs/pandoc-quarto-typst-templates.md | 26 ++++++- 7 files changed, 158 insertions(+), 46 deletions(-) create mode 100644 .claude/rules/dev-tools/dependency-updates.md diff --git a/.claude/rules/dev-tools/dependency-updates.md b/.claude/rules/dev-tools/dependency-updates.md new file mode 100644 index 00000000000..eb95c2d4528 --- /dev/null +++ b/.claude/rules/dev-tools/dependency-updates.md @@ -0,0 +1,10 @@ +--- +description: "Bundled dependency version bumps (Deno, Pandoc, Dart Sass, Typst, esbuild, veraPDF)" +paths: + - "configuration" + - "src/command/check/check.ts" +--- + +# Bundled Dependency Updates + +Bumping any bundled binary version starts here: see `dev-docs/upgrade-dependencies.md` for the general procedure (version numbers, `check.ts` constraints, the installer signing/notarization dry-run gate before merging). For Pandoc specifically, `dev-docs/upgrade-dependencies.md` links to `dev-docs/update-pandoc-checklist.md` (template resync, schema/lua-types checks, CI smoke-test path). diff --git a/.claude/rules/formats/pandoc-templates.md b/.claude/rules/formats/pandoc-templates.md index 68c8d56e80c..e25f230324e 100644 --- a/.claude/rules/formats/pandoc-templates.md +++ b/.claude/rules/formats/pandoc-templates.md @@ -1,12 +1,19 @@ --- paths: - "src/resources/formats/pdf/pandoc/**" + - "src/resources/formats/beamer/pandoc/**" - "src/resources/formats/typst/pandoc/**" + - "src/resources/formats/html/pandoc/**" + - "src/resources/formats/revealjs/pandoc/**" + - "src/resources/formats/asciidoc/pandoc/**" - "package/src/common/update-pandoc.ts" + - "dev-docs/update-pandoc-checklist.md" --- -# Pandoc Templates (LaTeX & Typst) +# Pandoc Templates (LaTeX, Typst, HTML, Reveal.js, AsciiDoc) For how Pandoc's LaTeX templates are copied into Quarto and restructured into a modular form (`latex.template`, `latex.common`, …), see `llm-docs/pandoc-quarto-latex-templates.md`. For the equivalent Typst template integration (`typst.template`, `template.typst`), see `llm-docs/pandoc-quarto-typst-templates.md`. + +Every format's `pandoc/` directory has the same split: a dev-reference-only copy of Pandoc's own template (unreferenced by any TypeScript code — kept only so a maintainer can diff it against a fresh Pandoc checkout) alongside Quarto's actual, wired-up template. `dev-docs/update-pandoc-checklist.md` has the full per-format file list (pdf, beamer, html, revealjs, asciidoc, typst) and the naming gotcha that makes the reference-copy file easy to miss when checking what's in a directory. diff --git a/dev-docs/update-pandoc-checklist.md b/dev-docs/update-pandoc-checklist.md index abc4206d779..4d1b21217c8 100644 --- a/dev-docs/update-pandoc-checklist.md +++ b/dev-docs/update-pandoc-checklist.md @@ -1,13 +1,35 @@ ## Updating the bundled version of Pandoc -Carlos needs to run this: +Updating Pandoc happens in two phases. Phase 1 (prep) needs no S3 access and can be done by anyone; Phase 2 (archival) is done by whoever holds S3 write credentials, right before merge. -- [ ] Ensure archives are upgraded -- [ ] Run `AWS_PROFILE=... ./package/src/quarto-bld update-pandoc PANDOC_VERSION` -- [ ] look at `git diff`, specifically for changes in Pandoc templates, and adjust as needed. +### Phase 1 - prep (no S3, anyone) -As a reminder, our templates are kept in the same directories as Pandoc's templates, but with different names. `git diff` will show the diff in Pandoc's template; we have to manually patch -ours. (We can't just use `patch` because the templates have diverged too much) +Patch Quarto's wired-up template files and regenerate the dev-reference copies + `src/core/pandoc/format-extension.ts`, then verify rendering - all while `configuration`'s `export PANDOC=` line stays pinned at the currently-archived version, so plain `./configure.sh`/`.cmd` keeps working for everyone else. + +Run the template/variant regeneration with `--skip-archive`, pointing `QUARTO_PANDOC` at a real binary of the target version: + +```bash +QUARTO_PANDOC=/path/to/pandoc-3.10.1/bin/pandoc \ + ./package/src/quarto-bld update-pandoc 3.10.1 --skip-archive +``` + +`--skip-archive` skips the S3 archival, the `configureDependency` download, and the `configuration` rewrite. It still runs `writePandocTemplates` (regenerates every mapped template + dev-reference copy) and `writeVariants` (regenerates `format-extension.ts`). `QUARTO_PANDOC` is **required** with `--skip-archive`: `writeVariants` shells out to that binary to enumerate formats/extensions, and without the override it would silently regenerate `format-extension.ts` from the old configured Pandoc. The command leaves everything as an uncommitted working-tree diff for you to review and commit yourself; it never commits. + +The regeneration only overwrites the files mapped from Pandoc in `writePandocTemplates`: the dev-reference copies, plus the few partials Quarto's templates include under Pandoc's own name (`fonts.latex`, `hypersetup.latex`, ...). It never reads or writes Quarto's own templates (`template.tex`, `common.latex`, `tables.tex`, `styles.html`, `template.html`, `quarto/*.typ`), so it cannot tell whether an upstream change was ported to them. Use its diff on each reference copy as the list of upstream changes, and check each one against the corresponding Quarto template listed below. A diff limited to `format-extension.ts` is a normal generated change to review. The same applies to the full Phase 2 run. + +Rendering changes can also be smoke-tested before archival: dispatch `.github/workflows/test-smokes.yml` with the `pandoc-override-version` input set to the new release tag (e.g. `3.10.1`). It downloads that Pandoc release directly from GitHub, points `quarto` at it via `QUARTO_PANDOC` for that run only, and leaves the archived/configured version untouched - no S3 access needed. Scope `buckets` to the relevant smoke-all dirs (e.g. `typst`, `latex`, `table`) for a faster signal. + +### Phase 2 - archival (S3 credential holder, before merge) + +Whoever holds S3 write credentials does this last, right before merging: + +- [ ] Bump `configuration`'s `export PANDOC=` line and the `Pandoc` entry in `src/command/check/check.ts`'s `versionConstraints` array to the new version (this constraint tracks the bundled version exactly). +- [ ] Ensure archives are upgraded. +- [ ] Run `AWS_PROFILE=... ./package/src/quarto-bld update-pandoc PANDOC_VERSION` (without `--skip-archive`) to archive the binary to S3, configure it locally, and regenerate templates/variants. +- [ ] Look at `git diff`, specifically for changes in Pandoc templates, and adjust as needed. As in Phase 1, a diff on an already-patched file means something was missed. +- [ ] Run the `create-release.yml` dry-run signing/notarization gate (see `dev-docs/upgrade-dependencies.md`), then merge. + +As a reminder, our templates are kept in the same directories as Pandoc's templates, but with different names. `git diff` will show the diff in Pandoc's template; we have to manually patch ours. (We can't just use `patch` because the templates have diverged too much) ### Pandoc templates @@ -25,6 +47,32 @@ Partials: - Ours: - src/resources/formats/beamer/pandoc/common.latex +#### pdf / latex + +- Pandoc's: src/resources/formats/pdf/pandoc/latex.template +- Ours: src/resources/formats/pdf/pandoc/template.tex (see `llm-docs/pandoc-quarto-latex-templates.md` for the full partial breakdown, including `latex.common`/`common.latex`) + +#### html + +- Pandoc's: src/resources/formats/html/pandoc/html.template, src/resources/formats/html/pandoc/html.styles +- Ours: src/resources/formats/html/pandoc/template.html, src/resources/formats/html/pandoc/styles.html + +#### revealjs + +- Pandoc's: src/resources/formats/revealjs/pandoc/revealjs.template +- Ours: src/resources/formats/revealjs/pandoc/template.html + +#### asciidoc + +- Pandoc's: src/resources/formats/asciidoc/pandoc/asciidoc.template +- Ours: src/resources/formats/asciidoc/pandoc/template.asciidoc + +#### typst + +See `llm-docs/pandoc-quarto-typst-templates.md` - typst has an extra wrinkle where Pandoc's own `template.typst` partial is also kept verbatim (same filename) alongside the renamed `typst.template`. + +**All of the "Pandoc's" files above are dev-reference-only**: none of them are read by any TypeScript code path (confirmed via `grep -rn "" src/ --include="*.ts"` returning zero matches for each, across every format listed here). They exist purely so `git diff` against a fresh Pandoc checkout shows what changed upstream - patch the corresponding "Ours" file (or its own sub-partials, e.g. `tables.tex`, `toc.tex`) to actually change rendered behavior, and update the "Pandoc's" copy too so the next resync's diff stays meaningful. It's easy to update one and miss the other (or vice versa) since the filenames differ only by which segment comes first - use an unfiltered `ls`/`Glob **/*` on the directory when checking what's there, not an extension-filtered glob (e.g. `*.latex` silently excludes `latex.common`, which ends in `.common`). Because `writePandocTemplates` (in `package/src/common/update-pandoc.ts`) overwrites every "Pandoc's" copy wholesale on each run, never hand-fix an upstream bug in one of those reference copies - the change would silently vanish on the next resync, and the reference copy must stay byte-identical to upstream so the diff stays meaningful, even when upstream has a typo. Behavioral fixes belong only in the corresponding "Ours" file (or its sub-partials). + ## Manual steps - [ ] Update schemas by inspecting [their changelog](https://github.com/jgm/pandoc/blob/main/changelog.md) for new commands, deprecation removals, etc diff --git a/dev-docs/upgrade-dependencies.md b/dev-docs/upgrade-dependencies.md index b51f82d1121..fcebcaf9da6 100644 --- a/dev-docs/upgrade-dependencies.md +++ b/dev-docs/upgrade-dependencies.md @@ -1,6 +1,8 @@ Change version numbers in `./configuration` to correspond to new versions. -Update hardcoded version strings in `src/command/check/check.ts` (the `versionConstraints` array in `checkVersions`) so that they match the new versions in `configuration`. The `configuration` file warns about this in a comment, and `tests/smoke/check/check-versions-no-mismatch.test.ts` fails until they match. +Update hardcoded version strings in `src/command/check/check.ts` (the `versionConstraints` array in `checkVersions`) so that they match the new versions in `configuration`. The `configuration` file warns about this in a comment, and `tests/smoke/check/check-versions-no-mismatch.test.ts` fails until they match. The constraint tracks the bundled version exactly, not a minimum floor. + +**Pandoc specifically** has its own checklist: [update-pandoc-checklist.md](update-pandoc-checklist.md) - template resync (with a naming gotcha that's easy to miss), schema/lua-types checks, and a `QUARTO_PANDOC`-based CI path for smoke-testing a version before it's archived to S3. ## Verify installer signing & notarization before merging (bundled binaries) diff --git a/llm-docs/localization-architecture.md b/llm-docs/localization-architecture.md index 0fe590714ae..c2890d40066 100644 --- a/llm-docs/localization-architecture.md +++ b/llm-docs/localization-architecture.md @@ -16,14 +16,13 @@ key_files: - src/resources/filters/modules/authors.lua - src/resources/filters/modules/callouts.lua - src/resources/filters/layout/meta.lua - - src/resources/formats/html/pandoc/html.template + - src/resources/formats/html/pandoc/template.html + - src/resources/formats/html/pandoc/toc.html - src/resources/formats/html/pandoc/title-block.html - src/resources/formats/html/templates/title-metadata.html - - src/resources/formats/pdf/pandoc/latex.template - src/resources/formats/pdf/pandoc/babel-lang.tex - src/resources/formats/pdf/pandoc/toc.tex - - src/resources/formats/beamer/pandoc/beamer.template - - src/resources/formats/typst/pandoc/typst.template + - src/resources/formats/beamer/pandoc/toc.tex - src/resources/formats/typst/pandoc/quarto/typst-template.typ - src/resources/formats/typst/pandoc/quarto/typst-show.typ - src/resources/extension-subtrees/orange-book/_extensions/orange-book/typst-show.typ @@ -151,7 +150,9 @@ These bypass both Pandoc metadata and Lua filters. The string lands directly in `quarto.*` is an internal namespace. The user-facing override path for localized strings is the top-level `language:` YAML key (resolved by `formatLanguage`, already merged into `options.format.language` before the builder runs). The schema does not advertise `variables.quarto.*` as a user option, but the merge in `generateDefaults` honors a user-set value on collision: it deep-merges via `mergeConfigs` (`src/core/config.ts`) so user-supplied keys win at any nesting depth while all other localized values remain available. For example, a user setting `variables.quarto.language.crossref-ch-prefix: Bouquin` overrides only that one leaf — `$quarto.language.toc-title-document$` still resolves to the localized fallback. (Same helper is used in `src/core/language.ts:formatLanguage` to merge user-supplied `language:` onto `_language.yml` defaults.) A non-object `variables.quarto` (string, number, array) is ignored defensively. New contributors to this file should design around the `language:` path, not around the escape hatch. -From any Pandoc template — built-in (`html.template`, `latex.template`, `typst-template.typ`), extension partial, or custom user template — every localized key is then accessible as: +From any Pandoc template — built-in (`template.html`, `template.tex`, `typst-template.typ`), extension partial, or custom user template — every localized key is then accessible as: + +> **Naming note**: Quarto's actual, rendered templates are `template.html` (HTML), `template.tex` (LaTeX/PDF and Beamer), and `quarto/typst-template.typ` (Typst). The similarly-named `html.template`, `latex.template`, `beamer.template`, `typst.template`, `revealjs.template`, and `asciidoc.template` files elsewhere in these `pandoc/` directories are dev-reference-only copies of Pandoc's own default templates (zero references in `src/**/*.ts` — confirmed via grep), kept solely so a maintainer can diff them against a fresh Pandoc checkout during a version bump. Citations below point at the real, rendered files. ```pandoc $quarto.language.$ @@ -173,9 +174,9 @@ It does **not** replace 2a — Lua filters still need `param("key")` because Pan Localization paths used: -- **Document `lang` attribute**: `_language.yml` lookup not involved. `lang:` flows as Pandoc-native metadata, template renders `` (`html.template:2`). -- **TOC title**: `$toc-title$` resolved via channel 2b. Used in `html.template:60`, `toc.html:3`. -- **Abstract title**: `$abstract-title$` resolved via 2b. Used in `html.template:51`, `title-block.html:15`. +- **Document `lang` attribute**: `_language.yml` lookup not involved. `lang:` flows as Pandoc-native metadata, template renders `` (`template.html:2`). +- **TOC title**: `$toc-title$` resolved via channel 2b. Used in `toc.html:3`. +- **Abstract title**: `$abstract-title$` resolved via 2b. Used in `title-block.html:15`. - **Title block labels** (`Authors`, `Affiliations`, `Published`, `Modified`, `Doi`, `Abstract`, `Keywords`): `$labels.*$` written into meta by `modules/authors.lua:854-913` (`computeLabels`). Templates: `title-metadata.html:3,4,32,43,52,61,74,83`, `manuscript/title-metadata.html:6,7,37,48,57,66,83,92`. - **Crossref text** (`Figure 1.1`, `Table 2.1`): assembled in Lua by `crossref/format.lua` using `title()` / `refPrefix()` which call `param("crossref--title"/"-prefix")`. Written as inline text directly into the AST. By the time Pandoc renders HTML, the localized prefix is already document content. - **Callout titles** (`Tip`, `Note`, etc.): `modules/callouts.lua:15,185` reads `param("callout--title", default)`, writes into the callout node. @@ -191,7 +192,7 @@ Three localization paths layered: Other Lua filters use `metaInjectLatex` for non-language LaTeX customization (`crossref/custom.lua:78`, `layout/meta.lua`, `quarto-post/landscape.lua`, etc. — they inject packages or styling, not localized strings). -3. **Pandoc template `$var$`** — `pdf/pandoc/toc.tex:3` and `pdf/pandoc/latex.template:93-94` use `$toc-title$` to set `\contentsname`. Beamer templates (`beamer/pandoc/beamer.template:146-151`, `beamer/pandoc/toc.tex:3`) likewise. +3. **Pandoc template `$var$`** — `pdf/pandoc/toc.tex:2-3` uses `$toc-title$` to set `\contentsname`. Beamer's `beamer/pandoc/toc.tex:2-3` does the same. PDF-side TS code does not read `format.language` directly except `src/format/pdf/format-pdf.ts:242` which registers `"babel-lang"` as a Quarto partial. diff --git a/llm-docs/pandoc-quarto-latex-templates.md b/llm-docs/pandoc-quarto-latex-templates.md index c75e11f5d9b..784bfb11bb3 100644 --- a/llm-docs/pandoc-quarto-latex-templates.md +++ b/llm-docs/pandoc-quarto-latex-templates.md @@ -1,3 +1,16 @@ +--- +main_commit: d30cdbb9e +analyzed_date: 2026-07-23 +key_files: + - package/src/common/update-pandoc.ts + - src/format/pdf/format-pdf.ts + - src/resources/formats/pdf/pandoc/template.tex + - src/resources/formats/pdf/pandoc/common.latex + - src/resources/formats/pdf/pandoc/pandoc.tex + - src/resources/formats/pdf/pandoc/document-metadata.latex + - src/resources/formats/beamer/pandoc/template.tex +--- + # Pandoc and Quarto LaTeX Templates This document describes how Quarto integrates Pandoc's LaTeX templates and transforms them into a modular structure suitable for Quarto's extended functionality. @@ -13,7 +26,7 @@ The `writePandocTemplates` function in `package/src/common/update-pandoc.ts` han ### Files Copied -Seven files are copied from Pandoc for LaTeX support: +**Stale-script warning**: `update-pandoc.ts`'s `templateDirFiles` mapping only lists `default.latex`, `common.latex`, `after-header-includes.latex`, `hypersetup.latex`, `font-settings.latex`, `fonts.latex`, and `passoptions.latex`. It does **not** include `document-metadata.latex` (added later, see below), so the script under-copies relative to what `format-pdf.ts` actually wires up. Don't trust the script's file list as the source of truth — cross-check against `partialNamesPandoc` in `format-pdf.ts` instead. | Pandoc Source | Quarto Destination | Notes | | ----------------------------- | ------------------ | -------------------------------------------------- | @@ -24,6 +37,7 @@ Seven files are copied from Pandoc for LaTeX support: | `font-settings.latex` | (unchanged) | User font configuration | | `fonts.latex` | (unchanged) | Font engine detection | | `passoptions.latex` | (unchanged) | Package options passthrough | +| `document-metadata.latex` | (unchanged) | PDF standard metadata (PDF/A, PDF/UA, PDF/X); added in Quarto 1.7 following Pandoc 3.6.3 support; **not copied by `update-pandoc.ts`'s current mapping** | ### Purpose of Each Pandoc File @@ -41,6 +55,8 @@ Seven files are copied from Pandoc for LaTeX support: **passoptions.latex**: Package options passthrough mechanism for hyperref, xcolor, and xeCJK. +**document-metadata.latex**: Emits a `\DocumentMetadata{...}` block (LuaLaTeX, 2023+ LaTeX) for PDF/A, PDF/UA, and PDF/X standards support, driven by the `pdfstandard` variable. Called first, before `passoptions.latex()`, in the pdf `template.tex`. **Beamer does not use this partial** — its `template.tex` has no `document-metadata.latex()` call, and `format-pdf.ts` filters it out of `beamerPartialNamesPandoc`. + ## 2. Quarto's Modular Template Structure Quarto breaks the Pandoc templates into a modular structure in `src/resources/formats/pdf/pandoc/`. This allows for: @@ -68,7 +84,7 @@ Assembles the document by including all partials in order. This is Quarto's main Quarto maintains two versions of the common preamble: -**latex.common** (264 lines): Pandoc's original, kept as reference. Contains the full common preamble: paragraph formatting, verbatim, highlighting, tables, graphics, strikeout, CSL citations, Babel, page style, tight lists, subfigure support, text direction, natbib/biblatex configuration. +**latex.common** (265 lines): Pandoc's original, kept as reference. Contains the full common preamble: paragraph formatting, verbatim, highlighting, tables, graphics, strikeout, CSL citations, Babel, page style, tight lists, subfigure support, text direction, natbib/biblatex configuration. **common.latex** (66 lines): Quarto's shortened version that: @@ -77,6 +93,8 @@ Quarto maintains two versions of the common preamble: This allows Quarto to selectively override specific parts of the common preamble while maintaining compatibility with Pandoc updates. +**Naming gotcha**: `latex.common` and `common.latex` are two *different* files that happen to share the same two words reversed - `latex.common` is Pandoc's own file (named `common.latex` in Pandoc's own repo, renamed on copy), `common.latex` is Quarto's own, unrelated in content. An extension-filtered glob like `*.latex` will silently miss `latex.common` (it ends in `.common`). When checking whether either file exists or is up to date, list the directory unfiltered rather than filtering by extension. + ### Quarto Partials (\*.tex) **Document Structure:** @@ -120,37 +138,38 @@ The `template.tex` orchestrates the full document: ### Preamble (before `\begin{document}`) -1. `$passoptions.latex()$` - Package options passthrough -2. `$doc-class.tex()$` - `\documentclass[...]{...}` declaration -3. (inline) - beamerarticle, xcolor, geometry, amsmath -4. (inline) - Section numbering configuration -5. `$fonts.latex()$` - Font engine detection (ifPDFTeX, ifXeTeX, ifLuaTeX) -6. `$font-settings.latex()$` - User font configuration -7. `$common.latex()$` - Common preamble setup, which internally calls: +1. `$document-metadata.latex()$` - `\DocumentMetadata{...}` for PDF/A, PDF/UA, PDF/X (pdf only — beamer's `template.tex` omits this call) +2. `$passoptions.latex()$` - Package options passthrough +3. `$doc-class.tex()$` - `\documentclass[...]{...}` declaration +4. (inline) - beamerarticle, xcolor, geometry, amsmath +5. (inline) - Section numbering configuration +6. `$fonts.latex()$` - Font engine detection (ifPDFTeX, ifXeTeX, ifLuaTeX) +7. `$font-settings.latex()$` - User font configuration +8. `$common.latex()$` - Common preamble setup, which internally calls: - `$pandoc.tex()$` → `$tables.tex()$`, `$graphics.tex()$`, `$citations.tex()$`, `$babel-lang.tex()$`, `$tightlist.tex()$`, `$biblio-config.tex()$` -8. `$after-header-includes.latex()$` - Bookmark, URL packages -9. `$hypersetup.latex()$` - Hyperref configuration -10. `$before-title.tex()$` - Pre-title content -11. `$title.tex()$` - `\title{}`, `\author{}`, `\date{}` +9. `$after-header-includes.latex()$` - Bookmark, URL packages +10. `$hypersetup.latex()$` - Hyperref configuration +11. `$before-title.tex()$` - Pre-title content +12. `$title.tex()$` - `\title{}`, `\author{}`, `\date{}` ### Document Body -12. `\begin{document}` -13. `$before-body.tex()$` - Frontmatter, `\maketitle`, abstract -14. (loop) - include-before content -15. `$toc.tex()$` - Table of contents -16. (inline) - linestretch, mainmatter -17. `$body$` - Document content -18. `$before-bib.tex()$` - Pre-bibliography content -19. (inline) - backmatter, nocite -20. `$biblio.tex()$` - Bibliography rendering -21. (loop) - include-after content -22. `$after-body.tex()$` - Post-body content -23. `\end{document}` +13. `\begin{document}` +14. `$before-body.tex()$` - Frontmatter, `\maketitle`, abstract +15. (loop) - include-before content +16. `$toc.tex()$` - Table of contents +17. (inline) - linestretch, mainmatter +18. `$body$` - Document content +19. `$before-bib.tex()$` - Pre-bibliography content +20. (inline) - backmatter, nocite +21. `$biblio.tex()$` - Bibliography rendering +22. (loop) - include-after content +23. `$after-body.tex()$` - Post-body content +24. `\end{document}` ## 4. TypeScript Configuration -The `pdfFormat` function in `src/format/pdf/format-pdf.ts` (lines 236-284) defines the template configuration: +The template configuration is defined inline inside `src/format/pdf/format-pdf.ts` (around lines 241-267, in the function that builds the PDF format): ```typescript // Quarto-maintained partials (.tex) @@ -172,10 +191,11 @@ const partialNamesQuarto: string[] = [ "toc", ]; -// Pandoc-derived partials (.latex) +// Pandoc-derived partials (.latex) — since Pandoc 3.6.3 const partialNamesPandoc: string[] = [ "after-header-includes", "common", + "document-metadata", "font-settings", "fonts", "hypersetup", @@ -188,6 +208,8 @@ The template context maps these to files: - Quarto partials: `pandoc/${name}.tex` - Pandoc partials: `pandoc/${name}.latex` +Beamer builds its own `beamerPartialNamesPandoc` by filtering `document-metadata` out of `partialNamesPandoc` (`.filter((name) => name !== "document-metadata")`), since beamer's `template.tex` never calls it. + ## 5. Parameter Summary Table ### Font Parameters (XeTeX/LuaTeX) diff --git a/llm-docs/pandoc-quarto-typst-templates.md b/llm-docs/pandoc-quarto-typst-templates.md index c0182dd640c..1d3b2a1aa92 100644 --- a/llm-docs/pandoc-quarto-typst-templates.md +++ b/llm-docs/pandoc-quarto-typst-templates.md @@ -1,3 +1,15 @@ +--- +main_commit: d30cdbb9e +analyzed_date: 2026-07-23 +key_files: + - package/src/common/update-pandoc.ts + - src/format/typst/format-typst.ts + - src/resources/formats/typst/pandoc/quarto/definitions.typ + - src/resources/formats/typst/pandoc/quarto/template.typ + - src/resources/formats/typst/pandoc/typst.template + - src/resources/formats/typst/pandoc/template.typst +--- + # Pandoc and Quarto Typst Templates This document describes how Quarto integrates Pandoc's typst templates and transforms them into a more modular structure suitable for Quarto's extended functionality. @@ -172,9 +184,19 @@ Handles bibliography rendering: - Sets bibliography style from CSL file or bibliography style option - Renders bibliography from specified files -### typst.template - Reference Copy +### typst.template and template.typst - Dev-Reference Copies Only + +`typst.template` (Pandoc's `default.typst`) and `template.typst` (Pandoc's own `template.typst`, kept under the same name) are **not referenced by any runtime TypeScript code path** — a repo-wide grep for either filename across `src/**/*.ts` returns zero matches. `format-typst.ts` wires up only the `quarto/` partials (see above) as the template context. + +These two files exist solely so a maintainer running the Pandoc update process (`writePandocTemplates` in `package/src/common/update-pandoc.ts`) can `git diff` them against a fresh Pandoc checkout to see what changed upstream, then manually port the relevant parts into `quarto/definitions.typ` and friends. They are not a "no custom template" fallback path — don't rely on them actually executing for any user-facing render. + +### Writer-Level Changes vs. Template-Level Changes + +Not every user-visible difference between Pandoc versions shows up as a diff in these template files. Some behavior lives in Pandoc's own Haskell **typst writer** — the code that converts the Pandoc AST into typst body markup — and is emitted directly into the rendered body regardless of which template (if any) is used. + +Example (verified 2026-07-23, Pandoc 3.10.0 → 3.10.1): the typst writer changed how it emits a `HorizontalRule` AST node, from `#horizontalrule` (calling a symbol the template defines) to `#divider()` (typst's own native function, added in typst 0.15). It also changed term-list (definition list) output from wrapping each description in `#block[...]` to emitting it bare, relying on an external `#set terms(hanging-indent: ...)`. Neither change touched `default.typst`/`template.typst` content in a way that mattered for Quarto: Quarto's `quarto/definitions.typ` keeps its own `#let horizontalrule = ...` (still used internally by `notes.typ`) and its own `#show terms.item` rule (which reformats the description regardless of whether Pandoc pre-wrapped it in `#block[...]`), and since Quarto always bundles a recent typst (0.15.1+) where `divider` is a native `std` builtin, `#divider()` in the body just resolves with no local definition needed. -This is the copy of Pandoc's `default.typst`, kept for reference. It is used when rendering without Quarto's template context (e.g., when a user provides a custom template). +**Implication for future Pandoc bumps**: a clean `data/templates/` diff between two Pandoc versions is necessary but not sufficient evidence that "nothing changed" for typst output. Confirm by actually rendering a document with both Pandoc versions (e.g. via a `QUARTO_PANDOC` env override against the bundled binary) and inspecting the generated `.typ` body content, not just diffing the template files. ## 3. Parameter Summary Table From 5f0fc58d4565ed7d94ec024a2e75b1cc1e180eb0 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 4 Nov 2026 15:52:30 +0100 Subject: [PATCH 04/11] Accept +1 Deno.env.get in package/src (update-pandoc.ts QUARTO_PANDOC override) The new Deno.env.get("QUARTO_PANDOC") in package/src/common/update-pandoc.ts raises the package/src count from 240 to 241. src/deno_ral has no env accessor yet, and package/src already reads QUARTO_PANDOC the same way in update-html-dependencies.ts, so there is nothing to migrate in this PR. Bump the baseline so the runtime-boundary check accepts the count. --- tools/runtime-boundary-baseline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/runtime-boundary-baseline.json b/tools/runtime-boundary-baseline.json index cf99f8c67f6..06886dae228 100644 --- a/tools/runtime-boundary-baseline.json +++ b/tools/runtime-boundary-baseline.json @@ -12,7 +12,7 @@ "stdImportFiles": 1 }, "package/src": { - "denoCalls": 240, + "denoCalls": 241, "stdImportFiles": 3 }, "tests": { From d12ad7212a3e157bae0e4353917576c0e53dc715 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 4 Nov 2026 16:03:11 +0100 Subject: [PATCH 05/11] Clarify Pandoc update messages and inline guidance Describe the required Pandoc override and the next action directly. Keep the full update procedure in the checklist so command output stays concise. --- package/src/common/update-pandoc.ts | 55 ++++++++--------------------- 1 file changed, 14 insertions(+), 41 deletions(-) diff --git a/package/src/common/update-pandoc.ts b/package/src/common/update-pandoc.ts index 9efcc351f83..424bc640cd6 100644 --- a/package/src/common/update-pandoc.ts +++ b/package/src/common/update-pandoc.ts @@ -43,7 +43,7 @@ export function updatePandoc() { .description("Updates Pandoc to the specified version") .option( "--skip-archive", - "Phase 1 prep only: regenerate templates and format-extension.ts from a QUARTO_PANDOC-provided binary without archiving to S3, configuring the binary, or bumping the configuration file. Requires QUARTO_PANDOC to point at a Pandoc binary.", + "Regenerate templates and format-extension.ts without archiving to S3, configuring the binary, or updating configuration. Requires QUARTO_PANDOC to resolve to the requested version.", ) // deno-lint-ignore no-explicit-any .action(async (options: any, version: string) => { @@ -53,32 +53,18 @@ export function updatePandoc() { const skipArchive = !!options.skipArchive; if (skipArchive) { - // Phase 1 (prep): no S3, no configuration rewrite. Regenerate the - // template files and format-extension.ts against a locally supplied - // binary. This requires a real - // Pandoc binary resolvable via QUARTO_PANDOC, because - // writeVariants() shells out to it to enumerate formats/extensions. + // writeVariants() needs the target Pandoc binary to enumerate formats/extensions. const overridePath = Deno.env.get("QUARTO_PANDOC"); if (!overridePath) { error( - "--skip-archive requires the QUARTO_PANDOC environment variable to " + - `point at a Pandoc ${version} binary. Without it, writeVariants() ` + - "would regenerate src/core/pandoc/format-extension.ts from whatever " + - "Pandoc is currently configured (likely the old version), silently " + - "corrupting it. Download the target release and set QUARTO_PANDOC, e.g.\n" + - ` QUARTO_PANDOC=/path/to/pandoc-${version}/bin/pandoc \\\n` + - ` ./package/src/quarto-bld update-pandoc ${version} --skip-archive`, + `QUARTO_PANDOC is not set. Set it to a Pandoc ${version} executable ` + + "before using --skip-archive.", ); throw new Error("QUARTO_PANDOC is not set"); } - // Fatal check: pandocBinaryPath() silently falls back to the bundled/ - // configured Pandoc (with only a warnOnce) if QUARTO_PANDOC is unset, - // nonexistent, or otherwise fails to resolve (src/core/resources.ts - // toolsPath()). If the resolved binary doesn't report the requested - // version, continuing would regenerate format-extension.ts from the - // wrong (likely old, bundled) Pandoc - the exact silent corruption - // this flag exists to prevent - so this must fail, not warn. + // An unresolved QUARTO_PANDOC can fall back to the bundled binary. + // Verify the version before generating format-extension.ts. let reported: string | undefined; try { reported = lines( @@ -91,7 +77,7 @@ export function updatePandoc() { } catch (_e) { error( "Could not run the QUARTO_PANDOC binary to verify its version. " + - `QUARTO_PANDOC is set to "${overridePath}" - confirm it points at a ` + + `QUARTO_PANDOC is set to "${overridePath}". Confirm it points at a ` + "runnable Pandoc executable.", ); throw new Error("QUARTO_PANDOC binary is not runnable"); @@ -102,15 +88,13 @@ export function updatePandoc() { error( `The Pandoc binary resolved via QUARTO_PANDOC reports version ` + `${reported}, not the requested ${version}. QUARTO_PANDOC is set to ` + - `"${overridePath}" - if that path doesn't exist or isn't a file, ` + - "pandocBinaryPath() silently fell back to the bundled/configured " + - "Pandoc, which would corrupt format-extension.ts if this continued. " + - "Fix QUARTO_PANDOC and retry.", + `"${overridePath}". An unresolved override can fall back to the ` + + `bundled binary. Set QUARTO_PANDOC to a Pandoc ${version} executable ` + + "and retry.", ); throw new Error("QUARTO_PANDOC does not resolve to the requested version"); } } else { - // Update the configuration file info(" updating configuration file."); const configFilePath = join( configuration.directoryInfo.root, @@ -136,7 +120,6 @@ export function updatePandoc() { // Archive to S3 await archiveBinaryDependency(pandocDependency, workingDir); - // Configure this version of pandoc await configureDependency( pandocDependency, join(configuration.directoryInfo.bin, "tools"), @@ -144,8 +127,7 @@ export function updatePandoc() { ); } - // Generate templates (always: downloads Pandoc's source zip from - // GitHub, never touches S3) + // Templates are downloaded from Pandoc's source archive on GitHub. await writePandocTemplates(configuration, version, workingDir); // Generate variants (always: uses the resolved Pandoc binary, which @@ -153,20 +135,11 @@ export function updatePandoc() { await writeVariants(configuration); }); - // print the completion message if (skipArchive) { console.log(bgBlack(brightWhite(bold( - "\n** Phase 1 (prep) complete. **" + - "\n** Templates and src/core/pandoc/format-extension.ts were regenerated " + - "as an UNCOMMITTED working-tree diff - review with `git diff` and commit " + - "yourself. **" + - "\n** Only Pandoc-named copies were rewritten. Port each upstream change " + - "they show into Quarto's own templates by hand; this run cannot tell " + - "whether that was done. **" + - "\n** The `configuration` file is still pinned. Phase 2 (bump " + - "`configuration` + src/command/check/check.ts, archive the binary to S3, " + - "run the create-release.yml dry-run) must be done by an S3 credential " + - "holder before merge. See dev-docs/update-pandoc-checklist.md. **", + "\n** Templates and src/core/pandoc/format-extension.ts regenerated. **" + + "\n** Review `git diff` and port relevant upstream changes to Quarto's templates. **" + + "\n** Configuration is unchanged. Complete dev-docs/update-pandoc-checklist.md before merging. **", )))); } else { console.log(bgBlack(brightWhite(bold( From f8c73f67b1ad4ef9fb7e8c422347da8fc033f7d1 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 4 Nov 2026 16:04:32 +0100 Subject: [PATCH 06/11] Consolidate Pandoc template maintenance guidance Keep reference-copy maintenance in the update checklist and point related architecture notes to it. Split the Typst writer example into shorter explanations and scope the bundled-version statement to the configured version. --- .claude/rules/formats/pandoc-templates.md | 2 +- dev-docs/update-pandoc-checklist.md | 22 ++++++++++++++-------- llm-docs/localization-architecture.md | 2 +- llm-docs/pandoc-quarto-latex-templates.md | 6 +++--- llm-docs/pandoc-quarto-typst-templates.md | 12 +++++++----- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/.claude/rules/formats/pandoc-templates.md b/.claude/rules/formats/pandoc-templates.md index e25f230324e..f875c8b922d 100644 --- a/.claude/rules/formats/pandoc-templates.md +++ b/.claude/rules/formats/pandoc-templates.md @@ -16,4 +16,4 @@ For how Pandoc's LaTeX templates are copied into Quarto and restructured into a For the equivalent Typst template integration (`typst.template`, `template.typst`), see `llm-docs/pandoc-quarto-typst-templates.md`. -Every format's `pandoc/` directory has the same split: a dev-reference-only copy of Pandoc's own template (unreferenced by any TypeScript code — kept only so a maintainer can diff it against a fresh Pandoc checkout) alongside Quarto's actual, wired-up template. `dev-docs/update-pandoc-checklist.md` has the full per-format file list (pdf, beamer, html, revealjs, asciidoc, typst) and the naming gotcha that makes the reference-copy file easy to miss when checking what's in a directory. +When updating Pandoc templates, read `dev-docs/update-pandoc-checklist.md` for the per-format file list, reference-copy maintenance, and filename checks. diff --git a/dev-docs/update-pandoc-checklist.md b/dev-docs/update-pandoc-checklist.md index 4d1b21217c8..8edf471b88a 100644 --- a/dev-docs/update-pandoc-checklist.md +++ b/dev-docs/update-pandoc-checklist.md @@ -4,20 +4,22 @@ Updating Pandoc happens in two phases. Phase 1 (prep) needs no S3 access and can ### Phase 1 - prep (no S3, anyone) -Patch Quarto's wired-up template files and regenerate the dev-reference copies + `src/core/pandoc/format-extension.ts`, then verify rendering - all while `configuration`'s `export PANDOC=` line stays pinned at the currently-archived version, so plain `./configure.sh`/`.cmd` keeps working for everyone else. +Regenerate the reference copies and `src/core/pandoc/format-extension.ts`, port relevant changes to Quarto's templates, and verify rendering. Keep `configuration`'s `export PANDOC=` line at the archived version so `./configure.sh`/`.cmd` continues to work. -Run the template/variant regeneration with `--skip-archive`, pointing `QUARTO_PANDOC` at a real binary of the target version: +Run the template/variant regeneration with `--skip-archive`, pointing `QUARTO_PANDOC` at an executable of the target version: ```bash QUARTO_PANDOC=/path/to/pandoc-3.10.1/bin/pandoc \ ./package/src/quarto-bld update-pandoc 3.10.1 --skip-archive ``` -`--skip-archive` skips the S3 archival, the `configureDependency` download, and the `configuration` rewrite. It still runs `writePandocTemplates` (regenerates every mapped template + dev-reference copy) and `writeVariants` (regenerates `format-extension.ts`). `QUARTO_PANDOC` is **required** with `--skip-archive`: `writeVariants` shells out to that binary to enumerate formats/extensions, and without the override it would silently regenerate `format-extension.ts` from the old configured Pandoc. The command leaves everything as an uncommitted working-tree diff for you to review and commit yourself; it never commits. +`--skip-archive` skips S3 archival, the `configureDependency` download, and the `configuration` rewrite. It still runs `writePandocTemplates` to regenerate the mapped template files and `writeVariants` to regenerate `format-extension.ts`. -The regeneration only overwrites the files mapped from Pandoc in `writePandocTemplates`: the dev-reference copies, plus the few partials Quarto's templates include under Pandoc's own name (`fonts.latex`, `hypersetup.latex`, ...). It never reads or writes Quarto's own templates (`template.tex`, `common.latex`, `tables.tex`, `styles.html`, `template.html`, `quarto/*.typ`), so it cannot tell whether an upstream change was ported to them. Use its diff on each reference copy as the list of upstream changes, and check each one against the corresponding Quarto template listed below. A diff limited to `format-extension.ts` is a normal generated change to review. The same applies to the full Phase 2 run. +`QUARTO_PANDOC` is required: `writeVariants` uses that binary to enumerate formats and extensions. The command checks its version before regenerating files. Review and commit the resulting working-tree diff. -Rendering changes can also be smoke-tested before archival: dispatch `.github/workflows/test-smokes.yml` with the `pandoc-override-version` input set to the new release tag (e.g. `3.10.1`). It downloads that Pandoc release directly from GitHub, points `quarto` at it via `QUARTO_PANDOC` for that run only, and leaves the archived/configured version untouched - no S3 access needed. Scope `buckets` to the relevant smoke-all dirs (e.g. `typst`, `latex`, `table`) for a faster signal. +`writePandocTemplates` overwrites the reference copies and mapped Pandoc partials such as `fonts.latex` and `hypersetup.latex`. Port relevant changes to Quarto's templates using the [file list below](#pandoc-templates). Review generated changes to `format-extension.ts` as well. The same review applies to the full Phase 2 run. + +To smoke-test rendering before archival, dispatch `.github/workflows/test-smokes.yml` with `pandoc-override-version` set to the new release tag (e.g. `3.10.1`). The workflow downloads Pandoc from GitHub and uses it via `QUARTO_PANDOC` for that run, without S3 access. Limit `buckets` to the relevant smoke-all directories (e.g. `typst`, `latex`, `table`) for a faster run. ### Phase 2 - archival (S3 credential holder, before merge) @@ -29,7 +31,7 @@ Whoever holds S3 write credentials does this last, right before merging: - [ ] Look at `git diff`, specifically for changes in Pandoc templates, and adjust as needed. As in Phase 1, a diff on an already-patched file means something was missed. - [ ] Run the `create-release.yml` dry-run signing/notarization gate (see `dev-docs/upgrade-dependencies.md`), then merge. -As a reminder, our templates are kept in the same directories as Pandoc's templates, but with different names. `git diff` will show the diff in Pandoc's template; we have to manually patch ours. (We can't just use `patch` because the templates have diverged too much) +Quarto's templates share directories with the Pandoc reference copies, but use different names. Review the reference-copy diffs and port relevant changes manually; the templates have diverged too much to use `patch`. ### Pandoc templates @@ -69,9 +71,13 @@ Partials: #### typst -See `llm-docs/pandoc-quarto-typst-templates.md` - typst has an extra wrinkle where Pandoc's own `template.typst` partial is also kept verbatim (same filename) alongside the renamed `typst.template`. +Typst keeps Pandoc's `template.typst` partial under its original name alongside the renamed `typst.template`. See `llm-docs/pandoc-quarto-typst-templates.md` for Quarto's partials. + +The files labelled "Pandoc's" are reference copies used to review upstream changes. Rendering uses the corresponding "Ours" files and their partials, such as `tables.tex` and `toc.tex`. -**All of the "Pandoc's" files above are dev-reference-only**: none of them are read by any TypeScript code path (confirmed via `grep -rn "" src/ --include="*.ts"` returning zero matches for each, across every format listed here). They exist purely so `git diff` against a fresh Pandoc checkout shows what changed upstream - patch the corresponding "Ours" file (or its own sub-partials, e.g. `tables.tex`, `toc.tex`) to actually change rendered behavior, and update the "Pandoc's" copy too so the next resync's diff stays meaningful. It's easy to update one and miss the other (or vice versa) since the filenames differ only by which segment comes first - use an unfiltered `ls`/`Glob **/*` on the directory when checking what's there, not an extension-filtered glob (e.g. `*.latex` silently excludes `latex.common`, which ends in `.common`). Because `writePandocTemplates` (in `package/src/common/update-pandoc.ts`) overwrites every "Pandoc's" copy wholesale on each run, never hand-fix an upstream bug in one of those reference copies - the change would silently vanish on the next resync, and the reference copy must stay byte-identical to upstream so the diff stays meaningful, even when upstream has a typo. Behavioral fixes belong only in the corresponding "Ours" file (or its sub-partials). +- Regenerate reference copies from upstream with `writePandocTemplates`; apply rendering changes to Quarto's templates and partials. +- Keep reference copies identical to upstream, including any upstream typos. Manual edits are overwritten on regeneration. Fix upstream bugs in Quarto's templates. +- List the directory without filtering by extension when checking the files: `*.latex` excludes `latex.common`, which ends in `.common`. ## Manual steps diff --git a/llm-docs/localization-architecture.md b/llm-docs/localization-architecture.md index c2890d40066..96423e413d6 100644 --- a/llm-docs/localization-architecture.md +++ b/llm-docs/localization-architecture.md @@ -152,7 +152,7 @@ These bypass both Pandoc metadata and Lua filters. The string lands directly in From any Pandoc template — built-in (`template.html`, `template.tex`, `typst-template.typ`), extension partial, or custom user template — every localized key is then accessible as: -> **Naming note**: Quarto's actual, rendered templates are `template.html` (HTML), `template.tex` (LaTeX/PDF and Beamer), and `quarto/typst-template.typ` (Typst). The similarly-named `html.template`, `latex.template`, `beamer.template`, `typst.template`, `revealjs.template`, and `asciidoc.template` files elsewhere in these `pandoc/` directories are dev-reference-only copies of Pandoc's own default templates (zero references in `src/**/*.ts` — confirmed via grep), kept solely so a maintainer can diff them against a fresh Pandoc checkout during a version bump. Citations below point at the real, rendered files. +> **Naming note**: Citations below refer to Quarto's rendering templates: `template.html` (HTML), `template.tex` (LaTeX/PDF and Beamer), and `quarto/typst-template.typ` (Typst). See `dev-docs/update-pandoc-checklist.md` for the corresponding Pandoc reference copies. ```pandoc $quarto.language.$ diff --git a/llm-docs/pandoc-quarto-latex-templates.md b/llm-docs/pandoc-quarto-latex-templates.md index 784bfb11bb3..31a007ef89b 100644 --- a/llm-docs/pandoc-quarto-latex-templates.md +++ b/llm-docs/pandoc-quarto-latex-templates.md @@ -26,7 +26,7 @@ The `writePandocTemplates` function in `package/src/common/update-pandoc.ts` han ### Files Copied -**Stale-script warning**: `update-pandoc.ts`'s `templateDirFiles` mapping only lists `default.latex`, `common.latex`, `after-header-includes.latex`, `hypersetup.latex`, `font-settings.latex`, `fonts.latex`, and `passoptions.latex`. It does **not** include `document-metadata.latex` (added later, see below), so the script under-copies relative to what `format-pdf.ts` actually wires up. Don't trust the script's file list as the source of truth — cross-check against `partialNamesPandoc` in `format-pdf.ts` instead. +The `templateDirFiles` mapping in `update-pandoc.ts` omits `document-metadata.latex`. Check `partialNamesPandoc` in `format-pdf.ts` for the complete list of Pandoc-derived partials. | Pandoc Source | Quarto Destination | Notes | | ----------------------------- | ------------------ | -------------------------------------------------- | @@ -93,7 +93,7 @@ Quarto maintains two versions of the common preamble: This allows Quarto to selectively override specific parts of the common preamble while maintaining compatibility with Pandoc updates. -**Naming gotcha**: `latex.common` and `common.latex` are two *different* files that happen to share the same two words reversed - `latex.common` is Pandoc's own file (named `common.latex` in Pandoc's own repo, renamed on copy), `common.latex` is Quarto's own, unrelated in content. An extension-filtered glob like `*.latex` will silently miss `latex.common` (it ends in `.common`). When checking whether either file exists or is up to date, list the directory unfiltered rather than filtering by extension. +Pandoc's `common.latex` is copied as `latex.common`; Quarto maintains its own `common.latex`. See `dev-docs/update-pandoc-checklist.md` for reference-copy maintenance and filename checks. ### Quarto Partials (\*.tex) @@ -191,7 +191,7 @@ const partialNamesQuarto: string[] = [ "toc", ]; -// Pandoc-derived partials (.latex) — since Pandoc 3.6.3 +// Pandoc-derived partials (.latex) const partialNamesPandoc: string[] = [ "after-header-includes", "common", diff --git a/llm-docs/pandoc-quarto-typst-templates.md b/llm-docs/pandoc-quarto-typst-templates.md index 1d3b2a1aa92..be25e9ce11d 100644 --- a/llm-docs/pandoc-quarto-typst-templates.md +++ b/llm-docs/pandoc-quarto-typst-templates.md @@ -184,19 +184,21 @@ Handles bibliography rendering: - Sets bibliography style from CSL file or bibliography style option - Renders bibliography from specified files -### typst.template and template.typst - Dev-Reference Copies Only +### typst.template and template.typst - Reference Copies -`typst.template` (Pandoc's `default.typst`) and `template.typst` (Pandoc's own `template.typst`, kept under the same name) are **not referenced by any runtime TypeScript code path** — a repo-wide grep for either filename across `src/**/*.ts` returns zero matches. `format-typst.ts` wires up only the `quarto/` partials (see above) as the template context. +`typst.template` (Pandoc's `default.typst`) and `template.typst` (kept under its original name) are reference copies. `format-typst.ts` uses the `quarto/` template and partials described above for rendering. -These two files exist solely so a maintainer running the Pandoc update process (`writePandocTemplates` in `package/src/common/update-pandoc.ts`) can `git diff` them against a fresh Pandoc checkout to see what changed upstream, then manually port the relevant parts into `quarto/definitions.typ` and friends. They are not a "no custom template" fallback path — don't rely on them actually executing for any user-facing render. +During a Pandoc update, review their diffs and port relevant changes to `quarto/definitions.typ` and the other Quarto partials. See `dev-docs/update-pandoc-checklist.md` for the regeneration and reference-copy maintenance procedure. ### Writer-Level Changes vs. Template-Level Changes Not every user-visible difference between Pandoc versions shows up as a diff in these template files. Some behavior lives in Pandoc's own Haskell **typst writer** — the code that converts the Pandoc AST into typst body markup — and is emitted directly into the rendered body regardless of which template (if any) is used. -Example (verified 2026-07-23, Pandoc 3.10.0 → 3.10.1): the typst writer changed how it emits a `HorizontalRule` AST node, from `#horizontalrule` (calling a symbol the template defines) to `#divider()` (typst's own native function, added in typst 0.15). It also changed term-list (definition list) output from wrapping each description in `#block[...]` to emitting it bare, relying on an external `#set terms(hanging-indent: ...)`. Neither change touched `default.typst`/`template.typst` content in a way that mattered for Quarto: Quarto's `quarto/definitions.typ` keeps its own `#let horizontalrule = ...` (still used internally by `notes.typ`) and its own `#show terms.item` rule (which reformats the description regardless of whether Pandoc pre-wrapped it in `#block[...]`), and since Quarto always bundles a recent typst (0.15.1+) where `divider` is a native `std` builtin, `#divider()` in the body just resolves with no local definition needed. +For example, Pandoc 3.10.1 changed `HorizontalRule` output from `#horizontalrule` to `#divider()`, a native function added in Typst 0.15. The bundled Typst version is 0.15.1, so this call needs no local definition. Quarto's `quarto/definitions.typ` still defines `horizontalrule` for use by `notes.typ`. -**Implication for future Pandoc bumps**: a clean `data/templates/` diff between two Pandoc versions is necessary but not sufficient evidence that "nothing changed" for typst output. Confirm by actually rendering a document with both Pandoc versions (e.g. via a `QUARTO_PANDOC` env override against the bundled binary) and inspecting the generated `.typ` body content, not just diffing the template files. +The same release changed definition-list output: descriptions are emitted directly rather than wrapped in `#block[...]`, relying on an external `#set terms(hanging-indent: ...)`. Quarto's `#show terms.item` rule reformats the description in either case. These changes were checked against Pandoc 3.10.0 on 2026-07-23; neither required a template change for Quarto. + +When updating Pandoc, render a document with both versions using `QUARTO_PANDOC` and compare the generated `.typ` body content. Template diffs alone do not show changes in writer output. ## 3. Parameter Summary Table From f8a0a9de846fdaeacc90d6a3c031433c809558c9 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 4 Nov 2026 17:06:03 +0100 Subject: [PATCH 07/11] Expect the Pandoc strict-mismatch note under a CI version override A test-smokes.yml dispatch with pandoc-override-version intentionally runs the suite against a Pandoc newer than the bundled one, while check.ts's strict constraint still names the archived version (Phase 1 keeps it unchanged on purpose). check-versions-no-mismatch.test.ts then failed on the expected "does not strictly match" note, turning a successful full-suite override run red. The pandoc-override action now exports the requested version as QUARTO_TEST_EXPECTED_PANDOC (alongside QUARTO_PANDOC). The test requires the Pandoc note to name exactly that version when set - proving the override binary is live - and keeps the no-mismatch assertion for Dart Sass, Deno, Typst, and for Pandoc on normal runs. Also cover macOS in the action: the QUARTO_PANDOC export steps were Linux/Windows-only, so an override dispatch with a macOS runner would silently test the bundled Pandoc there while the version test (and any rendering assertion) assumed the override. --- .../actions/pandoc-override/action.yml | 18 ++++-- dev-docs/update-pandoc-checklist.md | 2 +- .../check/check-versions-no-mismatch.test.ts | 63 ++++++++++++++++--- 3 files changed, 68 insertions(+), 15 deletions(-) diff --git a/.github/workflows/actions/pandoc-override/action.yml b/.github/workflows/actions/pandoc-override/action.yml index f94dd2dd901..5ec65b78d65 100644 --- a/.github/workflows/actions/pandoc-override/action.yml +++ b/.github/workflows/actions/pandoc-override/action.yml @@ -16,8 +16,8 @@ runs: with: pandoc-version: ${{ inputs.pandoc-version }} - - name: Point QUARTO_PANDOC at override (Linux) - if: runner.os == 'Linux' && inputs.pandoc-version != '' + - name: Point QUARTO_PANDOC at override (Linux, macOS) + if: runner.os != 'Windows' && inputs.pandoc-version != '' shell: bash run: echo "QUARTO_PANDOC=$(command -v pandoc)" >> "$GITHUB_ENV" @@ -27,8 +27,18 @@ runs: run: | "QUARTO_PANDOC=$((Get-Command pandoc).Source)" >> $env:GITHUB_ENV - - name: Confirm Pandoc override is live (Linux) - if: runner.os == 'Linux' && inputs.pandoc-version != '' + # Lets check-versions-no-mismatch.test.ts tell an intentional override + # apart from a real bundled-version mismatch. Passed via env so the + # dispatch input is never interpolated into the shell script. + - name: Record expected Pandoc version for the version-mismatch test + if: inputs.pandoc-version != '' + shell: bash + env: + PANDOC_VERSION: ${{ inputs.pandoc-version }} + run: echo "QUARTO_TEST_EXPECTED_PANDOC=$PANDOC_VERSION" >> "$GITHUB_ENV" + + - name: Confirm Pandoc override is live (Linux, macOS) + if: runner.os != 'Windows' && inputs.pandoc-version != '' shell: bash run: quarto pandoc --version diff --git a/dev-docs/update-pandoc-checklist.md b/dev-docs/update-pandoc-checklist.md index 8edf471b88a..de1e2838b30 100644 --- a/dev-docs/update-pandoc-checklist.md +++ b/dev-docs/update-pandoc-checklist.md @@ -19,7 +19,7 @@ QUARTO_PANDOC=/path/to/pandoc-3.10.1/bin/pandoc \ `writePandocTemplates` overwrites the reference copies and mapped Pandoc partials such as `fonts.latex` and `hypersetup.latex`. Port relevant changes to Quarto's templates using the [file list below](#pandoc-templates). Review generated changes to `format-extension.ts` as well. The same review applies to the full Phase 2 run. -To smoke-test rendering before archival, dispatch `.github/workflows/test-smokes.yml` with `pandoc-override-version` set to the new release tag (e.g. `3.10.1`). The workflow downloads Pandoc from GitHub and uses it via `QUARTO_PANDOC` for that run, without S3 access. Limit `buckets` to the relevant smoke-all directories (e.g. `typst`, `latex`, `table`) for a faster run. +To smoke-test rendering before archival, dispatch `.github/workflows/test-smokes.yml` with `pandoc-override-version` set to the new release tag (e.g. `3.10.1`). The workflow downloads Pandoc from GitHub and uses it via `QUARTO_PANDOC` for that run, without S3 access. Limit `buckets` to the relevant smoke-all directories (e.g. `typst`, `latex`, `table`) for a faster run. A full-suite run also works: the action exports the requested version as `QUARTO_TEST_EXPECTED_PANDOC`, and the `check versions` smoke test then expects Pandoc's strict-mismatch note for exactly that version instead of failing on it. ### Phase 2 - archival (S3 credential holder, before merge) diff --git a/tests/smoke/check/check-versions-no-mismatch.test.ts b/tests/smoke/check/check-versions-no-mismatch.test.ts index b080ab19c0e..02669bc05e3 100644 --- a/tests/smoke/check/check-versions-no-mismatch.test.ts +++ b/tests/smoke/check/check-versions-no-mismatch.test.ts @@ -5,21 +5,64 @@ * */ -import { testQuartoCmd } from "../../test.ts"; +import { testQuartoCmd, type Verify } from "../../test.ts"; import { printsMessage } from "../../verify.ts"; -// If this test fails, it indicates that there is a version mismatch +// If this test fails, it indicates that there is a version mismatch // Check 'src\command\check\check.ts' for the recorded versions, and compare to 'configuration' file + +// A test-smokes.yml dispatch can intentionally run against a Pandoc newer than +// the bundled one (pandoc-override-version input, via QUARTO_PANDOC). The +// pandoc-override action records the requested version in +// QUARTO_TEST_EXPECTED_PANDOC; the strict-mismatch note is then expected for +// Pandoc, and only for exactly that version. +const expectedPandocOverride = Deno.env.get("QUARTO_TEST_EXPECTED_PANDOC"); + +// Mirrors the padding/truncation check.ts applies before its strict +// comparison, so a requested "3.10" matches the reported "3.10.0". +const normalizePandocVersion = (version: string): string => { + const parts = version.split("."); + if (parts.length > 3) { + return parts.slice(0, 3).join("."); + } + return parts.concat(Array(3 - parts.length).fill("0")).join("."); +}; + +// Dart Sass, Deno and Typst are never overridden: no mismatch note for them. +const verifies: Verify[] = [ + printsMessage({ + level: "INFO", + regex: "(Dart Sass|Deno|Typst) version .*does not strictly match", + negate: true, + }), +]; + +if (expectedPandocOverride) { + // The override differs from the bundled version on purpose: the note must + // appear, naming exactly the requested version (anything else means the + // override resolved to the wrong binary). + verifies.push( + printsMessage({ + level: "INFO", + regex: `Pandoc version ${ + normalizePandocVersion(expectedPandocOverride).replaceAll(".", "\\.") + } does not strictly match`, + }), + ); +} else { + verifies.push( + printsMessage({ + level: "INFO", + regex: "Pandoc version .*does not strictly match", + negate: true, + }), + ); +} + testQuartoCmd( "check", ["versions"], - [ - printsMessage({ - level: "INFO", - regex: "does not strictly match", - negate: true // Verify this message does NOT appear - }) - ], + verifies, undefined, - "check versions --strict should not show version mismatches" + "check versions --strict should not show version mismatches", ); From dbafad8dfeea4514a989473d0f7f245c410d487c Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 4 Nov 2026 17:47:07 +0100 Subject: [PATCH 08/11] Accept an override matching the bundled Pandoc, bump tests baseline Follow-up to f8a0a9de8 addressing its re-review: - Dispatching with pandoc-override-version equal to the current release (e.g. 3.10) is a valid run - a plumbing check or a tag typed without the patch component - but check versions reports "Pandoc version 3.10.0: OK" there, not a mismatch note, and requiring the note failed the run. The assertion now accepts either line, as long as it names exactly the requested version; a wrong binary still fails. - The test's new Deno.env.get raises tests.denoCalls from 774 to 775 (counted with the checker's own regex: 0 before, 1 after); accept the increase in the baseline, same as 5f0fc58d4 did for package/src. --- dev-docs/update-pandoc-checklist.md | 2 +- tests/smoke/check/check-versions-no-mismatch.test.ts | 9 +++++---- tools/runtime-boundary-baseline.json | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dev-docs/update-pandoc-checklist.md b/dev-docs/update-pandoc-checklist.md index de1e2838b30..a3726173605 100644 --- a/dev-docs/update-pandoc-checklist.md +++ b/dev-docs/update-pandoc-checklist.md @@ -19,7 +19,7 @@ QUARTO_PANDOC=/path/to/pandoc-3.10.1/bin/pandoc \ `writePandocTemplates` overwrites the reference copies and mapped Pandoc partials such as `fonts.latex` and `hypersetup.latex`. Port relevant changes to Quarto's templates using the [file list below](#pandoc-templates). Review generated changes to `format-extension.ts` as well. The same review applies to the full Phase 2 run. -To smoke-test rendering before archival, dispatch `.github/workflows/test-smokes.yml` with `pandoc-override-version` set to the new release tag (e.g. `3.10.1`). The workflow downloads Pandoc from GitHub and uses it via `QUARTO_PANDOC` for that run, without S3 access. Limit `buckets` to the relevant smoke-all directories (e.g. `typst`, `latex`, `table`) for a faster run. A full-suite run also works: the action exports the requested version as `QUARTO_TEST_EXPECTED_PANDOC`, and the `check versions` smoke test then expects Pandoc's strict-mismatch note for exactly that version instead of failing on it. +To smoke-test rendering before archival, dispatch `.github/workflows/test-smokes.yml` with `pandoc-override-version` set to the new release tag (e.g. `3.10.1`). The workflow downloads Pandoc from GitHub and uses it via `QUARTO_PANDOC` for that run, without S3 access. Limit `buckets` to the relevant smoke-all directories (e.g. `typst`, `latex`, `table`) for a faster run. A full-suite run also works: the action exports the requested version as `QUARTO_TEST_EXPECTED_PANDOC`, and the `check versions` smoke test then verifies Pandoc reports exactly that version (as the strict-mismatch note, or as OK when it equals the bundled constraint) instead of failing on the note. ### Phase 2 - archival (S3 credential holder, before merge) diff --git a/tests/smoke/check/check-versions-no-mismatch.test.ts b/tests/smoke/check/check-versions-no-mismatch.test.ts index 02669bc05e3..0532036ffc0 100644 --- a/tests/smoke/check/check-versions-no-mismatch.test.ts +++ b/tests/smoke/check/check-versions-no-mismatch.test.ts @@ -38,15 +38,16 @@ const verifies: Verify[] = [ ]; if (expectedPandocOverride) { - // The override differs from the bundled version on purpose: the note must - // appear, naming exactly the requested version (anything else means the - // override resolved to the wrong binary). + // The binary check sees must report exactly the requested version: as the + // strict-mismatch note when it differs from the bundled constraint, or as + // the OK line when it equals it (overriding with the current release). + // Any other version means the override resolved to the wrong binary. verifies.push( printsMessage({ level: "INFO", regex: `Pandoc version ${ normalizePandocVersion(expectedPandocOverride).replaceAll(".", "\\.") - } does not strictly match`, + }(: OK| does not strictly match)`, }), ); } else { diff --git a/tools/runtime-boundary-baseline.json b/tools/runtime-boundary-baseline.json index 06886dae228..fb072f47d04 100644 --- a/tools/runtime-boundary-baseline.json +++ b/tools/runtime-boundary-baseline.json @@ -16,7 +16,7 @@ "stdImportFiles": 3 }, "tests": { - "denoCalls": 774, + "denoCalls": 775, "stdImportFiles": 176 } } From 6665318547ed72d36c3f277793b1fb1a7c0c0324 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 4 Nov 2026 19:58:45 +0100 Subject: [PATCH 09/11] Clarify Pandoc update and version-check guidance Describe both accepted override outcomes and the normalized version comparison so the test comments match its assertions. Make the regeneration review instructions explicit and keep the template documentation focused on lasting guidance. --- dev-docs/update-pandoc-checklist.md | 6 ++++-- dev-docs/upgrade-dependencies.md | 2 +- llm-docs/pandoc-quarto-typst-templates.md | 2 +- package/src/common/update-pandoc.ts | 5 ++--- .../check/check-versions-no-mismatch.test.ts | 21 +++++++------------ 5 files changed, 16 insertions(+), 20 deletions(-) diff --git a/dev-docs/update-pandoc-checklist.md b/dev-docs/update-pandoc-checklist.md index a3726173605..bd33f84cb5a 100644 --- a/dev-docs/update-pandoc-checklist.md +++ b/dev-docs/update-pandoc-checklist.md @@ -19,7 +19,9 @@ QUARTO_PANDOC=/path/to/pandoc-3.10.1/bin/pandoc \ `writePandocTemplates` overwrites the reference copies and mapped Pandoc partials such as `fonts.latex` and `hypersetup.latex`. Port relevant changes to Quarto's templates using the [file list below](#pandoc-templates). Review generated changes to `format-extension.ts` as well. The same review applies to the full Phase 2 run. -To smoke-test rendering before archival, dispatch `.github/workflows/test-smokes.yml` with `pandoc-override-version` set to the new release tag (e.g. `3.10.1`). The workflow downloads Pandoc from GitHub and uses it via `QUARTO_PANDOC` for that run, without S3 access. Limit `buckets` to the relevant smoke-all directories (e.g. `typst`, `latex`, `table`) for a faster run. A full-suite run also works: the action exports the requested version as `QUARTO_TEST_EXPECTED_PANDOC`, and the `check versions` smoke test then verifies Pandoc reports exactly that version (as the strict-mismatch note, or as OK when it equals the bundled constraint) instead of failing on the note. +To smoke-test rendering before archival, dispatch `.github/workflows/test-smokes.yml` with `pandoc-override-version` set to the new release tag (e.g. `3.10.1`). The workflow downloads Pandoc from GitHub and uses it via `QUARTO_PANDOC` for that run, without S3 access. Limit `buckets` to the relevant smoke-all directories (e.g. `typst`, `latex`, `table`) for a faster run. + +For a full-suite run, the action sets `QUARTO_TEST_EXPECTED_PANDOC`. The version check accepts either OK or a strict-mismatch note for the requested version, normalized as in `check.ts`. ### Phase 2 - archival (S3 credential holder, before merge) @@ -28,7 +30,7 @@ Whoever holds S3 write credentials does this last, right before merging: - [ ] Bump `configuration`'s `export PANDOC=` line and the `Pandoc` entry in `src/command/check/check.ts`'s `versionConstraints` array to the new version (this constraint tracks the bundled version exactly). - [ ] Ensure archives are upgraded. - [ ] Run `AWS_PROFILE=... ./package/src/quarto-bld update-pandoc PANDOC_VERSION` (without `--skip-archive`) to archive the binary to S3, configure it locally, and regenerate templates/variants. -- [ ] Look at `git diff`, specifically for changes in Pandoc templates, and adjust as needed. As in Phase 1, a diff on an already-patched file means something was missed. +- [ ] Review `git diff` for regenerated reference copies, mapped Pandoc partials, and `format-extension.ts`. Port any remaining upstream changes to Quarto's templates and reapply any Quarto changes overwritten in mapped partials. - [ ] Run the `create-release.yml` dry-run signing/notarization gate (see `dev-docs/upgrade-dependencies.md`), then merge. Quarto's templates share directories with the Pandoc reference copies, but use different names. Review the reference-copy diffs and port relevant changes manually; the templates have diverged too much to use `patch`. diff --git a/dev-docs/upgrade-dependencies.md b/dev-docs/upgrade-dependencies.md index fcebcaf9da6..718eb98a918 100644 --- a/dev-docs/upgrade-dependencies.md +++ b/dev-docs/upgrade-dependencies.md @@ -2,7 +2,7 @@ Change version numbers in `./configuration` to correspond to new versions. Update hardcoded version strings in `src/command/check/check.ts` (the `versionConstraints` array in `checkVersions`) so that they match the new versions in `configuration`. The `configuration` file warns about this in a comment, and `tests/smoke/check/check-versions-no-mismatch.test.ts` fails until they match. The constraint tracks the bundled version exactly, not a minimum floor. -**Pandoc specifically** has its own checklist: [update-pandoc-checklist.md](update-pandoc-checklist.md) - template resync (with a naming gotcha that's easy to miss), schema/lua-types checks, and a `QUARTO_PANDOC`-based CI path for smoke-testing a version before it's archived to S3. +For Pandoc, follow [update-pandoc-checklist.md](update-pandoc-checklist.md) for template updates and filename conventions, schema and Lua type checks, and CI smoke tests using `QUARTO_PANDOC` before S3 archival. ## Verify installer signing & notarization before merging (bundled binaries) diff --git a/llm-docs/pandoc-quarto-typst-templates.md b/llm-docs/pandoc-quarto-typst-templates.md index be25e9ce11d..ceb2b25fa68 100644 --- a/llm-docs/pandoc-quarto-typst-templates.md +++ b/llm-docs/pandoc-quarto-typst-templates.md @@ -196,7 +196,7 @@ Not every user-visible difference between Pandoc versions shows up as a diff in For example, Pandoc 3.10.1 changed `HorizontalRule` output from `#horizontalrule` to `#divider()`, a native function added in Typst 0.15. The bundled Typst version is 0.15.1, so this call needs no local definition. Quarto's `quarto/definitions.typ` still defines `horizontalrule` for use by `notes.typ`. -The same release changed definition-list output: descriptions are emitted directly rather than wrapped in `#block[...]`, relying on an external `#set terms(hanging-indent: ...)`. Quarto's `#show terms.item` rule reformats the description in either case. These changes were checked against Pandoc 3.10.0 on 2026-07-23; neither required a template change for Quarto. +The same release changed definition-list output: descriptions are emitted directly rather than wrapped in `#block[...]`, relying on an external `#set terms(hanging-indent: ...)`. Quarto's `#show terms.item` rule reformats the description in either case. Neither change required a template change for Quarto. When updating Pandoc, render a document with both versions using `QUARTO_PANDOC` and compare the generated `.typ` body content. Template diffs alone do not show changes in writer output. diff --git a/package/src/common/update-pandoc.ts b/package/src/common/update-pandoc.ts index 424bc640cd6..560394ca343 100644 --- a/package/src/common/update-pandoc.ts +++ b/package/src/common/update-pandoc.ts @@ -117,7 +117,6 @@ export function updatePandoc() { if (!skipArchive) { const pandocDependency = pandoc(version); - // Archive to S3 await archiveBinaryDependency(pandocDependency, workingDir); await configureDependency( @@ -130,8 +129,8 @@ export function updatePandoc() { // Templates are downloaded from Pandoc's source archive on GitHub. await writePandocTemplates(configuration, version, workingDir); - // Generate variants (always: uses the resolved Pandoc binary, which - // honors QUARTO_PANDOC) + // Generate format extensions using the resolved Pandoc binary, + // including any QUARTO_PANDOC override. await writeVariants(configuration); }); diff --git a/tests/smoke/check/check-versions-no-mismatch.test.ts b/tests/smoke/check/check-versions-no-mismatch.test.ts index 0532036ffc0..31c646ac6e4 100644 --- a/tests/smoke/check/check-versions-no-mismatch.test.ts +++ b/tests/smoke/check/check-versions-no-mismatch.test.ts @@ -8,14 +8,11 @@ import { testQuartoCmd, type Verify } from "../../test.ts"; import { printsMessage } from "../../verify.ts"; -// If this test fails, it indicates that there is a version mismatch -// Check 'src\command\check\check.ts' for the recorded versions, and compare to 'configuration' file +// For unexpected mismatches, compare versionConstraints in +// src/command/check/check.ts with configuration. -// A test-smokes.yml dispatch can intentionally run against a Pandoc newer than -// the bundled one (pandoc-override-version input, via QUARTO_PANDOC). The -// pandoc-override action records the requested version in -// QUARTO_TEST_EXPECTED_PANDOC; the strict-mismatch note is then expected for -// Pandoc, and only for exactly that version. +// QUARTO_TEST_EXPECTED_PANDOC records an intentional override. Accept either +// OK or a strict-mismatch note for that version, normalized as in check.ts. const expectedPandocOverride = Deno.env.get("QUARTO_TEST_EXPECTED_PANDOC"); // Mirrors the padding/truncation check.ts applies before its strict @@ -28,7 +25,7 @@ const normalizePandocVersion = (version: string): string => { return parts.concat(Array(3 - parts.length).fill("0")).join("."); }; -// Dart Sass, Deno and Typst are never overridden: no mismatch note for them. +// The Pandoc override does not change Dart Sass, Deno, or Typst. const verifies: Verify[] = [ printsMessage({ level: "INFO", @@ -38,10 +35,8 @@ const verifies: Verify[] = [ ]; if (expectedPandocOverride) { - // The binary check sees must report exactly the requested version: as the - // strict-mismatch note when it differs from the bundled constraint, or as - // the OK line when it equals it (overriding with the current release). - // Any other version means the override resolved to the wrong binary. + // Require the requested version, normalized as in check.ts. + // Accept OK or a strict-mismatch note. verifies.push( printsMessage({ level: "INFO", @@ -65,5 +60,5 @@ testQuartoCmd( ["versions"], verifies, undefined, - "check versions --strict should not show version mismatches", + "check versions should report expected dependency versions", ); From 4ef3efffdf2678ec9a4f60050e05c09382d5c833 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 4 Nov 2026 20:03:40 +0100 Subject: [PATCH 10/11] Trim repetitive Pandoc update guidance Keep the dependency rule focused on the detailed checklists and shorten the override descriptions. Remove the test comment that repeats the override contract above. --- .claude/rules/dev-tools/dependency-updates.md | 2 +- .github/workflows/actions/README.md | 2 +- .github/workflows/actions/pandoc-override/action.yml | 2 +- .github/workflows/test-smokes.yml | 2 +- tests/smoke/check/check-versions-no-mismatch.test.ts | 2 -- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.claude/rules/dev-tools/dependency-updates.md b/.claude/rules/dev-tools/dependency-updates.md index eb95c2d4528..3f93ac33054 100644 --- a/.claude/rules/dev-tools/dependency-updates.md +++ b/.claude/rules/dev-tools/dependency-updates.md @@ -7,4 +7,4 @@ paths: # Bundled Dependency Updates -Bumping any bundled binary version starts here: see `dev-docs/upgrade-dependencies.md` for the general procedure (version numbers, `check.ts` constraints, the installer signing/notarization dry-run gate before merging). For Pandoc specifically, `dev-docs/upgrade-dependencies.md` links to `dev-docs/update-pandoc-checklist.md` (template resync, schema/lua-types checks, CI smoke-test path). +When updating a bundled binary, follow `dev-docs/upgrade-dependencies.md`. For Pandoc, also follow `dev-docs/update-pandoc-checklist.md`. diff --git a/.github/workflows/actions/README.md b/.github/workflows/actions/README.md index b0665d2cec1..991c7b7be0f 100644 --- a/.github/workflows/actions/README.md +++ b/.github/workflows/actions/README.md @@ -7,7 +7,7 @@ Reusable steps shared across workflows in this repo. See also `.github/actions/` | [`amplitude`](amplitude/action.yml) | Sends an event to Amplitude. | | [`archive`](archive/action.yml) | Archives a dependency binary to S3. | | [`keychain`](keychain/action.yml) | Configures a P12 certificate into the keychain for signing. | -| [`pandoc-override`](pandoc-override/action.yml) | Installs an unarchived Pandoc release and points quarto at it via `QUARTO_PANDOC`, for smoke-testing a version before it has been archived to S3. | +| [`pandoc-override`](pandoc-override/action.yml) | Installs a Pandoc release for smoke tests before S3 archival and selects it through `QUARTO_PANDOC`. | | [`prevent-rerun`](prevent-rerun/action.yml) | Fails if the workflow is re-run (`github.run_attempt > 1`). | | [`quarto-dev`](quarto-dev/action.yml) | Configures the image for Quarto development (runs `configure.sh`/`.cmd`). | | [`sign-files`](sign-files/action.yml) | Installs and configures the environment to sign files. | diff --git a/.github/workflows/actions/pandoc-override/action.yml b/.github/workflows/actions/pandoc-override/action.yml index 5ec65b78d65..e350d4b8702 100644 --- a/.github/workflows/actions/pandoc-override/action.yml +++ b/.github/workflows/actions/pandoc-override/action.yml @@ -1,5 +1,5 @@ name: "Pandoc Override" -description: "Installs an unarchived Pandoc release and points quarto at it via QUARTO_PANDOC, for smoke-testing a version before it has been archived to S3. No-op unless pandoc-version is set - the archived/configured binary is used as normal otherwise." +description: "Installs a Pandoc release for smoke tests before S3 archival and selects it through QUARTO_PANDOC. If pandoc-version is empty, Quarto uses the configured binary." inputs: pandoc-version: diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 5cbabbc1339..9db1df1b731 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -72,7 +72,7 @@ on: type: string default: "" pandoc-override-version: - description: "Pandoc release tag to download from GitHub and use via QUARTO_PANDOC instead of the archived/configured version (e.g. 3.10.1) - for smoke-testing a version before it has been archived to S3. Leave empty for normal behavior." + description: "Pandoc release tag to download from GitHub for smoke tests before S3 archival (e.g. 3.10.1). Quarto selects it through QUARTO_PANDOC. Leave empty to use the configured binary." required: false type: string default: "" diff --git a/tests/smoke/check/check-versions-no-mismatch.test.ts b/tests/smoke/check/check-versions-no-mismatch.test.ts index 31c646ac6e4..86378e09b44 100644 --- a/tests/smoke/check/check-versions-no-mismatch.test.ts +++ b/tests/smoke/check/check-versions-no-mismatch.test.ts @@ -35,8 +35,6 @@ const verifies: Verify[] = [ ]; if (expectedPandocOverride) { - // Require the requested version, normalized as in check.ts. - // Accept OK or a strict-mismatch note. verifies.push( printsMessage({ level: "INFO", From 0abb19d1d4266f351722162f8f3699d129a75d20 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 4 Nov 2026 20:07:39 +0100 Subject: [PATCH 11/11] Improve flow and concision in Pandoc template guidance Keep the localization example next to its introduction and shorten the writer explanation. Remove a comment that repeats the adjacent version verifier. --- llm-docs/localization-architecture.md | 4 ++-- llm-docs/pandoc-quarto-typst-templates.md | 2 +- tests/smoke/check/check-versions-no-mismatch.test.ts | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/llm-docs/localization-architecture.md b/llm-docs/localization-architecture.md index 96423e413d6..e4d6fe904b9 100644 --- a/llm-docs/localization-architecture.md +++ b/llm-docs/localization-architecture.md @@ -152,12 +152,12 @@ These bypass both Pandoc metadata and Lua filters. The string lands directly in From any Pandoc template — built-in (`template.html`, `template.tex`, `typst-template.typ`), extension partial, or custom user template — every localized key is then accessible as: -> **Naming note**: Citations below refer to Quarto's rendering templates: `template.html` (HTML), `template.tex` (LaTeX/PDF and Beamer), and `quarto/typst-template.typ` (Typst). See `dev-docs/update-pandoc-checklist.md` for the corresponding Pandoc reference copies. - ```pandoc $quarto.language.$ ``` +> **Naming note**: Citations below refer to Quarto's rendering templates: `template.html` (HTML), `template.tex` (LaTeX/PDF and Beamer), and `quarto/typst-template.typ` (Typst). See `dev-docs/update-pandoc-checklist.md` for the corresponding Pandoc reference copies. + For example, `$quarto.language.crossref-ch-prefix$` resolves to `Chapitre` when `lang: fr` is set. Hyphens within the leaf segment are valid Pandoc template-variable syntax (manual: *"Variable names can include letters, numbers, underscores, hyphens, and periods"*). This is the channel of choice for new template consumers because: diff --git a/llm-docs/pandoc-quarto-typst-templates.md b/llm-docs/pandoc-quarto-typst-templates.md index ceb2b25fa68..a94cb9175ed 100644 --- a/llm-docs/pandoc-quarto-typst-templates.md +++ b/llm-docs/pandoc-quarto-typst-templates.md @@ -192,7 +192,7 @@ During a Pandoc update, review their diffs and port relevant changes to `quarto/ ### Writer-Level Changes vs. Template-Level Changes -Not every user-visible difference between Pandoc versions shows up as a diff in these template files. Some behavior lives in Pandoc's own Haskell **typst writer** — the code that converts the Pandoc AST into typst body markup — and is emitted directly into the rendered body regardless of which template (if any) is used. +Pandoc's Typst writer converts the AST into body markup. Changes to its output can affect rendering regardless of the template used. For example, Pandoc 3.10.1 changed `HorizontalRule` output from `#horizontalrule` to `#divider()`, a native function added in Typst 0.15. The bundled Typst version is 0.15.1, so this call needs no local definition. Quarto's `quarto/definitions.typ` still defines `horizontalrule` for use by `notes.typ`. diff --git a/tests/smoke/check/check-versions-no-mismatch.test.ts b/tests/smoke/check/check-versions-no-mismatch.test.ts index 86378e09b44..326eee93ee0 100644 --- a/tests/smoke/check/check-versions-no-mismatch.test.ts +++ b/tests/smoke/check/check-versions-no-mismatch.test.ts @@ -25,7 +25,6 @@ const normalizePandocVersion = (version: string): string => { return parts.concat(Array(3 - parts.length).fill("0")).join("."); }; -// The Pandoc override does not change Dart Sass, Deno, or Typst. const verifies: Verify[] = [ printsMessage({ level: "INFO",