From 2eb2205bbc73e5c488a82d9c0df964fea549f8e8 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sun, 15 Nov 2026 19:54:35 -0500 Subject: [PATCH] Run host REPL bench tests on main --- .github/workflows/ci.yml | 24 ++++++++++--- examples/repl-bench/README.md | 6 ++-- .../repl-bench/journeys/repl-bench.test.md | 3 ++ scripts/tests/ci-workflow.test.ts | 35 ++++++++++++++++--- specs/testing-spec.md | 6 ++++ 5 files changed, 62 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f0ce8f0f..3b4c35a21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,7 +155,7 @@ jobs: strategy: fail-fast: false matrix: - host: [macos-latest, ubuntu-latest] + host: ${{ fromJSON(github.event_name == 'push' && '["macos-latest","ubuntu-latest"]' || '["ubuntu-latest"]') }} runs-on: ${{ matrix.host }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -175,19 +175,33 @@ jobs: deno-version: v2.9.5 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + if: github.event_name == 'push' with: bun-version: 1.4.0 - - name: Prepare the worktree and exercise the REPL bench + - name: Prepare the worktree + working-directory: repl bench checkout + run: deno task setup + + - name: Check REPL bench tools + working-directory: repl bench checkout + run: | + deno test --allow-all --cached-only --frozen --node-modules-dir=none \ + --ignore=examples/repl-bench/tests/driver-exit.test.ts,examples/repl-bench/tests/teardown.test.ts \ + examples/repl-bench/tests + + - name: Exercise the REPL through a host terminal + if: github.event_name == 'push' working-directory: repl bench checkout run: | - deno task setup - deno test --allow-all --cached-only --frozen --node-modules-dir=none examples/repl-bench/tests + deno test --allow-all --cached-only --frozen --node-modules-dir=none \ + examples/repl-bench/tests/driver-exit.test.ts \ + examples/repl-bench/tests/teardown.test.ts deno run --allow-all --cached-only --frozen --node-modules-dir=none \ packages/cli/src/deno.ts test examples/repl-bench/journeys - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: failure() || cancelled() + if: github.event_name == 'push' && (failure() || cancelled()) with: name: repl-bench-${{ matrix.host }}-attempt-${{ github.run_attempt }} path: | diff --git a/examples/repl-bench/README.md b/examples/repl-bench/README.md index e0af36d95..dc5acb6a0 100644 --- a/examples/repl-bench/README.md +++ b/examples/repl-bench/README.md @@ -451,8 +451,10 @@ check. A viewport match can also find the same text in the Draft or Transcript. Use `` inside an active bench to save the visible viewport and bind the image path. -The required macOS and Linux bench jobs run from a checkout whose path -contains spaces. Their failure artifacts include saved screenshots. +CI runs the bench tools on pull requests and drives a real terminal on pushes +to `main`. The hosted [testing specification](../../specs/testing-spec.md#ci-topology) +describes the runner matrix. The checkout path contains spaces, and terminal +failure artifacts include saved screenshots. Run [repl-shared-terminal.test.md](journeys/repl-shared-terminal.test.md) to check these launch transitions in one test. Start at `120x30`, submit the marker entry, and resize to `72x20` with diff --git a/examples/repl-bench/journeys/repl-bench.test.md b/examples/repl-bench/journeys/repl-bench.test.md index 4492b1829..eab0d3d8b 100644 --- a/examples/repl-bench/journeys/repl-bench.test.md +++ b/examples/repl-bench/journeys/repl-bench.test.md @@ -26,8 +26,11 @@ + + + diff --git a/scripts/tests/ci-workflow.test.ts b/scripts/tests/ci-workflow.test.ts index 20b667b2d..85aadd5f0 100644 --- a/scripts/tests/ci-workflow.test.ts +++ b/scripts/tests/ci-workflow.test.ts @@ -33,6 +33,7 @@ interface Step { interface Strategy { failFast: unknown; shards: unknown; + hosts: unknown; } interface Job { @@ -92,7 +93,7 @@ function steps(value: unknown, label: string): Step[] { function strategy(value: unknown, label: string): Strategy { const record = object(value, label); const matrix = object(record.matrix, `${label}.matrix`); - return { failFast: record["fail-fast"], shards: matrix.shard }; + return { failFast: record["fail-fast"], shards: matrix.shard, hosts: matrix.host }; } function job(value: unknown, label: string): Job { @@ -691,12 +692,36 @@ describe("the conditional CI jobs", () => { }); describe("the dedicated ReplBench job", () => { - it("runs the TypeScript tests outside the runtime shards", function* () { + it("requires the bench tools on each event and runs both hosts on main", function* () { const bench = (yield* workflow())["repl-bench"]; - const commands = bench?.steps.flatMap((step) => step.run?.split("\n") ?? []) ?? []; - expect(commands.some((line) => /^deno test .+ examples\/repl-bench\/tests$/.test(line))).toBe( - true, + const tools = bench?.steps.find((step) => step.name === "Check REPL bench tools"); + + expect(bench?.if).toBeUndefined(); + expect(bench?.strategy?.hosts).toBe( + "${{ fromJSON(github.event_name == 'push' && '[\"macos-latest\",\"ubuntu-latest\"]' || '[\"ubuntu-latest\"]') }}", + ); + expect(tools?.if).toBeUndefined(); + expect(tools?.run).toContain("deno test "); + expect(tools?.run?.trim().endsWith("examples/repl-bench/tests")).toBe(true); + }); + + it("runs the tests that drive a host terminal only on pushes to main", function* () { + const bench = (yield* workflow())["repl-bench"]; + const tools = bench?.steps.find((step) => step.name === "Check REPL bench tools"); + const terminal = bench?.steps.find((step) => + step.run?.includes("test examples/repl-bench/journeys"), ); + const nativeTests = [ + "examples/repl-bench/tests/driver-exit.test.ts", + "examples/repl-bench/tests/teardown.test.ts", + ]; + + expect(terminal?.if).toBe("github.event_name == 'push'"); + expect(terminal?.run).toContain("deno test "); + expect(tools?.run).toContain(`--ignore=${nativeTests.join(",")}`); + for (const file of nativeTests) { + expect(terminal?.run).toContain(file); + } }); }); diff --git a/specs/testing-spec.md b/specs/testing-spec.md index 08879d330..bffc6eb6b 100644 --- a/specs/testing-spec.md +++ b/specs/testing-spec.md @@ -1018,6 +1018,12 @@ out, or unexpectedly skips makes that dependency non-success. Concurrency inside a shard is not enabled. +The required `repl-bench` job checks the bench tools on Ubuntu for pull +requests. Pushes to `main` run the tools, the driver exit and teardown tests, +and the Markdown terminal journeys on macOS and Ubuntu. Those terminal tests +exercise the host's pseudo-terminal and retain screen and terminal evidence +on failure. `green` requires the job on both events. + ### Remeasurement and calibration Remeasure after any change to the corpus, the exclusions, or a runner command.