Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 19 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ jobs:
strategy:
fail-fast: false
matrix:
host: [macos-latest, ubuntu-latest]
host: ${{ fromJSON(github.event_name == 'push' && '["macos-latest","ubuntu-latest"]' || '["ubuntu-latest"]') }}
runs-on: ${{ matrix.host }}
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
Expand All @@ -175,19 +175,33 @@ jobs:
deno-version: v2.9.5

- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
if: github.event_name == 'push'
with:
bun-version: 1.4.0

- name: Prepare the worktree and exercise the REPL bench
- name: Prepare the worktree
working-directory: repl bench checkout
run: deno task setup

- name: Check REPL bench tools
working-directory: repl bench checkout
run: |
deno test --allow-all --cached-only --frozen --node-modules-dir=none \
--ignore=examples/repl-bench/tests/driver-exit.test.ts,examples/repl-bench/tests/teardown.test.ts \
examples/repl-bench/tests

- name: Exercise the REPL through a host terminal
if: github.event_name == 'push'
working-directory: repl bench checkout
run: |
deno task setup
deno test --allow-all --cached-only --frozen --node-modules-dir=none examples/repl-bench/tests
deno test --allow-all --cached-only --frozen --node-modules-dir=none \
examples/repl-bench/tests/driver-exit.test.ts \
examples/repl-bench/tests/teardown.test.ts
deno run --allow-all --cached-only --frozen --node-modules-dir=none \
packages/cli/src/deno.ts test examples/repl-bench/journeys

- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: failure() || cancelled()
if: github.event_name == 'push' && (failure() || cancelled())
with:
name: repl-bench-${{ matrix.host }}-attempt-${{ github.run_attempt }}
path: |
Expand Down
6 changes: 4 additions & 2 deletions examples/repl-bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -451,8 +451,10 @@ check. A viewport match can also find the same text in the Draft or Transcript.
Use `<Terminal.Screenshot name="narrow-sidekick" as="image" />` inside an active
bench to save the visible viewport and bind the image path.

The required macOS and Linux bench jobs run from a checkout whose path
contains spaces. Their failure artifacts include saved screenshots.
CI runs the bench tools on pull requests and drives a real terminal on pushes
to `main`. The hosted [testing specification](../../specs/testing-spec.md#ci-topology)
describes the runner matrix. The checkout path contains spaces, and terminal
failure artifacts include saved screenshots.

Run [repl-shared-terminal.test.md](journeys/repl-shared-terminal.test.md) to check these launch transitions in one
test. Start at `120x30`, submit the marker entry, and resize to `72x20` with
Expand Down
3 changes: 3 additions & 0 deletions examples/repl-bench/journeys/repl-bench.test.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,11 @@
<Test name="a resized Sidekick view reopens at its last terminal size">
<ReplBench columns={120} rows={30} expectExit={true} as="closed">
<Cwd as="directory" />
<Repl.Focus.Draft />
<Terminal.WaitForText text="> DRAFT" />
<Terminal.WaitForText text="SIDEKICK" row={1} as="sidekickTab" />
<Terminal.Click at={sidekickTab} />
<Terminal.WaitForText text="> › message the sidekick" />
<ReplBench.Resize columns={72} rows={20} />
<ReplBench.WaitForText text="Entries * Sidekick" />
<ReplBench.WaitForText text="message the sidekick" />
Expand Down
35 changes: 30 additions & 5 deletions scripts/tests/ci-workflow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ interface Step {
interface Strategy {
failFast: unknown;
shards: unknown;
hosts: unknown;
}

interface Job {
Expand Down Expand Up @@ -92,7 +93,7 @@ function steps(value: unknown, label: string): Step[] {
function strategy(value: unknown, label: string): Strategy {
const record = object(value, label);
const matrix = object(record.matrix, `${label}.matrix`);
return { failFast: record["fail-fast"], shards: matrix.shard };
return { failFast: record["fail-fast"], shards: matrix.shard, hosts: matrix.host };
}

function job(value: unknown, label: string): Job {
Expand Down Expand Up @@ -691,12 +692,36 @@ describe("the conditional CI jobs", () => {
});

describe("the dedicated ReplBench job", () => {
it("runs the TypeScript tests outside the runtime shards", function* () {
it("requires the bench tools on each event and runs both hosts on main", function* () {
const bench = (yield* workflow())["repl-bench"];
const commands = bench?.steps.flatMap((step) => step.run?.split("\n") ?? []) ?? [];
expect(commands.some((line) => /^deno test .+ examples\/repl-bench\/tests$/.test(line))).toBe(
true,
const tools = bench?.steps.find((step) => step.name === "Check REPL bench tools");

expect(bench?.if).toBeUndefined();
expect(bench?.strategy?.hosts).toBe(
"${{ fromJSON(github.event_name == 'push' && '[\"macos-latest\",\"ubuntu-latest\"]' || '[\"ubuntu-latest\"]') }}",
);
expect(tools?.if).toBeUndefined();
expect(tools?.run).toContain("deno test ");
expect(tools?.run?.trim().endsWith("examples/repl-bench/tests")).toBe(true);
});

it("runs the tests that drive a host terminal only on pushes to main", function* () {
const bench = (yield* workflow())["repl-bench"];
const tools = bench?.steps.find((step) => step.name === "Check REPL bench tools");
const terminal = bench?.steps.find((step) =>
step.run?.includes("test examples/repl-bench/journeys"),
);
const nativeTests = [
"examples/repl-bench/tests/driver-exit.test.ts",
"examples/repl-bench/tests/teardown.test.ts",
];

expect(terminal?.if).toBe("github.event_name == 'push'");
expect(terminal?.run).toContain("deno test ");
expect(tools?.run).toContain(`--ignore=${nativeTests.join(",")}`);
for (const file of nativeTests) {
expect(terminal?.run).toContain(file);
}
});
});

Expand Down
6 changes: 6 additions & 0 deletions specs/testing-spec.md
Original file line number Diff line number Diff line change
Expand Up @@ -1018,6 +1018,12 @@ out, or unexpectedly skips makes that dependency non-success.

Concurrency inside a shard is not enabled.

The required `repl-bench` job checks the bench tools on Ubuntu for pull
requests. Pushes to `main` run the tools, the driver exit and teardown tests,
and the Markdown terminal journeys on macOS and Ubuntu. Those terminal tests
exercise the host's pseudo-terminal and retain screen and terminal evidence
on failure. `green` requires the job on both events.

### Remeasurement and calibration

Remeasure after any change to the corpus, the exclusions, or a runner command.
Expand Down
Loading