Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 63 additions & 8 deletions test/docs/library-examples.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@

import { describe, expect, test } from "bun:test";
import { spawnSync } from "node:child_process";
import { readFileSync, symlinkSync, writeFileSync } from "node:fs";
import { existsSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { ROOT } from "../root.js";
import { SETTINGS_SCHEMA_PATH } from "../settings-schema.js";
import { SETTINGS_SCHEMA_PATH, schemaNotBuilt } from "../settings-schema.js";
import { withTempDir } from "../temp-dir.js";

const PAGE = "docs/reference/library.md";
Expand Down Expand Up @@ -55,9 +55,13 @@ export function tsFences(markdown: string): Fence[] {
* The fences as one module: only the two package paths are rewritten, on the
* lines they already occupy, so a program line is a page line by offset alone
* and nothing else about a fence is repaired before tsc sees it. The page
* imports each name once, in the first example that uses it.
* imports each name once, in the first example that uses it. `schema` is where
* the package's schema subpath lands.
*/
export function examplesProgram(fences: readonly Fence[]): {
export function examplesProgram(
fences: readonly Fence[],
schema: string,
): {
text: string;
pageLine: (programLine: number) => number;
} {
Expand All @@ -67,7 +71,7 @@ export function examplesProgram(fences: readonly Fence[]): {
for (const fence of fences) {
starts.push(line);
const body = fence.body
.replaceAll(`"${PACKAGE}/settings.schema.json"`, JSON.stringify(SETTINGS_SCHEMA_PATH))
.replaceAll(`"${PACKAGE}/settings.schema.json"`, JSON.stringify(schema))
.replaceAll(`"${PACKAGE}"`, JSON.stringify(ENTRY));
parts.push(body);
line += body.split("\n").length;
Expand All @@ -92,15 +96,39 @@ export function examplesProgram(fences: readonly Fence[]): {
/** `tsc --pretty false` prints `<file>(<line>,<col>): error TS<code>: <message>`, continuation lines indented. */
const DIAGNOSTIC = /^(.+?)\((\d+),(\d+)\): error (TS\d+): (.*)$/;

/**
* Whether the program imports `module` (a static import, a re-export, a dynamic import, each spelled as the page
* spells it: `from`, or `import` and an optional parenthesis, then the quoted path); a string value naming the path
* is not a dependency.
*/
function importsModule(program: string, module: string): boolean {
const quoted = JSON.stringify(module);
for (let at = program.indexOf(quoted); at !== -1; at = program.indexOf(quoted, at + 1)) {
if (/\b(?:from|import)\s*\(?\s*$/.test(program.slice(0, at))) {
return true;
}
}
return false;
}

/**
* Every diagnostic tsc reports for the page's fences, as `<label>:<page line>: TS<code>: <message>`.
* A diagnostic outside the examples file keeps its own path. The compile runs in a temp project
* that symlinks this checkout's node_modules, so `types` and the runtime dependencies resolve
* exactly as they do for src/; the project is removed on every path.
* exactly as they do for src/; the project is removed on every path. A fence that imports the
* schema subpath needs the built schema, so its absence is named before tsc would misreport it as
* a module the page cannot find.
*/
export function compileExamples(markdown: string, label: string): Promise<string[]> {
export function compileExamples(
markdown: string,
label: string,
schema = SETTINGS_SCHEMA_PATH,
): Promise<string[]> {
const fences = tsFences(markdown);
const { text, pageLine } = examplesProgram(fences);
const { text, pageLine } = examplesProgram(fences, schema);
if (importsModule(text, schema) && !existsSync(schema)) {
return Promise.reject(schemaNotBuilt(schema));
}
return withTempDir("gsac-library-examples-", (dir) => {
symlinkSync(join(ROOT, "node_modules"), join(dir, "node_modules"), "dir");
const examples = join(dir, "examples.ts");
Expand Down Expand Up @@ -233,4 +261,31 @@ describe(`${PAGE} examples`, () => {
}
},
);

// The schema subpath resolves to a built file no checkout starts with; a page importing it names the build step,
// and a page that only spells the subpath as a string value compiles without the file.
test("a fence importing the schema subpath on an unbuilt checkout fails naming the build step, not a missing module", () =>
withTempDir("gsac-library-unbuilt-", async (dir) => {
const unbuilt = join(dir, "lib", "settings.schema.json");
const importsSchema = [
"```ts",
'import schema from "@vivswan/github-settings-as-code/settings.schema.json" with { type: "json" };',
"",
"console.log(schema.$schema);",
"```",
"",
].join("\n");
await expect(compileExamples(importsSchema, "page.md", unbuilt)).rejects.toThrow(
/lib\/settings\.schema\.json is not built; run `bun run build:schema`$/,
);
const noSchema = [
"```ts",
'import { SECTION_KEYS } from "@vivswan/github-settings-as-code";',
"",
'console.log(SECTION_KEYS.length, "@vivswan/github-settings-as-code/settings.schema.json");',
"```",
"",
].join("\n");
expect(await compileExamples(noSchema, "page.md", unbuilt)).toEqual([]);
}));
});
80 changes: 42 additions & 38 deletions test/sections/refusal-messages.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ const INVARIANT_PREFIX = "BUG:";
* invariant a "BUG:" message for the developer holding the stack
* DATA_CONSTANTS a named constant holding a key, a pattern piece, or a vocabulary; prose inside one is a message
* DATA_FUNCTIONS a function that assembles regex source
* DATA_ARGUMENTS a data argument of a local factory (`bareRule("creation")`)
* DATA_ARGUMENTS a data argument of a local factory (`bareRule("creation")`, the kind of `knobbedList`)
*/
const VOCABULARY_CALLS: ReadonlySet<string> = new Set([
"enum",
Expand Down Expand Up @@ -145,10 +145,11 @@ const SCHEMA_TWIN_CALLS: ReadonlySet<string> = new Set(["meta", "conditional"]);
const DATA_FUNCTIONS: ReadonlySet<string> = new Set(["render", "codePointEscape"]);

/**
* The local factories and helpers whose string arguments are data (a key name, a rule type, a failure code, a size
* measure, a definition id), by argument index, and those whose string arguments are pieces a longer message is
* composed from (pinned by containment). Auditable by name; a control below fails naming an entry no source declares
* or imports.
* The local factories and helpers whose string arguments are data (a key name, a rule type, a wrapper kind, a
* failure code, a size measure, a definition id, a noun), by argument index, and those whose string arguments are
* pieces a longer message is composed from (pinned by containment). A key or a noun that renders inside a message
* is data too: its pin by containment would hold wherever a test spells the key. Auditable by name; a control below
* fails naming an entry no source declares or imports.
*/
const DATA_ARGUMENTS: Readonly<Record<string, readonly number[]>> = {
actorList: [0, 1],
Expand All @@ -161,6 +162,10 @@ const DATA_ARGUMENTS: Readonly<Record<string, readonly number[]>> = {
boundedString: [1],
sealedSecretConfig: [0],
variableConfig: [0],
wrappedList: [1],
knobbedList: [1],
commitMessageFamily: [0, 1],
githubName: [0],
};

const PIECE_ARGUMENTS: Readonly<Record<string, readonly number[]>> = {
Expand All @@ -171,8 +176,6 @@ const PIECE_ARGUMENTS: Readonly<Record<string, readonly number[]>> = {
duplicateVariableNameIssues: [1],
duplicateSecretNameIssues: [1],
holderError: [0],
commitMessageFamily: [0, 1],
githubName: [0],
renamedKeyError: [0, 1, 2, 3],
};

Expand Down Expand Up @@ -982,6 +985,7 @@ describe("every parse-refusal message a user can read is pinned by a test", () =
'const twin = z.object({}).meta({ anyOf: [{ required: ["contexts"] }] });',
'const pattern = new RegExp("^[a-z]+$"); const names = new Set(["a", "b"]); const ok = !"(|^$".includes(previous);',
'function render(): string { return "[^"; }',
'const knob = knobbedList(entry, "knobbed", (knobs) => knobs);',
'throw new Error("BUG: a planted invariant names no user");',
// messages, every spelling
'export const Word = z.string({ error: "probe" });',
Expand Down Expand Up @@ -1029,7 +1033,7 @@ describe("every parse-refusal message a user can read is pinned by a test", () =
' ...identifiedBy("planted", "name", "planted thing"),',
' noun: "planted noun",',
' validate: (label: string) => [{ path: "[0].key", message: WHY(label) }, { path: "[1].key", message: "probe" }, { path: "[2].key", message: plantedWhy() }, ...malformedListIssues()],',
' shape: loosen(z.object({ title: z.string({ error: "Please declare a title" }) })),',
' shape: open({ title: z.string({ error: "Please declare a title" }) }),',
' closedSurface: { consequence: "the planted body carries only the name", other: { consequence: "drops keys" } },',
' endpoints: { list: { statuses: { 200: "the planted list, which no refusal quotes" } } },',
"};",
Expand Down Expand Up @@ -1066,33 +1070,33 @@ describe("every parse-refusal message a user can read is pinned by a test", () =
);
expect(unpinnedMessages(planted, prose)).toEqual([
'src/sections/planted/schema.ts:13: "a planted directive sentence inside an as const table"',
'src/sections/planted/schema.ts:18: "probe"',
'src/sections/planted/schema.ts:19: "probe2"',
'src/sections/planted/schema.ts:20: "Missing"',
'src/sections/planted/schema.ts:20: "Bad"',
'src/sections/planted/schema.ts:21: "too long"',
'src/sections/planted/schema.ts:22: "probe3"',
'src/sections/planted/schema.ts:22: "a planted table entry read by property"',
'src/sections/planted/schema.ts:24: "probe4"',
'src/sections/planted/schema.ts:25: "probe5"',
'src/sections/planted/schema.ts:27: "probe6"',
'src/sections/planted/schema.ts:28: ": !!!"',
'src/sections/planted/schema.ts:29: "Color required"',
'src/sections/planted/schema.ts:30: "probe7"',
'src/sections/planted/schema.ts:31: "probe8"',
'src/sections/planted/schema.ts:31: "probe9"',
'src/sections/planted/schema.ts:32: "probe10"',
'src/sections/planted/schema.ts:33: ": too short"',
'src/sections/planted/schema.ts:34: "Please "',
'src/sections/planted/schema.ts:34: "enter text"',
'src/sections/planted/schema.ts:35: "probe tagged"',
'src/sections/planted/schema.ts:36: "a planted rule message"',
'src/sections/planted/schema.ts:37: "Please pick a planted state"',
'src/sections/planted/schema.ts:37: "probe lit"',
'src/sections/planted/schema.ts:38: "a planted key holds only "',
'src/sections/planted/schema.ts:38: "; remove the other characters"',
'src/sections/planted/schema.ts:39: "a planted reason for a q escape"',
'src/sections/planted/schema.ts:40: "a planted sentence inside a data constant"',
'src/sections/planted/schema.ts:19: "probe"',
'src/sections/planted/schema.ts:20: "probe2"',
'src/sections/planted/schema.ts:21: "Missing"',
'src/sections/planted/schema.ts:21: "Bad"',
'src/sections/planted/schema.ts:22: "too long"',
'src/sections/planted/schema.ts:23: "probe3"',
'src/sections/planted/schema.ts:23: "a planted table entry read by property"',
'src/sections/planted/schema.ts:25: "probe4"',
'src/sections/planted/schema.ts:26: "probe5"',
'src/sections/planted/schema.ts:28: "probe6"',
'src/sections/planted/schema.ts:29: ": !!!"',
'src/sections/planted/schema.ts:30: "Color required"',
'src/sections/planted/schema.ts:31: "probe7"',
'src/sections/planted/schema.ts:32: "probe8"',
'src/sections/planted/schema.ts:32: "probe9"',
'src/sections/planted/schema.ts:33: "probe10"',
'src/sections/planted/schema.ts:34: ": too short"',
'src/sections/planted/schema.ts:35: "Please "',
'src/sections/planted/schema.ts:35: "enter text"',
'src/sections/planted/schema.ts:36: "probe tagged"',
'src/sections/planted/schema.ts:37: "a planted rule message"',
'src/sections/planted/schema.ts:38: "Please pick a planted state"',
'src/sections/planted/schema.ts:38: "probe lit"',
'src/sections/planted/schema.ts:39: "a planted key holds only "',
'src/sections/planted/schema.ts:39: "; remove the other characters"',
'src/sections/planted/schema.ts:40: "a planted reason for a q escape"',
'src/sections/planted/schema.ts:41: "a planted sentence inside a data constant"',
'src/sections/planted/index.ts:1: "the key "',
'src/sections/planted/index.ts:1: " would silently do nothing on this endpoint"',
'src/sections/planted/index.ts:6: "an optional planted directive"',
Expand Down Expand Up @@ -1155,7 +1159,7 @@ describe("every parse-refusal message a user can read is pinned by a test", () =
"settings:\n planted:\n - key: probe\nexpect:\n outcomes:\n planted: probe2\n",
);
expect(unpinnedMessages([plantedSlice], inputsOnly)).toContain(
'src/sections/planted/schema.ts:18: "probe"',
'src/sections/planted/schema.ts:19: "probe"',
);
const scenario = [
"expect:",
Expand All @@ -1166,8 +1170,8 @@ describe("every parse-refusal message a user can read is pinned by a test", () =
].join("\n");
const pins = testStrings("test/planted/scenarios/planted.yml", scenario);
const left = unpinnedMessages([plantedSlice], pins);
expect(left).not.toContain('src/sections/planted/schema.ts:18: "probe"');
expect(left).not.toContain('src/sections/planted/schema.ts:19: "probe2"');
expect(left).not.toContain('src/sections/planted/schema.ts:19: "probe"');
expect(left).not.toContain('src/sections/planted/schema.ts:20: "probe2"');
});

test("a source that does not parse fails naming it instead of reading as pinned (control)", () => {
Expand Down
13 changes: 11 additions & 2 deletions test/settings-schema.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { existsSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { join, relative } from "node:path";
import { ROOT } from "./root.js";

/** The built, gitignored schema; `bun run test` and `bun run fuzz` run `bun run build:schema` before loading it. */
Expand All @@ -11,11 +11,20 @@ export interface SettingsSchemaFile {
[keyword: string]: unknown;
}

/**
* The one line a test fails with when the built schema is absent, raised before any step that would misreport the
* absence as a defect of its own (tsc's TS2307 against a docs page, ajv's compile error): the generator only writes
* the file into the tree, so a test cannot build it elsewhere and names the build step instead.
*/
export function schemaNotBuilt(path: string): Error {
return new Error(`${relative(ROOT, path)} is not built; run \`bun run build:schema\``);
}

/** Read at run time rather than imported: `tsc` resolves a JSON import, so an import would fail every typecheck on
* a checkout that has not built the file. */
export function readSettingsSchema(): SettingsSchemaFile {
if (!existsSync(SETTINGS_SCHEMA_PATH)) {
throw new Error("lib/settings.schema.json is not built; run `bun run build:schema`");
throw schemaNotBuilt(SETTINGS_SCHEMA_PATH);
}
return JSON.parse(readFileSync(SETTINGS_SCHEMA_PATH, "utf8")) as SettingsSchemaFile;
}
Loading