From 474dc3bae2fae742dd0543941930558cabf3bc2f Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:37:16 -0400 Subject: [PATCH] feat(discovery)!: match exclude patterns with picomatch under an owned glob grammar The exclude discovery filter compiled its own RegExp from each pattern, with "*" the only operator and every other character literal. Each pattern is now compiled once at the input boundary with picomatch (the posix entry, nocase, dot, posix classes, no fast paths, debug), and the boundary owns the grammar: name characters, at most one "/", "*", "?", "[abc]", "[!abc]" or "[^abc]", and one leading "!". Anything else is refused when the inputs are parsed, naming the written pattern, the reason, and the fix, because the pattern is the user's input and because picomatch reads braces, parentheses, quotes, backslashes, a leading "./", and a run of stars as regex syntax through paths no option closes, compiling a wrong or never-matching filter silently. DiscoveryFilters.exclude carries the compiled ExcludePattern (pattern, scope, regex) instead of strings, so a config built from argv still equals one built from the environment, and compileExcludePattern with ExcludePattern join the library surface. The e2e oracle's independent matcher learns "?" and a leading "!", and two scenarios pin the new forms end to end. The upgrading guide gains section 61. BREAKING CHANGE: In an exclude pattern, "?" now matches one character, "[abc]" and "[!abc]" are classes, and a leading "!" negates the pattern; each matched nothing before, so a pattern that relied on one of them matching literally is deleted and one that meant a glob is rewritten in that grammar. A leading "./" (drop it), a run of stars (write one "*"), surrounding whitespace from a library caller (trim it), a "." or ".." side, a second "/", a doubled "!", and any regex token ("(a)+", "\d", "{a..b}", quotes) fail the run naming the fix. Library callers supply exclude as ExcludePattern values from compileExcludePattern instead of strings. --- action.yml | 11 +- bun.lock | 4 + docs/operate/multi-repo.md | 5 +- docs/reference/inputs.md | 2 +- docs/reference/library.md | 2 + docs/upgrading/v2-to-v3.md | 6 + package.json | 2 + src/discovery/discover.ts | 371 +++++++++++++++++- src/flows/input-readers.ts | 12 +- src/flows/inputs.ts | 9 +- src/index.ts | 2 + src/problem.ts | 28 +- src/text.ts | 5 + test/action/run.test.ts | 6 + test/cli/inputs.test.ts | 3 +- test/discovery/discover.test.ts | 344 +++++++++++++++- test/e2e/oracle.ts | 12 +- .../multi-discovery-exclude-negation.yml | 45 +++ .../e2e/scenarios/multi-discovery-exclude.yml | 57 +++ test/private.test.ts | 1 + test/problem.test.ts | 11 +- 21 files changed, 890 insertions(+), 48 deletions(-) create mode 100644 test/e2e/scenarios/multi-discovery-exclude-negation.yml create mode 100644 test/e2e/scenarios/multi-discovery-exclude.yml diff --git a/action.yml b/action.yml index 3c81ef4a3..9ee498411 100644 --- a/action.yml +++ b/action.yml @@ -146,8 +146,15 @@ inputs: default: "" exclude: description: >- - Comma- or newline-separated wildcard patterns removing repositories from - repos: "*" discovery. + Comma- or newline-separated glob patterns removing repositories from + repos: "*" discovery, compared case-insensitively against owner/name + when the pattern has a "/" and against the name alone otherwise. A star + matches any run of characters, "?" one character, "[abc]" one of a set, + "[!abc]" or "[^abc]" one outside it, and one leading "!" negates the + pattern. Beyond those a pattern holds only letters, digits, ".", "-", + and "_", with at most one "/", nothing empty around it, and no side that + is just "." or ".."; anything else (a leading "./", a run of stars, a + regex token such as "(a)+") fails the run naming the fix. required: false default: "" topics: diff --git a/bun.lock b/bun.lock index c33a2715f..ee570f027 100644 --- a/bun.lock +++ b/bun.lock @@ -21,6 +21,7 @@ "consola": "3.4.2", "neverthrow": "8.2.0", "picocolors": "1.1.1", + "picomatch": "4.0.7", "yaml": "2.9.1", "zod": "4.6.5", }, @@ -34,6 +35,7 @@ "@octokit/openapi-webhooks": "12.1.0", "@types/bun": "1.4.2", "@types/node": "26.6.4", + "@types/picomatch": "4.0.3", "action-docs": "2.5.1", "ajv": "8.20.0", "ajv-formats": "3.0.1", @@ -313,6 +315,8 @@ "@types/node": ["@types/node@26.6.4", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-ldVPDCzj7fsaGZrLB0NuHuTvJcsNasysBAqMolr/cgxrLd1xbqxIr3XJiPnHHJUCxj5sNF1vnRj9aWnrVh5Jcg=="], + "@types/picomatch": ["@types/picomatch@4.0.3", "", {}, "sha512-iG0T6+nYJ9FAPmx9SsUlnwcq1ZVRuCXcVEvWnntoPlrOpwtSTKNDC9uVAxTsC3PUvJ+99n4RpAcNgBbHX3JSnQ=="], + "@typescript/typescript-aix-ppc64": ["@typescript/typescript-aix-ppc64@7.0.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ=="], "@typescript/typescript-darwin-arm64": ["@typescript/typescript-darwin-arm64@7.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA=="], diff --git a/docs/operate/multi-repo.md b/docs/operate/multi-repo.md index 372ceaf30..751f7953e 100644 --- a/docs/operate/multi-repo.md +++ b/docs/operate/multi-repo.md @@ -57,7 +57,10 @@ Discovery takes six filter inputs that apply only to `repos: "*"`; setting any o - `archived` defaults to `skip`, because settings writes fail on archived repositories; `archived: only` is mostly useful with `mode: check`. - `forks` includes, excludes, or keeps only forks. - `topics` keeps repositories carrying at least one listed topic, so a single marker topic can opt repositories in. It is unrelated to the `topics` settings section. -- `exclude` takes wildcard patterns where `*` matches anything: a pattern containing `/` is matched against the full `owner/name`, any other against the name alone, case-insensitively. +- `exclude` takes glob patterns, matched case-insensitively: a pattern containing `/` against the full `owner/name`, any other against the name alone. + - `*` matches any run of characters, `?` one character, `[abc]` one of a set, `[!abc]` or `[^abc]` one outside it. + - One leading `!` negates the pattern, so `!svc-*` drops everything except the services. + - Beyond those operators a pattern holds only name characters (letters, digits, `.`, `-`, `_`), at most one `/` with a non-empty glob on each side, and no side that is just `.` or `..`. Anything else (a leading `./`, a run of stars, a brace, a parenthesis, a backslash, a quote, a second `/`, a class holding `*`) fails the run before discovery starts, naming the fix, as does a class like `[z-a]`. The matcher is [picomatch](https://github.com/micromatch/picomatch), confined to that grammar. - `affiliation` selects which relationships to the token's user qualify, passed to GitHub's `/user/repos` listing: `owner` (the default), `collaborator`, or `organization_member`. The list replaces the default, so widening discovery beyond owned repositories takes `owner,collaborator`. ## Fallback for repositories without a settings file diff --git a/docs/reference/inputs.md b/docs/reference/inputs.md index 6fbe40e38..4d8403b6a 100644 --- a/docs/reference/inputs.md +++ b/docs/reference/inputs.md @@ -38,7 +38,7 @@ A list input (`sections`, `required-sections`, `repos`, `exclude`, `topics`, `af | `visibility` |

Keeps only repositories of this visibility in repos: "*" discovery: all (default), public, private, or internal.

| `false` | `""` | | `archived` |

Archived-repository policy for repos: "*" discovery: skip (default), include, or only.

| `false` | `""` | | `forks` |

Fork policy for repos: "*" discovery: include (default), exclude, or only.

| `false` | `""` | -| `exclude` |

Comma- or newline-separated wildcard patterns removing repositories from repos: "*" discovery.

| `false` | `""` | +| `exclude` |

Comma- or newline-separated glob patterns removing repositories from repos: "*" discovery, compared case-insensitively against owner/name when the pattern has a "/" and against the name alone otherwise. A star matches any run of characters, "?" one character, "[abc]" one of a set, "[!abc]" or "[^abc]" one outside it, and one leading "!" negates the pattern. Beyond those a pattern holds only letters, digits, ".", "-", and "_", with at most one "/", nothing empty around it, and no side that is just "." or ".."; anything else (a leading "./", a run of stars, a regex token such as "(a)+") fails the run naming the fix.

| `false` | `""` | | `topics` |

Comma- or newline-separated topics, of which repos: "*" discovery keeps the repositories carrying at least one.

| `false` | `""` | | `affiliation` |

Comma- or newline-separated affiliations for repos: "*" discovery: owner (default), collaborator, or organization_member.

| `false` | `""` | diff --git a/docs/reference/library.md b/docs/reference/library.md index 747c943a7..5309ae7f3 100644 --- a/docs/reference/library.md +++ b/docs/reference/library.md @@ -246,6 +246,8 @@ const renderExitCode = runRender( | `discoverRepos` | function | The repositories a token can see, filtered | | `DiscoveryFilters` | type | The filters: `visibility`, `archived`, `forks`, `affiliation`, `topics`, `exclude` | | `DEFAULT_DISCOVERY_FILTERS` | const | The action's defaults | +| `compileExcludePattern` | function | One `exclude` glob compiled once, or the problem naming why it is unusable and the fix | +| `ExcludePattern` | type | A compiled `exclude` entry: the operator's text, the scope its slash chose, and the regex | | `parseReposInput` | function | The `repos` input form: slugs, or `"*"` for discovery | | `resolveCentralTargets` | function | The `/.yml` files of a repos-dir as targets, with warnings for the files it skipped | | `dedupeTargets` | function | Merge the central and the remote targets, central first; a remote target whose slug a central one already names is dropped with a notice | diff --git a/docs/upgrading/v2-to-v3.md b/docs/upgrading/v2-to-v3.md index e42613dc7..46cdd39f8 100644 --- a/docs/upgrading/v2-to-v3.md +++ b/docs/upgrading/v2-to-v3.md @@ -1281,6 +1281,12 @@ v3 tryList("/repos/o/r/labels", { perPage: 100 }) // re Fix: add `tryList` to every `GitHubClient` double: answer the pages your `tryRequest` route table would (`per_page=N&page=K` from 1, until a page is short) and return their bodies as `{ data: [...] }`; a client over another transport returns each page's body as received, and honors `until` by stopping after the first page it accepts. +## 61. Discovery: `exclude` patterns are globs under an owned grammar + +For every `repos: "*"` workflow that sets `exclude`, and for `@vivswan/github-settings-as-code` consumers building `DiscoveryFilters`. In v2 an `exclude` pattern knew one operator, `*`, and matched every other character literally, so a `?`, a `[`, or a leading `!` could only ever match nothing. In v3 the pattern is a glob matched by picomatch inside a grammar the input boundary owns: `*` any run of characters, `?` one character, `[abc]` one of a set, `[!abc]` or `[^abc]` one outside it, one leading `!` negating the whole pattern, and beyond those only name characters (letters, digits, `.`, `-`, `_`) with at most one `/`; the scope rule is unchanged (a pattern with `/` matches `owner/name`, any other the name alone, case-insensitively). A pattern that gains glob meaning matched nothing in v2 and matches now: `exclude: "!svc-*"` discovered every repository, and now excludes every one whose name is not `svc-*` (`notice: repos: "*" discovery skipped 2 repositories by exclude pattern "!svc-*": o/tooling, o/website`). A pattern outside the grammar (a leading `./`, a run of stars, a `.` or `..` side, a second `/`, a doubled `!!`, a regex token such as `(a)+`) is your input, so it fails the run before discovery starts, naming every flaw and its fix, never repaired: `exclude: "./tmp-**"` matched nothing in v2 and now exits with `error: the "exclude" input pattern "./tmp-**" is not a usable glob: it starts with "./", which names no owner, so drop every leading "./", and it holds a run of stars, which matches no more than one "*" does, so write one "*". Write ...`. + +Fix: delete a pattern that relied on `?`, `[`, `]`, or `!` matching literally, since it never matched a repository; rewrite one that meant a glob in the grammar above, following the fix each refusal names. For library callers, `DiscoveryFilters.exclude` takes compiled `ExcludePattern` values instead of strings: run each pattern through `compileExcludePattern` at your input boundary and pass the `Result`'s value. + ## Order of operations 1. Rename any settings file whose path contains a comma, rename `undeclared` to `_undeclared` in every settings file, and move every other underscore key into a YAML comment; the v2 line accepts the old spellings only, so do all three together with the pin move. diff --git a/package.json b/package.json index 676623401..c04d26ce1 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,7 @@ "consola": "3.4.2", "neverthrow": "8.2.0", "picocolors": "1.1.1", + "picomatch": "4.0.7", "yaml": "2.9.1", "zod": "4.6.5" }, @@ -90,6 +91,7 @@ "@octokit/openapi-webhooks": "12.1.0", "@types/bun": "1.4.2", "@types/node": "26.6.4", + "@types/picomatch": "4.0.3", "action-docs": "2.5.1", "ajv": "8.20.0", "ajv-formats": "3.0.1", diff --git a/src/discovery/discover.ts b/src/discovery/discover.ts index 8ba87960e..bc9c2b70b 100644 --- a/src/discovery/discover.ts +++ b/src/discovery/discover.ts @@ -5,14 +5,15 @@ */ import type { components } from "@octokit/openapi-types"; -import { err, ok, ResultAsync } from "neverthrow"; +import { err, ok, type Result, ResultAsync } from "neverthrow"; +import picomatch from "picomatch"; import type { GitHubClient } from "../github/api.js"; import { classifyApiError } from "../github/api-error.js"; import { classifyVisibility } from "../github/repo-visibility.js"; import { isPrivate, markPrivate, type Private } from "../private.js"; import { revealPrivate } from "../private-open.js"; import type { ProblemOf } from "../problem.js"; -import { countNoun } from "../text.js"; +import { countNoun, quote } from "../text.js"; type DiscoveredRepo = Pick< components["schemas"]["repository"], @@ -63,7 +64,7 @@ export interface DiscoveryFilters { forks: (typeof FORKS_FILTERS)[number]; affiliation: string[]; topics: string[]; - exclude: string[]; + exclude: ExcludePattern[]; } export const DEFAULT_DISCOVERY_FILTERS: DiscoveryFilters = { @@ -75,14 +76,362 @@ export const DEFAULT_DISCOVERY_FILTERS: DiscoveryFilters = { exclude: [], }; -function compileExcludePattern(pattern: string): RegExp { - const escaped = pattern.replace(/[.+^${}()|[\]\\?]/g, "\\$&").replace(/\*/g, ".*"); - return new RegExp(`^${escaped}$`, "i"); +/** + * One `exclude` entry, compiled once at the input boundary. Plain data on purpose: a config built from argv must + * equal one built from the environment, so the matcher is picomatch's RegExp, not a closure over it. + */ +export interface ExcludePattern { + /** The operator's text, for the skip notice. */ + readonly pattern: string; + /** "slug" when the pattern has a "/" and is compared against owner/name, "name" when against the name alone. */ + readonly scope: "slug" | "name"; + readonly regex: RegExp; +} + +/** + * `nocase`: repository names compare case-insensitively on GitHub. `dot`: a leading "." is an ordinary character + * in a repository name (".github"), not the hidden-file marker picomatch skips by default. `posix`: "[!ab]" is a + * negated class, as in a shell, instead of a class holding "!". `fastpaths: false`: the shortcut regexes picomatch + * mints for star-led patterns demand a character after a literal dot, so "*.*" would miss "archive."; the full + * parser keeps "*" any run of characters. `debug`: an invalid class like "[z-a]" throws instead of compiling to a + * never-matching regex. `windows: false`: the main entry otherwise follows the host OS, and a slug is a GitHub + * identifier, not a path, so the matcher is the same on every runner. `literalBrackets: false`: a class compiles + * to the class alone, not an alternation with its literal text, which no name holds; thousands of classes then + * compile in milliseconds instead of reaching the engine's size limit. + */ +const EXCLUDE_GLOB_OPTIONS = { + nocase: true, + dot: true, + posix: true, + fastpaths: false, + debug: true, + windows: false, + literalBrackets: false, +} as const; + +/** Every character no owner or repository name holds and no glob operator here uses. */ +const EXCLUDE_STRAYS = /[^\w.\-/*?[\]!^]/g; + +/** + * A class: "!" or "^" may open it, its members are name characters, and a "-" is a range between two of them or + * a literal first or last. Nothing else, so turning a backwards range around can never pair a new one. + */ +const EXCLUDE_CLASS = /^\[[!^]?(?:-?(?:[\w.](?:-[\w.])?)+-?|-)\]$/; + +/** One side of the "/": name characters, "*", "?", and classes. */ +const EXCLUDE_SEGMENT = new RegExp(`^(?:[\\w.\\-*?]|${EXCLUDE_CLASS.source.slice(1, -1)})+$`); + +/** picomatch refuses a longer pattern; judged here so the refusal names the fix beside the pattern's other flaws. */ +const EXCLUDE_MAX_LENGTH = picomatch.constants.MAX_LENGTH; + +/** `where` of every unpaired bracket: a "[" pairs with the next "]" on its side of the "/", nothing else does. */ +function unpairedBrackets(view: string): number[] { + const unpaired: boolean[] = []; + let open = -1; + for (let i = 0; i < view.length; i++) { + const char = view[i]; + if (char === "[") { + if (open >= 0) { + unpaired[i] = true; + } else { + open = i; + } + } else if (char === "]") { + if (open >= 0) { + open = -1; + } else { + unpaired[i] = true; + } + } else if (char === "/" && open >= 0) { + unpaired[open] = true; + open = -1; + } + } + if (open >= 0) { + unpaired[open] = true; + } + const where: number[] = []; + for (let i = 0; i < view.length; i++) { + if (unpaired[i]) { + where.push(i); + } + } + return where; +} + +/** `where` of every "!" or "^" that neither negates (a "!" first) nor opens a class (right after "["). */ +function misplacedMarks(view: string): number[] { + const where: number[] = []; + for (let i = 0; i < view.length; i++) { + const char = view[i]; + if ((char === "!" || char === "^") && !(i === 0 && char === "!") && view[i - 1] !== "[") { + where.push(i); + } + } + return where; +} + +function without(view: string, where: readonly number[]): string { + const skip = new Set(where); + return view + .split("") + .filter((_, i) => !skip.has(i)) + .join(""); +} + +/** The classes of a view whose brackets pair, in order: each "[" up to the next "]", with where it starts. */ +function classesOf(view: string): Array<{ at: number; text: string }> { + const classes: Array<{ at: number; text: string }> = []; + let open = view.indexOf("["); + while (open >= 0) { + const close = view.indexOf("]", open); + if (close < 0) { + break; + } + classes.push({ at: open, text: view.slice(open, close + 1) }); + open = view.indexOf("[", close); + } + return classes; +} + +/** The members of one class with every literal "-" moved last, the one interior place the grammar refuses it. */ +function withHyphensLast(members: string): string { + const kept: string[] = []; + let literal = false; + for (let i = 0; i < members.length; i++) { + const char = members[i] ?? ""; + const high = members[i + 2] ?? ""; + if (char === "-") { + literal = true; + } else if (members[i + 1] === "-" && /[\w.]/.test(high)) { + kept.push(`${char}-${high}`); + i += 2; + } else { + kept.push(char); + } + } + return `${kept.join("")}${literal ? "-" : ""}`; +} + +const listed = (items: readonly T[], render: (item: T) => string): string => + items.map(render).join(" and "); + +/** A step's result: the view with its fix applied and the sentence naming it; `done` ends the pass. */ +interface Pass { + readonly view: string; + readonly flaws: readonly string[]; + readonly done?: boolean; + /** The view compiled, from the step that probes the engine, so the clean path compiles once. */ + readonly regex?: RegExp; +} + +/** Every step reads only the view the steps before it left, so none can judge the raw pattern. */ +type Step = (view: string) => Pass; + +const unchanged = (view: string): Pass => ({ view, flaws: [] }); + +const applied = (view: string, fixed: string, flaw: string): Pass => + fixed === view ? unchanged(view) : { view: fixed, flaws: [flaw] }; + +const named = (view: string, flaw: string): Pass => ({ view, flaws: [flaw] }); + +/** The classes are rewritten in one walk, so a pattern of many flawed classes costs one pass, not one per class. */ +const rewriteClasses: Step = (view) => { + const flaws: string[] = []; + let rebuilt = ""; + let cursor = 0; + for (const { at, text } of classesOf(view)) { + rebuilt += view.slice(cursor, at); + cursor = at + text.length; + let cls = text; + const rewrite = (to: string, flaw: string): void => { + if (to !== cls) { + flaws.push(flaw); + cls = to; + } + }; + const kept = cls.replace(/[*?]/g, ""); + rewrite(kept, `the class "${cls}" takes name characters only, so write it as "${kept}"`); + if (/^\[[!^]?\]$/.test(cls)) { + rewrite("", `the class "${cls}" matches nothing, so delete it`); + continue; + } + if (!EXCLUDE_CLASS.test(cls)) { + const [, opener = "", members = ""] = /^\[([!^]?)(.*)\]$/.exec(cls) ?? []; + const settled = `[${opener}${withHyphensLast(members)}]`; + rewrite( + settled, + `the class "${cls}" has a "-" where it reads as a range operator, so write it as "${settled}"`, + ); + } + const backwards: string[] = []; + const turned = cls.replace(/([\w.])-([\w.])/g, (range, low: string, high: string) => { + if (low <= high) { + return range; + } + backwards.push(range); + return `${high}-${low}`; + }); + rewrite( + turned, + `the class "${cls}" has the backwards range ${listed(backwards, quote)}, so write it as "${turned}"`, + ); + rebuilt += cls; + } + return { view: rebuilt + view.slice(cursor), flaws }; +}; + +const ENGINE_REFUSAL = + "it compiles to a regular expression the runtime refuses, so shorten it or split it across several patterns"; + +/** + * The grammar is owned here, not left to picomatch, because picomatch accepts far more than a repository name can + * hold ("(a)+" a quantifier, "\d" a digit class, "{az..bz}" a range, "./" silently dropped, "***" losing the + * escape of a later dot) and compiles several such inputs to a wrong or never-matching regex with no option to + * refuse them. The pattern is the user's input, so a flaw is refused naming its fix, never repaired, and every + * flaw of one pattern is named at once, so N flaws cost one run to discover, not N: each step names one fix and + * hands the next step the view with it applied, in the order the sentences take. What is left for the user to + * write is named last with no fix applied: the length cap, an empty body, the sides around a "/", a side that is + * only dots, and a regex the engine itself refuses. + */ +const EXCLUDE_STEPS: readonly Step[] = [ + (view) => applied(view, view.trim(), "it is wrapped in whitespace, so trim it"), + (view) => { + const strays = [...new Set(view.match(EXCLUDE_STRAYS) ?? [])]; + return applied( + view, + view.replace(EXCLUDE_STRAYS, ""), + `it holds ${strays.map(quote).join(", ")}, which no owner or repository name contains, so remove ` + + `${strays.length === 1 ? "it" : "them"}`, + ); + }, + (view) => { + const where = unpairedBrackets(view); + return applied( + view, + without(view, where), + `its ${listed(where, (i) => `${quote(view[i])} at ${i}`)} ${where.length === 1 ? "has" : "have"} no ` + + `partner, so remove ${where.length === 1 ? "it" : "them"}`, + ); + }, + (view) => { + const where = misplacedMarks(view); + const one = where.length === 1; + return applied( + view, + without(view, where), + `its ${listed(where, (i) => `${quote(view[i])} at ${i}`)} neither negate${one ? "s" : ""} ` + + `nor open${one ? "s" : ""} a class, so remove ${one ? "it" : "them"}`, + ); + }, + rewriteClasses, + (view) => + applied( + view, + view.replace(/^(!?)(?:\.\/)+/, "$1"), + `it starts with "./", which names no owner, so drop every leading "./"`, + ), + (view) => + applied( + view, + view.replace(/\*{2,}/g, "*"), + `it holds a run of stars, which matches no more than one "*" does, so write one "*"`, + ), + (view) => + view.length > EXCLUDE_MAX_LENGTH + ? named( + view, + `it is ${view.length} characters long, past the cap of ${EXCLUDE_MAX_LENGTH}, so shorten it`, + ) + : unchanged(view), + (view) => { + const negated = view.startsWith("!"); + const body = negated ? view.slice(1) : view; + return body === "" + ? { view, flaws: [negated ? `"!" has nothing to negate` : "it is empty"], done: true } + : unchanged(view); + }, + (view) => { + const segments = view.replace(/^!/, "").split("/"); + return segments.length > 2 || segments.includes("") + ? named(view, `it takes at most one "/", with a non-empty glob on each side of it`) + : unchanged(view); + }, + (view) => + view + .replace(/^!/, "") + .split("/") + .some((side) => side === "." || side === "..") + ? named(view, `"." and ".." are never names`) + : unchanged(view), + (view) => + view + .replace(/^!/, "") + .split("/") + .every((side) => side === "" || EXCLUDE_SEGMENT.test(side)) + ? unchanged(view) + : named( + view, + `it is not name characters, "*", "?", and classes "[abc]" or "[!abc]" on each side of the "/"`, + ), + (view) => { + if (view.length > EXCLUDE_MAX_LENGTH) { + return unchanged(view); + } + try { + // V8 compiles a RegExp on its first use, so the engine's own refusal (a class too wide for its tables, a + // nesting too deep) surfaces here, in the same refusal as the other flaws. + const regex = picomatch.makeRe(view, EXCLUDE_GLOB_OPTIONS); + regex.test(""); + return { view, flaws: [], regex }; + } catch { + return named(view, ENGINE_REFUSAL); + } + }, +]; + +/** Every flaw of the pattern, and the compiled view when the engine step reached and accepted it. */ +function judgeExcludePattern(pattern: string): { flaws: string[]; regex: RegExp | undefined } { + const flaws: string[] = []; + let view = pattern; + let regex: RegExp | undefined; + for (const step of EXCLUDE_STEPS) { + const next = step(view); + for (const flaw of next.flaws) { + flaws.push(flaw); + } + view = next.view; + regex = next.regex; + if (next.done) { + break; + } + } + return { flaws, regex }; +} + +/** + * The scope split mirrors the repos-dir `.yml` vs `/.yml` pairing; picomatch's own basename + * mode cannot express it (a slashed pattern stops matching the whole slug under it). + */ +export function compileExcludePattern( + pattern: string, +): Result> { + const { flaws, regex } = judgeExcludePattern(pattern); + if (flaws.length > 0) { + return err({ code: "input-exclude-pattern-invalid", pattern, reason: flaws.join(", and ") }); + } + if (regex === undefined) { + throw new Error( + `BUG: the exclude pattern ${quote(pattern)} passed every step but the engine step never ran`, + ); + } + const scope = pattern.includes("/") ? "slug" : "name"; + return ok({ pattern, scope, regex }); } -export function excludeMatches(pattern: string, slug: string): boolean { - const candidate = pattern.includes("/") ? slug : (slug.split("/")[1] ?? slug); - return compileExcludePattern(pattern).test(candidate); +export function excludeMatches(exclude: ExcludePattern, slug: string): boolean { + const candidate = exclude.scope === "slug" ? slug : slug.slice(slug.indexOf("/") + 1); + return exclude.regex.test(candidate); } export interface DiscoveryResult { @@ -188,8 +537,8 @@ function applyFilters( return null; }, (repo) => { - const hit = filters.exclude.find((pattern) => excludeMatches(pattern, repo.full_name)); - return hit ? `exclude pattern "${hit}"` : null; + const hit = filters.exclude.find((exclude) => excludeMatches(exclude, repo.full_name)); + return hit ? `exclude pattern "${hit.pattern}"` : null; }, ]; const kept: DiscoveredRepoRef[] = []; diff --git a/src/flows/input-readers.ts b/src/flows/input-readers.ts index 59b0b1843..6a3c56122 100644 --- a/src/flows/input-readers.ts +++ b/src/flows/input-readers.ts @@ -7,8 +7,10 @@ import { err, ok, type Result, safeTry } from "neverthrow"; import { AFFILIATIONS, ARCHIVED_FILTERS, + compileExcludePattern, DEFAULT_DISCOVERY_FILTERS, type DiscoveryFilters, + type ExcludePattern, FORKS_FILTERS, VISIBILITY_FILTERS, } from "../discovery/discover.js"; @@ -196,13 +198,9 @@ export function readDiscoveryFilters( allowed: AFFILIATIONS, }); } - const exclude = input.list("exclude"); - const unmatchable = exclude.find((pattern) => { - const parts = pattern.split("/"); - return parts.length > 2 || (parts.length === 2 && (!parts[0] || !parts[1])); - }); - if (unmatchable !== undefined) { - return err({ code: "input-exclude-pattern-invalid", pattern: unmatchable }); + const exclude: ExcludePattern[] = []; + for (const pattern of input.list("exclude")) { + exclude.push(yield* compileExcludePattern(pattern)); } const discoveryFilters: DiscoveryFilters = { visibility, diff --git a/src/flows/inputs.ts b/src/flows/inputs.ts index 02f98360a..72607acca 100644 --- a/src/flows/inputs.ts +++ b/src/flows/inputs.ts @@ -181,8 +181,13 @@ export const INPUT_DECLS = { }, exclude: { description: - 'Comma- or newline-separated wildcard patterns removing repositories from repos: "*" ' + - "discovery.", + 'Comma- or newline-separated glob patterns removing repositories from repos: "*" discovery, compared ' + + 'case-insensitively against owner/name when the pattern has a "/" and against the name alone ' + + 'otherwise. A star matches any run of characters, "?" one character, "[abc]" one of a set, "[!abc]" ' + + 'or "[^abc]" one outside it, and one leading "!" negates the pattern. Beyond those a pattern holds ' + + 'only letters, digits, ".", "-", and "_", with at most one "/", nothing empty around it, and no side ' + + 'that is just "." or ".."; anything else (a leading "./", a run of stars, a regex token such as ' + + '"(a)+") fails the run naming the fix.', default: "", list: true, }, diff --git a/src/index.ts b/src/index.ts index 4b93f06d4..6fc989d0b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,9 +6,11 @@ export { resolveCentralTargets } from "./discovery/central.js"; export { + compileExcludePattern, DEFAULT_DISCOVERY_FILTERS, type DiscoveryFilters, discoverRepos, + type ExcludePattern, } from "./discovery/discover.js"; export { parseReposInput } from "./discovery/repos-input.js"; export { diff --git a/src/problem.ts b/src/problem.ts index fe01c8aee..594712f7c 100644 --- a/src/problem.ts +++ b/src/problem.ts @@ -10,7 +10,7 @@ import { describeKind } from "./plain-data.js"; import type { SectionKey } from "./schema.js"; -import { agree, countNoun } from "./text.js"; +import { agree, countNoun, quote } from "./text.js"; /** * The advice appended to a transient (non-permission) API failure: a network @@ -131,7 +131,11 @@ export type Problem = readonly entry: string; readonly allowed: readonly string[]; } - | { readonly code: "input-exclude-pattern-invalid"; readonly pattern: string } + | { + readonly code: "input-exclude-pattern-invalid"; + readonly pattern: string; + readonly reason: string; + } | { readonly code: "input-repository-with-multi" } | { readonly code: "input-settings-file-with-multi" } | { @@ -332,10 +336,6 @@ const PASSTHROUGH_ADVICE = "through, except in closed sections and strict nested objects like actions.cache, which reject " + "unrecognized keys)"; -function quote(value: unknown): string { - return JSON.stringify(String(value)); -} - function describeCentralFile(file: CentralFileProblem): string { switch (file.kind) { case "not-a-slug": @@ -509,12 +509,20 @@ export function describeProblem(problem: Problem): string { return 'the "private-report" input delivers reports only for redacted targets, but "private-repos" is "show", so nothing is redacted and no report would ever be sent. Set private-repos: redact, or set private-report: none'; case "input-affiliation-unsupported": return `the "affiliation" input entry "${problem.entry}" is not a supported affiliation, so discovery cannot build the /user/repos query. Use a comma-separated list of ${quoteList(problem.allowed)}`; - case "input-exclude-pattern-invalid": + case "input-exclude-pattern-invalid": { + const shown = + problem.pattern.length > 60 ? `${problem.pattern.slice(0, 60)}...` : problem.pattern; + // A position names a character in the pattern as the fixes before it leave it, counted from 0. + const positioned = / at \d+/.test(problem.reason) + ? ", applying the fixes in order with positions counted from 0 in the pattern as the earlier fixes leave it" + : ""; return ( - `the "exclude" input pattern "${problem.pattern}" can never match an owner/name repository: a ` + - `pattern takes at most one "/", with a non-empty glob on each side of it. Use ` + - `"" or "/", where "*" matches any characters` + `the "exclude" input pattern ${quote(shown)} is not a usable glob: ${problem.reason}${positioned}. Write ` + + `"" or "/", where a glob is letters, digits, ".", "-", "_", "*", "?", ` + + `and the classes "[abc]", "[!abc]", or "[^abc]" over those characters, with one optional leading "!" ` + + `negating the whole pattern` ); + } case "input-repository-with-multi": return 'the "repository" input cannot be combined with "repos" or "repos-dir"; multi-repo targets come from those inputs. Remove "repository", or remove the multi-repo inputs to stay in single-repo mode'; case "input-settings-file-with-multi": diff --git a/src/text.ts b/src/text.ts index b0fc8782d..7f7468f4c 100644 --- a/src/text.ts +++ b/src/text.ts @@ -7,3 +7,8 @@ export function agree(count: number, one: string, many: string): string { export function countNoun(count: number, one: string, many: string): string { return `${count} ${agree(count, one, many)}`; } + +/** A user-supplied value quoted for a message, so a quote or backslash inside it stays distinguishable. */ +export function quote(value: unknown): string { + return JSON.stringify(String(value)); +} diff --git a/test/action/run.test.ts b/test/action/run.test.ts index cb0bc529d..0a471da31 100644 --- a/test/action/run.test.ts +++ b/test/action/run.test.ts @@ -156,6 +156,7 @@ describe("run in multi-repo mode (env glue)", () => { const EXCLUDE = (pattern: string): Problem => ({ code: "input-exclude-pattern-invalid", pattern, + reason: 'it takes at most one "/", with a non-empty glob on each side of it', }); // A row names the whole Problem where that is the rule, and a message fragment where the whole message would restate // the input's option list or a library's own wording. @@ -180,6 +181,11 @@ describe("run in multi-repo mode (env glue)", () => { ["exclude: a/b/c", { INPUT_REPOS: "*", INPUT_EXCLUDE: "a/b/c" }, EXCLUDE("a/b/c")], ["exclude: octo/", { INPUT_REPOS: "*", INPUT_EXCLUDE: "octo/" }, EXCLUDE("octo/")], ["exclude: /repo", { INPUT_REPOS: "*", INPUT_EXCLUDE: "/repo" }, EXCLUDE("/repo")], + [ + "exclude: svc-[ab", + { INPUT_REPOS: "*", INPUT_EXCLUDE: "svc-[ab" }, + 'the "exclude" input pattern "svc-[ab" is not a usable glob: its "[" at 4 has no partner, so remove it', + ], [ "a filter with an explicit repos list", { INPUT_REPOS: "o/a", INPUT_FORKS: "exclude" }, diff --git a/test/cli/inputs.test.ts b/test/cli/inputs.test.ts index 7d6568c9e..75d3dc017 100644 --- a/test/cli/inputs.test.ts +++ b/test/cli/inputs.test.ts @@ -485,7 +485,8 @@ describe("the per-mode flag split", () => { exclude: { argv: discovery, pair: ["x/*", "tmp-*"], - kept: (cfg) => (cfg.kind === "multi" ? cfg.discoveryFilters.exclude : []), + kept: (cfg) => + cfg.kind === "multi" ? cfg.discoveryFilters.exclude.map((e) => e.pattern) : [], }, topics: { argv: discovery, diff --git a/test/discovery/discover.test.ts b/test/discovery/discover.test.ts index 8ad14066f..4a358da94 100644 --- a/test/discovery/discover.test.ts +++ b/test/discovery/discover.test.ts @@ -1,10 +1,13 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, spyOn, test } from "bun:test"; import { err } from "neverthrow"; +import picomatch from "picomatch"; import { + compileExcludePattern, DEFAULT_DISCOVERY_FILTERS, type DiscoveredRepoRef, type DiscoveryFilters, discoverRepos, + type ExcludePattern, excludeMatches, type FilteredRepoRef, formatSkipNotice, @@ -13,21 +16,348 @@ import { markPrivate } from "../../src/private.js"; import { describeProblem } from "../../src/problem.js"; import { MockApi } from "../mock-api.js"; -describe("excludeMatches", () => { +/** The compiled form of a pattern the test knows to be valid; a refusal is a test bug, named. */ +const compiled = (pattern: string): ExcludePattern => + compileExcludePattern(pattern).match( + (exclude) => exclude, + (problem) => { + throw new Error(describeProblem(problem)); + }, + ); + +describe("compileExcludePattern", () => { test.each<[string, string, string, boolean]>([ ["* spans any characters, anchored at both ends", "tmp-*", "o/tmp-x", true], ["* spans any characters, anchored at both ends", "tmp", "o/tmp-x", false], ["* spans any characters, anchored at both ends", "*-archive", "o/old-archive", true], - ["regex metacharacters are literal", "a.b", "o/a.b", true], - ["regex metacharacters are literal", "a.b", "o/axb", false], + ["* spans a leading dot, which names a repository, not a hidden file", "*", "o/.github", true], + ["a dot is literal", "a.b", "o/a.b", true], + ["a dot is literal", "a.b", "o/axb", false], ["matching is case-insensitive", "TMP-*", "o/tmp-x", true], + ["matching is case-insensitive", "octo/*", "OCTO/x", true], ["a pattern with a slash matches the full slug", "octo/*", "octo/anything", true], ["a pattern with a slash matches the full slug", "octo/*", "example-org/anything", false], ["a pattern without a slash matches the name only", "web*", "weborg/api", false], ["a pattern without a slash matches the name only", "web*", "anyowner/web-x", true], + ["? matches exactly one character", "tmp-?", "o/tmp-1", true], + ["? matches exactly one character", "tmp-?", "o/tmp-10", false], + ["? matches exactly one character", "tmp-?", "o/tmp-", false], + ["[abc] matches one of the set", "svc-[ab]", "o/svc-a", true], + ["[abc] matches one of the set", "svc-[ab]", "o/svc-c", false], + ["[!abc] matches one character outside the set", "svc-[!ab]", "o/svc-c", true], + ["[!abc] matches one character outside the set", "svc-[!ab]", "o/svc-a", false], + ["a leading ! negates the whole pattern", "!svc-*", "o/tooling", true], + ["a leading ! negates the whole pattern", "!svc-*", "o/svc-a", false], + ["a leading ! negates the whole pattern", "!octo/*", "octo/x", false], + ["a leading ! negates the whole pattern", "!octo/*", "other/x", true], + ["* around a literal dot spans an empty run too (no fast path)", "*.*", "o/archive.", true], + ["* around a literal dot spans an empty run too (no fast path)", "*.*", "o/.github", true], + ["* around a literal dot spans an empty run too (no fast path)", "*.*", "o/archive", false], ])("%s: %p against %p is %p", (_rule, pattern, slug, matches) => { - expect(excludeMatches(pattern, slug)).toBe(matches); + expect(excludeMatches(compiled(pattern), slug)).toBe(matches); + }); + + test("the compiled entry keeps the operator's text for the skip notice and the scope its slash chose", () => { + expect(compiled("TMP-*")).toMatchObject({ pattern: "TMP-*", scope: "name" }); + expect(compiled("octo/*")).toMatchObject({ pattern: "octo/*", scope: "slug" }); + }); + + test("an accepted pattern is compiled once: the engine step's regex is the one kept", () => { + const makeRe = spyOn(picomatch, "makeRe"); + try { + expect(compileExcludePattern("svc-[ab]").isOk()).toBe(true); + expect(makeRe).toHaveBeenCalledTimes(1); + } finally { + makeRe.mockRestore(); + } + }); + + // Every flaw of one pattern is named at once, each sentence one fix, in the order the fixes apply. + const DOT_SLASH = 'it starts with "./", which names no owner, so drop every leading "./"'; + const STARS = + 'it holds a run of stars, which matches no more than one "*" does, so write one "*"'; + test.each<[string, string]>([ + ["./tmp-**", `${DOT_SLASH}, and ${STARS}`], + ["!./", `${DOT_SLASH}, and "!" has nothing to negate`], + ["././x", DOT_SLASH], + [ + " ./x/. ", + `it is wrapped in whitespace, so trim it, and ${DOT_SLASH}, and "." and ".." are never names`, + ], + [" ", "it is wrapped in whitespace, so trim it, and it is empty"], + [ + "./***x y", + `it holds " ", which no owner or repository name contains, so remove it, and ${DOT_SLASH}, and ${STARS}`, + ], + ["a/**/c", `${STARS}, and it takes at most one "/", with a non-empty glob on each side of it`], + [ + "[**]", + 'the class "[**]" takes name characters only, so write it as "[]", and the class "[]" matches nothing, ' + + "so delete it, and it is empty", + ], + [ + "svc-(a)+", + 'it holds "(", ")", "+", which no owner or repository name contains, so remove them', + ], + [ + `!./${"a".repeat(65_536)}`, + `${DOT_SLASH}, and it is 65537 characters long, past the cap of 65536, so shorten it`, + ], + [ + `./${"a".repeat(65_536)}[z-a]`, + `the class "[z-a]" has the backwards range "z-a", so write it as "[a-z]", and ${DOT_SLASH}, and it is ` + + "65541 characters long, past the cap of 65536, so shorten it", + ], + [ + "./[z-*]", + `the class "[z-*]" takes name characters only, so write it as "[z-]", and ${DOT_SLASH}`, + ], + [ + "svc-[!z-a0-9]", + 'the class "[!z-a0-9]" has the backwards range "z-a", so write it as "[!a-z0-9]"', + ], + [ + "[0a--]", + 'the class "[0a--]" has a "-" where it reads as a range operator, so write it as "[0a-]"', + ], + ["[-z-a]", 'the class "[-z-a]" has the backwards range "z-a", so write it as "[-a-z]"'], + [ + "[9-0a--]", + 'the class "[9-0a--]" has a "-" where it reads as a range operator, so write it as "[9-0a-]", and ' + + 'the class "[9-0a-]" has the backwards range "9-0", so write it as "[0-9a-]"', + ], + [ + "[z-a_-.]", + 'the class "[z-a_-.]" has the backwards range "z-a" and "_-.", so write it as "[a-z.-_]"', + ], + [ + "!!./tmp-**", + `its "!" at 1 neither negates nor opens a class, so remove it, and ${DOT_SLASH}, and ${STARS}`, + ], + [ + "x/.!", + 'its "!" at 3 neither negates nor opens a class, so remove it, and "." and ".." are never names', + ], + ["[a/b]", 'its "[" at 0 and "]" at 4 have no partner, so remove them'], + ["*[]*", `the class "[]" matches nothing, so delete it, and ${STARS}`], + ["[]./x", `the class "[]" matches nothing, so delete it, and ${DOT_SLASH}`], + ["svc-[ab", 'its "[" at 4 has no partner, so remove it'], + [ + "x[!a", + 'its "[" at 1 has no partner, so remove it, and its "!" at 1 neither negates nor opens a class, so remove it', + ], + ])("%p is refused naming every flaw", (pattern, reason) => { + expect(compileExcludePattern(pattern)).toEqual( + err({ code: "input-exclude-pattern-invalid", pattern, reason }), + ); + }); + + test("a stray quote or backslash is quoted so it stays distinguishable from the punctuation", () => { + expect(compileExcludePattern('svc-"a"')).toEqual( + err({ + code: "input-exclude-pattern-invalid", + pattern: 'svc-"a"', + reason: 'it holds "\\"", which no owner or repository name contains, so remove it', + }), + ); + const slash = compileExcludePattern("svc-\\d"); + expect(slash.isErr() && slash.error.reason).toBe( + 'it holds "\\\\", which no owner or repository name contains, so remove it', + ); + const problem = compileExcludePattern('svc-"a"'); + expect(problem.isErr() && describeProblem(problem.error)).toContain( + 'the "exclude" input pattern "svc-\\"a\\"" is not a usable glob', + ); }); + + // A refusal is a program: each sentence names one fix a machine can apply (so a reader can too), in order, and + // the result is accepted or carries only what the user has to write (an empty body, the sides around a "/", a + // side that is only dots, a shorter pattern), every one of which the first refusal named already. If a + // sentence has no machine fix and is not one of those, it is not a fix, and the applier throws. + const REMAINDERS = [ + /^it is empty$/, + /^"!" has nothing to negate$/, + /^it takes at most one "\/"/, + /^"\." and "\.\." are never names$/, + /^it is \d+ characters long, past the cap/, + /^it compiles to a regular expression the runtime refuses/, + ]; + const isRemainder = (sentence: string): boolean => + REMAINDERS.some((remainder) => remainder.test(sentence)); + const quoted = (text: string): string[] => + [...text.matchAll(/"((?:\\.|[^"\\])*)"/g)].map((m) => JSON.parse(`"${m[1]}"`) as string); + const applyFix = (view: string, sentence: string): string => { + if (sentence.startsWith("it is wrapped in whitespace")) { + return view.trim(); + } + if (sentence.startsWith("it holds a run of stars")) { + return view.replace(/\*{2,}/g, "*"); + } + if (sentence.startsWith("it holds ")) { + const chars = quoted(sentence.slice(0, sentence.indexOf(", which"))); + return view + .split("") + .filter((char) => !chars.includes(char)) + .join(""); + } + if (sentence.startsWith("its ")) { + const where = [...sentence.matchAll(/ at (\d+)/g)].map((m) => Number(m[1])); + return view + .split("") + .filter((_, i) => !where.includes(i)) + .join(""); + } + if (sentence.startsWith('it starts with "./"')) { + return view.replace(/^(!?)(?:\.\/)+/, "$1"); + } + if (sentence.startsWith("the class ")) { + const parts = quoted(sentence); + const cls = parts[0] ?? ""; + return view.replace(cls, sentence.endsWith("so delete it") ? "" : (parts.at(-1) ?? "")); + } + if (isRemainder(sentence)) { + return view; + } + throw new Error(`no machine fix for: ${sentence}`); + }; + const oneRound = (pattern: string): void => { + const first = compileExcludePattern(pattern); + if (first.isOk()) { + return; + } + const sentences = first.error.reason.split(", and "); + const second = compileExcludePattern(sentences.reduce(applyFix, pattern)); + if (second.isErr()) { + for (const left of second.error.reason.split(", and ")) { + const shown = JSON.stringify(pattern).slice(0, 80); + const detail = `${shown} left "${left}" after: ${first.error.reason.slice(0, 400)}`; + expect(isRemainder(left), detail).toBe(true); + expect(sentences, detail).toContain(left); + } + } + }; + test.each([ + "!!./tmp-**", + "x/.!", + "./***x y", + "a/**/c", + "[**]", + "./[z-*]", + "[0a--]", + "[-a---b]", + "!./", + "./", + "x[!a", + "[a/b]", + "[[a]]", + `./${"a".repeat(65_536)}[z-a]`, + `!./${"a".repeat(65_536)}`, + "*[]*", + "[]./x", + `./${"?".repeat(60_000)}`, + ])("the fixes named for %p apply in one round", oneRound); + // Builder controls ran the same generator at 4000 patterns (seed 20261005, with a 65,540-character piece) and + // 50,000 patterns (seed 0x9e3779b9, 96 pieces, up to 12 per pattern); the committed set stays small. + test("the fixes named for 300 generated patterns apply in one round", () => { + const pieces = [ + "tmp", + "svc-a", + ".github", + "*", + "**", + "***", + "?", + "./", + "././", + "!", + "!!", + "^", + "/", + "..", + ".", + "(", + "+", + " ", + "\\", + '"', + "[", + "]", + "[ab]", + "[!ab]", + "[z-a]", + "[a*]", + "[]", + "[!]", + "[0a--]", + "[--a]", + "[-a-]", + "[a-z-9]", + "[[a]]", + "[9-0_-.]", + "-", + "_", + ]; + let seed = 20261005; + const next = (): number => { + seed = (seed * 1103515245 + 12345) % 2147483648; + return seed; + }; + for (let n = 0; n < 300; n++) { + const count = 1 + (next() % 5); + let pattern = ""; + for (let i = 0; i < count; i++) { + pattern += pieces[next() % pieces.length]; + } + oneRound(pattern); + } + }); + + // Every row is a pattern picomatch itself would accept or refuse with its own words, several compiled to a wrong + // or never-matching regex: "(a)+" a quantifier, "\d" a digit class, "{az..bz}" a range, a leading "./" dropped, + // "***a." with the dot unescaped, "!!" an empty body, "[a/b]" a class across the slash, "[z-a]" and the length + // cap thrown as regex errors. A pattern is the user's input, so each is refused naming the fix, never repaired. + test.each<[string, string]>([ + [" tmp-* ", "trim it"], + ["tmp-*\n", "trim it"], + ["svc-(a)+", 'it holds "("'], + ["svc-\\d", 'it holds "\\\\"'], + ["svc-{az..bz}", 'it holds "{"'], + ['svc-"a"', 'it holds "\\""'], + ["a|b", 'it holds "|"'], + ["tmp x", 'it holds " "'], + ["a/b/c", 'at most one "/"'], + ["octo/", 'at most one "/"'], + ["/repo", 'at most one "/"'], + ["./tmp-*", 'drop every leading "./"'], + ["!./svc-*", 'drop every leading "./"'], + ["./", 'drop every leading "./"'], + ["!", "nothing to negate"], + ["!/x", 'at most one "/"'], + ["x/..", '"." and ".." are never names'], + ["x/.", '"." and ".." are never names'], + ["tmp-**", 'write one "*"'], + ["***a.", 'write one "*"'], + ["o/**", 'write one "*"'], + ["!!svc-*", "neither negates nor opens a class"], + ["a!b", "neither negates nor opens a class"], + ["^a", 'its "^" at 0 neither negates nor opens a class'], + ["[a/b]", "have no partner"], + ["svc-[ab", "has no partner"], + ["[]", 'the class "[]" matches nothing, so delete it'], + ["[*]", 'the class "[*]" takes name characters only'], + ["[z-a]", 'has the backwards range "z-a"'], + ["a".repeat(70_000), "past the cap of 65536, so shorten it"], + ])( + "%p is refused with the reason, not thrown or compiled to something else", + (pattern, reason) => { + const result = compileExcludePattern(pattern); + expect(result.isErr()).toBe(true); + if (result.isErr()) { + expect(result.error).toMatchObject({ code: "input-exclude-pattern-invalid", pattern }); + expect(result.error).toHaveProperty("reason", expect.stringContaining(reason)); + } + }, + ); }); describe("discoverRepos", () => { @@ -206,7 +536,7 @@ describe("discoverRepos", () => { data: [{ full_name: "o/keep" }, { full_name: "o/tmp-1" }, { full_name: "octo/keep" }], }, }, - { exclude: ["tmp-*", "octo/*"] }, + { exclude: [compiled("tmp-*"), compiled("octo/*")] }, ); expect(slugs(discovered.repos)).toEqual(["o/keep"]); expect(filteredSlugs(discovered.filtered)).toEqual([ @@ -220,7 +550,7 @@ describe("discoverRepos", () => { { [OWNED]: { data: [{ full_name: "o/tmp-fork", archived: true, fork: true }] }, }, - { forks: "exclude", exclude: ["tmp-*"] }, + { forks: "exclude", exclude: [compiled("tmp-*")] }, ); expect(discovered.repos).toEqual([]); expect(filteredSlugs(discovered.filtered)).toEqual([ diff --git a/test/e2e/oracle.ts b/test/e2e/oracle.ts index 2ba6e1ac4..c7903e84b 100644 --- a/test/e2e/oracle.ts +++ b/test/e2e/oracle.ts @@ -614,11 +614,15 @@ export interface DiscoveryFilters { } /** - * An INDEPENDENT glob matcher, deliberately not src's excludeMatches (a RegExp compile), so a bug in - * either surfaces as a disagreement. A pattern with "/" matches the full slug, otherwise the name - * portion, mirroring the repos-dir .yml vs /.yml split. + * An INDEPENDENT glob matcher, deliberately not src's compileExcludePattern (a picomatch compile), so a bug in + * either surfaces as a disagreement; it covers the forms the scenarios write ("*", "?", a leading "!"), not + * the whole picomatch grammar. A pattern with "/" matches the full slug, otherwise the name portion, + * mirroring the repos-dir .yml vs /.yml split. */ function globMatches(pattern: string, slug: string): boolean { + if (pattern.startsWith("!")) { + return !globMatches(pattern.slice(1), slug); + } const target = (pattern.includes("/") ? slug : (slug.split("/")[1] ?? slug)).toLowerCase(); const pat = pattern.toLowerCase(); let p = 0; @@ -626,7 +630,7 @@ function globMatches(pattern: string, slug: string): boolean { let star = -1; let mark = 0; while (t < target.length) { - if (p < pat.length && (pat[p] === target[t] || pat[p] === "*")) { + if (p < pat.length && (pat[p] === target[t] || pat[p] === "*" || pat[p] === "?")) { if (pat[p] === "*") { star = p; mark = t; diff --git a/test/e2e/scenarios/multi-discovery-exclude-negation.yml b/test/e2e/scenarios/multi-discovery-exclude-negation.yml new file mode 100644 index 000000000..180457f63 --- /dev/null +++ b/test/e2e/scenarios/multi-discovery-exclude-negation.yml @@ -0,0 +1,45 @@ +# Multi-repo repos: "*" discovery with a negated exclude glob. "!svc-*" excludes +# every repository whose name does NOT match svc-*, so the exclude input doubles +# as an allowlist: svc-a and svc-b are kept, the other two are dropped under the +# one negated pattern, and the action issues zero requests to them. +name: multi-discovery-exclude-negation +settings: {} +discovery: + inputs: + exclude: "!svc-*" + pool: + - {slug: e2e-owner/svc-a} + - {slug: e2e-owner/svc-b} + - {slug: e2e-owner/tooling} + - {slug: e2e-owner/website} +repos: + e2e-owner/svc-a: + settings: + labels: + - name: managed + color: "00ff00" + expect: + result: applied + e2e-owner/svc-b: + settings: + labels: + - name: managed + color: "00ff00" + expect: + result: applied +token_permissions: + issues: write + contents: read +expect: + exit_code: 0 + result: applied + repos_result: + e2e-owner/svc-a: applied + e2e-owner/svc-b: applied + stdout_contains: + - 'skipped 2 repositories by exclude pattern "!svc-*": e2e-owner/tooling, e2e-owner/website' + never: + - "GET /repos/e2e-owner/tooling" + - "GET /repos/e2e-owner/website" + requests_contain: + - "GET /user/repos" diff --git a/test/e2e/scenarios/multi-discovery-exclude.yml b/test/e2e/scenarios/multi-discovery-exclude.yml new file mode 100644 index 000000000..833c6d821 --- /dev/null +++ b/test/e2e/scenarios/multi-discovery-exclude.yml @@ -0,0 +1,57 @@ +# Multi-repo repos: "*" discovery with exclude globs. The pool carries two +# managed services, tmp-1, tmp-20, and legacy-old. "tmp-?" matches one +# character after the dash, so it drops tmp-1 and keeps tmp-20, and +# "E2E-Owner/legacy-*" matches the whole owner/name case-insensitively. The +# action must issue zero requests to the two excluded repos, and the skip notice +# names the pattern that fired. +name: multi-discovery-exclude +settings: {} +discovery: + inputs: + exclude: "tmp-?,E2E-Owner/legacy-*" + pool: + - {slug: e2e-owner/svc-a} + - {slug: e2e-owner/svc-b} + - {slug: e2e-owner/tmp-1} + - {slug: e2e-owner/tmp-20} + - {slug: e2e-owner/legacy-old} +repos: + e2e-owner/svc-a: + settings: + labels: + - name: managed + color: "00ff00" + expect: + result: applied + e2e-owner/svc-b: + settings: + labels: + - name: managed + color: "00ff00" + expect: + result: applied + e2e-owner/tmp-20: + settings: + labels: + - name: managed + color: "00ff00" + expect: + result: applied +token_permissions: + issues: write + contents: read +expect: + exit_code: 0 + result: applied + repos_result: + e2e-owner/svc-a: applied + e2e-owner/svc-b: applied + e2e-owner/tmp-20: applied + stdout_contains: + - 'skipped 1 repository by exclude pattern "tmp-?": e2e-owner/tmp-1' + - 'skipped 1 repository by exclude pattern "E2E-Owner/legacy-*": e2e-owner/legacy-old' + never: + - "GET /repos/e2e-owner/tmp-1" + - "GET /repos/e2e-owner/legacy-old" + requests_contain: + - "GET /user/repos" diff --git a/test/private.test.ts b/test/private.test.ts index 239a331ad..cc67caf7a 100644 --- a/test/private.test.ts +++ b/test/private.test.ts @@ -33,6 +33,7 @@ const OPENER_IMPORTERS: Record = { "DEFAULT_DISCOVERY_FILTERS", "FORKS_FILTERS", "VISIBILITY_FILTERS", + "compileExcludePattern", "discoverRepos", "excludeMatches", "formatSkipNotice", diff --git a/test/problem.test.ts b/test/problem.test.ts index 8c1430e8a..b8b76d960 100644 --- a/test/problem.test.ts +++ b/test/problem.test.ts @@ -127,8 +127,15 @@ const SPECIMENS = { 'the "affiliation" input entry "member" is not a supported affiliation, so discovery cannot build the /user/repos query. Use a comma-separated list of "owner", "collaborator", "organization_member"', ], "input-exclude-pattern-invalid": [ - { code: "input-exclude-pattern-invalid", pattern: "a/b/c" }, - 'the "exclude" input pattern "a/b/c" can never match an owner/name repository: a pattern takes at most one "/", with a non-empty glob on each side of it. Use "" or "/", where "*" matches any characters', + { + code: "input-exclude-pattern-invalid", + pattern: "a/b/c", + reason: 'it takes at most one "/", with a non-empty glob on each side of it', + }, + 'the "exclude" input pattern "a/b/c" is not a usable glob: it takes at most one "/", with a non-empty glob ' + + 'on each side of it. Write "" or "/", where a glob is letters, digits, ' + + '".", "-", "_", "*", "?", and the classes "[abc]", "[!abc]", or "[^abc]" over those characters, with one ' + + 'optional leading "!" negating the whole pattern', ], "discovery-filters-without-wildcard": [ { code: "discovery-filters-without-wildcard", filters: ["forks"], targets: "single-repo" },