diff --git a/action.yml b/action.yml index 3c81ef4a3..9ee498411 100644 --- a/action.yml +++ b/action.yml @@ -146,8 +146,15 @@ inputs: default: "" exclude: description: >- - Comma- or newline-separated wildcard patterns removing repositories from - repos: "*" discovery. + Comma- or newline-separated glob patterns removing repositories from + repos: "*" discovery, compared case-insensitively against owner/name + when the pattern has a "/" and against the name alone otherwise. A star + matches any run of characters, "?" one character, "[abc]" one of a set, + "[!abc]" or "[^abc]" one outside it, and one leading "!" negates the + pattern. Beyond those a pattern holds only letters, digits, ".", "-", + and "_", with at most one "/", nothing empty around it, and no side that + is just "." or ".."; anything else (a leading "./", a run of stars, a + regex token such as "(a)+") fails the run naming the fix. required: false default: "" topics: diff --git a/bun.lock b/bun.lock index c33a2715f..ee570f027 100644 --- a/bun.lock +++ b/bun.lock @@ -21,6 +21,7 @@ "consola": "3.4.2", "neverthrow": "8.2.0", "picocolors": "1.1.1", + "picomatch": "4.0.7", "yaml": "2.9.1", "zod": "4.6.5", }, @@ -34,6 +35,7 @@ "@octokit/openapi-webhooks": "12.1.0", "@types/bun": "1.4.2", "@types/node": "26.6.4", + "@types/picomatch": "4.0.3", "action-docs": "2.5.1", "ajv": "8.20.0", "ajv-formats": "3.0.1", @@ -313,6 +315,8 @@ "@types/node": ["@types/node@26.6.4", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-ldVPDCzj7fsaGZrLB0NuHuTvJcsNasysBAqMolr/cgxrLd1xbqxIr3XJiPnHHJUCxj5sNF1vnRj9aWnrVh5Jcg=="], + "@types/picomatch": ["@types/picomatch@4.0.3", "", {}, "sha512-iG0T6+nYJ9FAPmx9SsUlnwcq1ZVRuCXcVEvWnntoPlrOpwtSTKNDC9uVAxTsC3PUvJ+99n4RpAcNgBbHX3JSnQ=="], + "@typescript/typescript-aix-ppc64": ["@typescript/typescript-aix-ppc64@7.0.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ=="], "@typescript/typescript-darwin-arm64": ["@typescript/typescript-darwin-arm64@7.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA=="], diff --git a/docs/operate/multi-repo.md b/docs/operate/multi-repo.md index 372ceaf30..751f7953e 100644 --- a/docs/operate/multi-repo.md +++ b/docs/operate/multi-repo.md @@ -57,7 +57,10 @@ Discovery takes six filter inputs that apply only to `repos: "*"`; setting any o - `archived` defaults to `skip`, because settings writes fail on archived repositories; `archived: only` is mostly useful with `mode: check`. - `forks` includes, excludes, or keeps only forks. - `topics` keeps repositories carrying at least one listed topic, so a single marker topic can opt repositories in. It is unrelated to the `topics` settings section. -- `exclude` takes wildcard patterns where `*` matches anything: a pattern containing `/` is matched against the full `owner/name`, any other against the name alone, case-insensitively. +- `exclude` takes glob patterns, matched case-insensitively: a pattern containing `/` against the full `owner/name`, any other against the name alone. + - `*` matches any run of characters, `?` one character, `[abc]` one of a set, `[!abc]` or `[^abc]` one outside it. + - One leading `!` negates the pattern, so `!svc-*` drops everything except the services. + - Beyond those operators a pattern holds only name characters (letters, digits, `.`, `-`, `_`), at most one `/` with a non-empty glob on each side, and no side that is just `.` or `..`. Anything else (a leading `./`, a run of stars, a brace, a parenthesis, a backslash, a quote, a second `/`, a class holding `*`) fails the run before discovery starts, naming the fix, as does a class like `[z-a]`. The matcher is [picomatch](https://github.com/micromatch/picomatch), confined to that grammar. - `affiliation` selects which relationships to the token's user qualify, passed to GitHub's `/user/repos` listing: `owner` (the default), `collaborator`, or `organization_member`. The list replaces the default, so widening discovery beyond owned repositories takes `owner,collaborator`. ## Fallback for repositories without a settings file diff --git a/docs/reference/inputs.md b/docs/reference/inputs.md index 6fbe40e38..4d8403b6a 100644 --- a/docs/reference/inputs.md +++ b/docs/reference/inputs.md @@ -38,7 +38,7 @@ A list input (`sections`, `required-sections`, `repos`, `exclude`, `topics`, `af | `visibility` |

Keeps only repositories of this visibility in repos: "*" discovery: all (default), public, private, or internal.

| `false` | `""` | | `archived` |

Archived-repository policy for repos: "*" discovery: skip (default), include, or only.

| `false` | `""` | | `forks` |

Fork policy for repos: "*" discovery: include (default), exclude, or only.

| `false` | `""` | -| `exclude` |

Comma- or newline-separated wildcard patterns removing repositories from repos: "*" discovery.

| `false` | `""` | +| `exclude` |

Comma- or newline-separated glob patterns removing repositories from repos: "*" discovery, compared case-insensitively against owner/name when the pattern has a "/" and against the name alone otherwise. A star matches any run of characters, "?" one character, "[abc]" one of a set, "[!abc]" or "[^abc]" one outside it, and one leading "!" negates the pattern. Beyond those a pattern holds only letters, digits, ".", "-", and "_", with at most one "/", nothing empty around it, and no side that is just "." or ".."; anything else (a leading "./", a run of stars, a regex token such as "(a)+") fails the run naming the fix.

| `false` | `""` | | `topics` |

Comma- or newline-separated topics, of which repos: "*" discovery keeps the repositories carrying at least one.

| `false` | `""` | | `affiliation` |

Comma- or newline-separated affiliations for repos: "*" discovery: owner (default), collaborator, or organization_member.

| `false` | `""` | diff --git a/docs/reference/library.md b/docs/reference/library.md index 747c943a7..5309ae7f3 100644 --- a/docs/reference/library.md +++ b/docs/reference/library.md @@ -246,6 +246,8 @@ const renderExitCode = runRender( | `discoverRepos` | function | The repositories a token can see, filtered | | `DiscoveryFilters` | type | The filters: `visibility`, `archived`, `forks`, `affiliation`, `topics`, `exclude` | | `DEFAULT_DISCOVERY_FILTERS` | const | The action's defaults | +| `compileExcludePattern` | function | One `exclude` glob compiled once, or the problem naming why it is unusable and the fix | +| `ExcludePattern` | type | A compiled `exclude` entry: the operator's text, the scope its slash chose, and the regex | | `parseReposInput` | function | The `repos` input form: slugs, or `"*"` for discovery | | `resolveCentralTargets` | function | The `/.yml` files of a repos-dir as targets, with warnings for the files it skipped | | `dedupeTargets` | function | Merge the central and the remote targets, central first; a remote target whose slug a central one already names is dropped with a notice | diff --git a/docs/upgrading/v2-to-v3.md b/docs/upgrading/v2-to-v3.md index e42613dc7..46cdd39f8 100644 --- a/docs/upgrading/v2-to-v3.md +++ b/docs/upgrading/v2-to-v3.md @@ -1281,6 +1281,12 @@ v3 tryList("/repos/o/r/labels", { perPage: 100 }) // re Fix: add `tryList` to every `GitHubClient` double: answer the pages your `tryRequest` route table would (`per_page=N&page=K` from 1, until a page is short) and return their bodies as `{ data: [...] }`; a client over another transport returns each page's body as received, and honors `until` by stopping after the first page it accepts. +## 61. Discovery: `exclude` patterns are globs under an owned grammar + +For every `repos: "*"` workflow that sets `exclude`, and for `@vivswan/github-settings-as-code` consumers building `DiscoveryFilters`. In v2 an `exclude` pattern knew one operator, `*`, and matched every other character literally, so a `?`, a `[`, or a leading `!` could only ever match nothing. In v3 the pattern is a glob matched by picomatch inside a grammar the input boundary owns: `*` any run of characters, `?` one character, `[abc]` one of a set, `[!abc]` or `[^abc]` one outside it, one leading `!` negating the whole pattern, and beyond those only name characters (letters, digits, `.`, `-`, `_`) with at most one `/`; the scope rule is unchanged (a pattern with `/` matches `owner/name`, any other the name alone, case-insensitively). A pattern that gains glob meaning matched nothing in v2 and matches now: `exclude: "!svc-*"` discovered every repository, and now excludes every one whose name is not `svc-*` (`notice: repos: "*" discovery skipped 2 repositories by exclude pattern "!svc-*": o/tooling, o/website`). A pattern outside the grammar (a leading `./`, a run of stars, a `.` or `..` side, a second `/`, a doubled `!!`, a regex token such as `(a)+`) is your input, so it fails the run before discovery starts, naming every flaw and its fix, never repaired: `exclude: "./tmp-**"` matched nothing in v2 and now exits with `error: the "exclude" input pattern "./tmp-**" is not a usable glob: it starts with "./", which names no owner, so drop every leading "./", and it holds a run of stars, which matches no more than one "*" does, so write one "*". Write ...`. + +Fix: delete a pattern that relied on `?`, `[`, `]`, or `!` matching literally, since it never matched a repository; rewrite one that meant a glob in the grammar above, following the fix each refusal names. For library callers, `DiscoveryFilters.exclude` takes compiled `ExcludePattern` values instead of strings: run each pattern through `compileExcludePattern` at your input boundary and pass the `Result`'s value. + ## Order of operations 1. Rename any settings file whose path contains a comma, rename `undeclared` to `_undeclared` in every settings file, and move every other underscore key into a YAML comment; the v2 line accepts the old spellings only, so do all three together with the pin move. diff --git a/package.json b/package.json index 676623401..c04d26ce1 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,7 @@ "consola": "3.4.2", "neverthrow": "8.2.0", "picocolors": "1.1.1", + "picomatch": "4.0.7", "yaml": "2.9.1", "zod": "4.6.5" }, @@ -90,6 +91,7 @@ "@octokit/openapi-webhooks": "12.1.0", "@types/bun": "1.4.2", "@types/node": "26.6.4", + "@types/picomatch": "4.0.3", "action-docs": "2.5.1", "ajv": "8.20.0", "ajv-formats": "3.0.1", diff --git a/src/discovery/discover.ts b/src/discovery/discover.ts index 8ba87960e..bc9c2b70b 100644 --- a/src/discovery/discover.ts +++ b/src/discovery/discover.ts @@ -5,14 +5,15 @@ */ import type { components } from "@octokit/openapi-types"; -import { err, ok, ResultAsync } from "neverthrow"; +import { err, ok, type Result, ResultAsync } from "neverthrow"; +import picomatch from "picomatch"; import type { GitHubClient } from "../github/api.js"; import { classifyApiError } from "../github/api-error.js"; import { classifyVisibility } from "../github/repo-visibility.js"; import { isPrivate, markPrivate, type Private } from "../private.js"; import { revealPrivate } from "../private-open.js"; import type { ProblemOf } from "../problem.js"; -import { countNoun } from "../text.js"; +import { countNoun, quote } from "../text.js"; type DiscoveredRepo = Pick< components["schemas"]["repository"], @@ -63,7 +64,7 @@ export interface DiscoveryFilters { forks: (typeof FORKS_FILTERS)[number]; affiliation: string[]; topics: string[]; - exclude: string[]; + exclude: ExcludePattern[]; } export const DEFAULT_DISCOVERY_FILTERS: DiscoveryFilters = { @@ -75,14 +76,362 @@ export const DEFAULT_DISCOVERY_FILTERS: DiscoveryFilters = { exclude: [], }; -function compileExcludePattern(pattern: string): RegExp { - const escaped = pattern.replace(/[.+^${}()|[\]\\?]/g, "\\$&").replace(/\*/g, ".*"); - return new RegExp(`^${escaped}$`, "i"); +/** + * One `exclude` entry, compiled once at the input boundary. Plain data on purpose: a config built from argv must + * equal one built from the environment, so the matcher is picomatch's RegExp, not a closure over it. + */ +export interface ExcludePattern { + /** The operator's text, for the skip notice. */ + readonly pattern: string; + /** "slug" when the pattern has a "/" and is compared against owner/name, "name" when against the name alone. */ + readonly scope: "slug" | "name"; + readonly regex: RegExp; +} + +/** + * `nocase`: repository names compare case-insensitively on GitHub. `dot`: a leading "." is an ordinary character + * in a repository name (".github"), not the hidden-file marker picomatch skips by default. `posix`: "[!ab]" is a + * negated class, as in a shell, instead of a class holding "!". `fastpaths: false`: the shortcut regexes picomatch + * mints for star-led patterns demand a character after a literal dot, so "*.*" would miss "archive."; the full + * parser keeps "*" any run of characters. `debug`: an invalid class like "[z-a]" throws instead of compiling to a + * never-matching regex. `windows: false`: the main entry otherwise follows the host OS, and a slug is a GitHub + * identifier, not a path, so the matcher is the same on every runner. `literalBrackets: false`: a class compiles + * to the class alone, not an alternation with its literal text, which no name holds; thousands of classes then + * compile in milliseconds instead of reaching the engine's size limit. + */ +const EXCLUDE_GLOB_OPTIONS = { + nocase: true, + dot: true, + posix: true, + fastpaths: false, + debug: true, + windows: false, + literalBrackets: false, +} as const; + +/** Every character no owner or repository name holds and no glob operator here uses. */ +const EXCLUDE_STRAYS = /[^\w.\-/*?[\]!^]/g; + +/** + * A class: "!" or "^" may open it, its members are name characters, and a "-" is a range between two of them or + * a literal first or last. Nothing else, so turning a backwards range around can never pair a new one. + */ +const EXCLUDE_CLASS = /^\[[!^]?(?:-?(?:[\w.](?:-[\w.])?)+-?|-)\]$/; + +/** One side of the "/": name characters, "*", "?", and classes. */ +const EXCLUDE_SEGMENT = new RegExp(`^(?:[\\w.\\-*?]|${EXCLUDE_CLASS.source.slice(1, -1)})+$`); + +/** picomatch refuses a longer pattern; judged here so the refusal names the fix beside the pattern's other flaws. */ +const EXCLUDE_MAX_LENGTH = picomatch.constants.MAX_LENGTH; + +/** `where` of every unpaired bracket: a "[" pairs with the next "]" on its side of the "/", nothing else does. */ +function unpairedBrackets(view: string): number[] { + const unpaired: boolean[] = []; + let open = -1; + for (let i = 0; i < view.length; i++) { + const char = view[i]; + if (char === "[") { + if (open >= 0) { + unpaired[i] = true; + } else { + open = i; + } + } else if (char === "]") { + if (open >= 0) { + open = -1; + } else { + unpaired[i] = true; + } + } else if (char === "/" && open >= 0) { + unpaired[open] = true; + open = -1; + } + } + if (open >= 0) { + unpaired[open] = true; + } + const where: number[] = []; + for (let i = 0; i < view.length; i++) { + if (unpaired[i]) { + where.push(i); + } + } + return where; +} + +/** `where` of every "!" or "^" that neither negates (a "!" first) nor opens a class (right after "["). */ +function misplacedMarks(view: string): number[] { + const where: number[] = []; + for (let i = 0; i < view.length; i++) { + const char = view[i]; + if ((char === "!" || char === "^") && !(i === 0 && char === "!") && view[i - 1] !== "[") { + where.push(i); + } + } + return where; +} + +function without(view: string, where: readonly number[]): string { + const skip = new Set(where); + return view + .split("") + .filter((_, i) => !skip.has(i)) + .join(""); +} + +/** The classes of a view whose brackets pair, in order: each "[" up to the next "]", with where it starts. */ +function classesOf(view: string): Array<{ at: number; text: string }> { + const classes: Array<{ at: number; text: string }> = []; + let open = view.indexOf("["); + while (open >= 0) { + const close = view.indexOf("]", open); + if (close < 0) { + break; + } + classes.push({ at: open, text: view.slice(open, close + 1) }); + open = view.indexOf("[", close); + } + return classes; +} + +/** The members of one class with every literal "-" moved last, the one interior place the grammar refuses it. */ +function withHyphensLast(members: string): string { + const kept: string[] = []; + let literal = false; + for (let i = 0; i < members.length; i++) { + const char = members[i] ?? ""; + const high = members[i + 2] ?? ""; + if (char === "-") { + literal = true; + } else if (members[i + 1] === "-" && /[\w.]/.test(high)) { + kept.push(`${char}-${high}`); + i += 2; + } else { + kept.push(char); + } + } + return `${kept.join("")}${literal ? "-" : ""}`; +} + +const listed = (items: readonly T[], render: (item: T) => string): string => + items.map(render).join(" and "); + +/** A step's result: the view with its fix applied and the sentence naming it; `done` ends the pass. */ +interface Pass { + readonly view: string; + readonly flaws: readonly string[]; + readonly done?: boolean; + /** The view compiled, from the step that probes the engine, so the clean path compiles once. */ + readonly regex?: RegExp; +} + +/** Every step reads only the view the steps before it left, so none can judge the raw pattern. */ +type Step = (view: string) => Pass; + +const unchanged = (view: string): Pass => ({ view, flaws: [] }); + +const applied = (view: string, fixed: string, flaw: string): Pass => + fixed === view ? unchanged(view) : { view: fixed, flaws: [flaw] }; + +const named = (view: string, flaw: string): Pass => ({ view, flaws: [flaw] }); + +/** The classes are rewritten in one walk, so a pattern of many flawed classes costs one pass, not one per class. */ +const rewriteClasses: Step = (view) => { + const flaws: string[] = []; + let rebuilt = ""; + let cursor = 0; + for (const { at, text } of classesOf(view)) { + rebuilt += view.slice(cursor, at); + cursor = at + text.length; + let cls = text; + const rewrite = (to: string, flaw: string): void => { + if (to !== cls) { + flaws.push(flaw); + cls = to; + } + }; + const kept = cls.replace(/[*?]/g, ""); + rewrite(kept, `the class "${cls}" takes name characters only, so write it as "${kept}"`); + if (/^\[[!^]?\]$/.test(cls)) { + rewrite("", `the class "${cls}" matches nothing, so delete it`); + continue; + } + if (!EXCLUDE_CLASS.test(cls)) { + const [, opener = "", members = ""] = /^\[([!^]?)(.*)\]$/.exec(cls) ?? []; + const settled = `[${opener}${withHyphensLast(members)}]`; + rewrite( + settled, + `the class "${cls}" has a "-" where it reads as a range operator, so write it as "${settled}"`, + ); + } + const backwards: string[] = []; + const turned = cls.replace(/([\w.])-([\w.])/g, (range, low: string, high: string) => { + if (low <= high) { + return range; + } + backwards.push(range); + return `${high}-${low}`; + }); + rewrite( + turned, + `the class "${cls}" has the backwards range ${listed(backwards, quote)}, so write it as "${turned}"`, + ); + rebuilt += cls; + } + return { view: rebuilt + view.slice(cursor), flaws }; +}; + +const ENGINE_REFUSAL = + "it compiles to a regular expression the runtime refuses, so shorten it or split it across several patterns"; + +/** + * The grammar is owned here, not left to picomatch, because picomatch accepts far more than a repository name can + * hold ("(a)+" a quantifier, "\d" a digit class, "{az..bz}" a range, "./" silently dropped, "***" losing the + * escape of a later dot) and compiles several such inputs to a wrong or never-matching regex with no option to + * refuse them. The pattern is the user's input, so a flaw is refused naming its fix, never repaired, and every + * flaw of one pattern is named at once, so N flaws cost one run to discover, not N: each step names one fix and + * hands the next step the view with it applied, in the order the sentences take. What is left for the user to + * write is named last with no fix applied: the length cap, an empty body, the sides around a "/", a side that is + * only dots, and a regex the engine itself refuses. + */ +const EXCLUDE_STEPS: readonly Step[] = [ + (view) => applied(view, view.trim(), "it is wrapped in whitespace, so trim it"), + (view) => { + const strays = [...new Set(view.match(EXCLUDE_STRAYS) ?? [])]; + return applied( + view, + view.replace(EXCLUDE_STRAYS, ""), + `it holds ${strays.map(quote).join(", ")}, which no owner or repository name contains, so remove ` + + `${strays.length === 1 ? "it" : "them"}`, + ); + }, + (view) => { + const where = unpairedBrackets(view); + return applied( + view, + without(view, where), + `its ${listed(where, (i) => `${quote(view[i])} at ${i}`)} ${where.length === 1 ? "has" : "have"} no ` + + `partner, so remove ${where.length === 1 ? "it" : "them"}`, + ); + }, + (view) => { + const where = misplacedMarks(view); + const one = where.length === 1; + return applied( + view, + without(view, where), + `its ${listed(where, (i) => `${quote(view[i])} at ${i}`)} neither negate${one ? "s" : ""} ` + + `nor open${one ? "s" : ""} a class, so remove ${one ? "it" : "them"}`, + ); + }, + rewriteClasses, + (view) => + applied( + view, + view.replace(/^(!?)(?:\.\/)+/, "$1"), + `it starts with "./", which names no owner, so drop every leading "./"`, + ), + (view) => + applied( + view, + view.replace(/\*{2,}/g, "*"), + `it holds a run of stars, which matches no more than one "*" does, so write one "*"`, + ), + (view) => + view.length > EXCLUDE_MAX_LENGTH + ? named( + view, + `it is ${view.length} characters long, past the cap of ${EXCLUDE_MAX_LENGTH}, so shorten it`, + ) + : unchanged(view), + (view) => { + const negated = view.startsWith("!"); + const body = negated ? view.slice(1) : view; + return body === "" + ? { view, flaws: [negated ? `"!" has nothing to negate` : "it is empty"], done: true } + : unchanged(view); + }, + (view) => { + const segments = view.replace(/^!/, "").split("/"); + return segments.length > 2 || segments.includes("") + ? named(view, `it takes at most one "/", with a non-empty glob on each side of it`) + : unchanged(view); + }, + (view) => + view + .replace(/^!/, "") + .split("/") + .some((side) => side === "." || side === "..") + ? named(view, `"." and ".." are never names`) + : unchanged(view), + (view) => + view + .replace(/^!/, "") + .split("/") + .every((side) => side === "" || EXCLUDE_SEGMENT.test(side)) + ? unchanged(view) + : named( + view, + `it is not name characters, "*", "?", and classes "[abc]" or "[!abc]" on each side of the "/"`, + ), + (view) => { + if (view.length > EXCLUDE_MAX_LENGTH) { + return unchanged(view); + } + try { + // V8 compiles a RegExp on its first use, so the engine's own refusal (a class too wide for its tables, a + // nesting too deep) surfaces here, in the same refusal as the other flaws. + const regex = picomatch.makeRe(view, EXCLUDE_GLOB_OPTIONS); + regex.test(""); + return { view, flaws: [], regex }; + } catch { + return named(view, ENGINE_REFUSAL); + } + }, +]; + +/** Every flaw of the pattern, and the compiled view when the engine step reached and accepted it. */ +function judgeExcludePattern(pattern: string): { flaws: string[]; regex: RegExp | undefined } { + const flaws: string[] = []; + let view = pattern; + let regex: RegExp | undefined; + for (const step of EXCLUDE_STEPS) { + const next = step(view); + for (const flaw of next.flaws) { + flaws.push(flaw); + } + view = next.view; + regex = next.regex; + if (next.done) { + break; + } + } + return { flaws, regex }; +} + +/** + * The scope split mirrors the repos-dir `.yml` vs `/.yml` pairing; picomatch's own basename + * mode cannot express it (a slashed pattern stops matching the whole slug under it). + */ +export function compileExcludePattern( + pattern: string, +): Result> { + const { flaws, regex } = judgeExcludePattern(pattern); + if (flaws.length > 0) { + return err({ code: "input-exclude-pattern-invalid", pattern, reason: flaws.join(", and ") }); + } + if (regex === undefined) { + throw new Error( + `BUG: the exclude pattern ${quote(pattern)} passed every step but the engine step never ran`, + ); + } + const scope = pattern.includes("/") ? "slug" : "name"; + return ok({ pattern, scope, regex }); } -export function excludeMatches(pattern: string, slug: string): boolean { - const candidate = pattern.includes("/") ? slug : (slug.split("/")[1] ?? slug); - return compileExcludePattern(pattern).test(candidate); +export function excludeMatches(exclude: ExcludePattern, slug: string): boolean { + const candidate = exclude.scope === "slug" ? slug : slug.slice(slug.indexOf("/") + 1); + return exclude.regex.test(candidate); } export interface DiscoveryResult { @@ -188,8 +537,8 @@ function applyFilters( return null; }, (repo) => { - const hit = filters.exclude.find((pattern) => excludeMatches(pattern, repo.full_name)); - return hit ? `exclude pattern "${hit}"` : null; + const hit = filters.exclude.find((exclude) => excludeMatches(exclude, repo.full_name)); + return hit ? `exclude pattern "${hit.pattern}"` : null; }, ]; const kept: DiscoveredRepoRef[] = []; diff --git a/src/flows/input-readers.ts b/src/flows/input-readers.ts index 59b0b1843..6a3c56122 100644 --- a/src/flows/input-readers.ts +++ b/src/flows/input-readers.ts @@ -7,8 +7,10 @@ import { err, ok, type Result, safeTry } from "neverthrow"; import { AFFILIATIONS, ARCHIVED_FILTERS, + compileExcludePattern, DEFAULT_DISCOVERY_FILTERS, type DiscoveryFilters, + type ExcludePattern, FORKS_FILTERS, VISIBILITY_FILTERS, } from "../discovery/discover.js"; @@ -196,13 +198,9 @@ export function readDiscoveryFilters( allowed: AFFILIATIONS, }); } - const exclude = input.list("exclude"); - const unmatchable = exclude.find((pattern) => { - const parts = pattern.split("/"); - return parts.length > 2 || (parts.length === 2 && (!parts[0] || !parts[1])); - }); - if (unmatchable !== undefined) { - return err({ code: "input-exclude-pattern-invalid", pattern: unmatchable }); + const exclude: ExcludePattern[] = []; + for (const pattern of input.list("exclude")) { + exclude.push(yield* compileExcludePattern(pattern)); } const discoveryFilters: DiscoveryFilters = { visibility, diff --git a/src/flows/inputs.ts b/src/flows/inputs.ts index 02f98360a..72607acca 100644 --- a/src/flows/inputs.ts +++ b/src/flows/inputs.ts @@ -181,8 +181,13 @@ export const INPUT_DECLS = { }, exclude: { description: - 'Comma- or newline-separated wildcard patterns removing repositories from repos: "*" ' + - "discovery.", + 'Comma- or newline-separated glob patterns removing repositories from repos: "*" discovery, compared ' + + 'case-insensitively against owner/name when the pattern has a "/" and against the name alone ' + + 'otherwise. A star matches any run of characters, "?" one character, "[abc]" one of a set, "[!abc]" ' + + 'or "[^abc]" one outside it, and one leading "!" negates the pattern. Beyond those a pattern holds ' + + 'only letters, digits, ".", "-", and "_", with at most one "/", nothing empty around it, and no side ' + + 'that is just "." or ".."; anything else (a leading "./", a run of stars, a regex token such as ' + + '"(a)+") fails the run naming the fix.', default: "", list: true, }, diff --git a/src/index.ts b/src/index.ts index 4b93f06d4..6fc989d0b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,9 +6,11 @@ export { resolveCentralTargets } from "./discovery/central.js"; export { + compileExcludePattern, DEFAULT_DISCOVERY_FILTERS, type DiscoveryFilters, discoverRepos, + type ExcludePattern, } from "./discovery/discover.js"; export { parseReposInput } from "./discovery/repos-input.js"; export { diff --git a/src/problem.ts b/src/problem.ts index fe01c8aee..594712f7c 100644 --- a/src/problem.ts +++ b/src/problem.ts @@ -10,7 +10,7 @@ import { describeKind } from "./plain-data.js"; import type { SectionKey } from "./schema.js"; -import { agree, countNoun } from "./text.js"; +import { agree, countNoun, quote } from "./text.js"; /** * The advice appended to a transient (non-permission) API failure: a network @@ -131,7 +131,11 @@ export type Problem = readonly entry: string; readonly allowed: readonly string[]; } - | { readonly code: "input-exclude-pattern-invalid"; readonly pattern: string } + | { + readonly code: "input-exclude-pattern-invalid"; + readonly pattern: string; + readonly reason: string; + } | { readonly code: "input-repository-with-multi" } | { readonly code: "input-settings-file-with-multi" } | { @@ -332,10 +336,6 @@ const PASSTHROUGH_ADVICE = "through, except in closed sections and strict nested objects like actions.cache, which reject " + "unrecognized keys)"; -function quote(value: unknown): string { - return JSON.stringify(String(value)); -} - function describeCentralFile(file: CentralFileProblem): string { switch (file.kind) { case "not-a-slug": @@ -509,12 +509,20 @@ export function describeProblem(problem: Problem): string { return 'the "private-report" input delivers reports only for redacted targets, but "private-repos" is "show", so nothing is redacted and no report would ever be sent. Set private-repos: redact, or set private-report: none'; case "input-affiliation-unsupported": return `the "affiliation" input entry "${problem.entry}" is not a supported affiliation, so discovery cannot build the /user/repos query. Use a comma-separated list of ${quoteList(problem.allowed)}`; - case "input-exclude-pattern-invalid": + case "input-exclude-pattern-invalid": { + const shown = + problem.pattern.length > 60 ? `${problem.pattern.slice(0, 60)}...` : problem.pattern; + // A position names a character in the pattern as the fixes before it leave it, counted from 0. + const positioned = / at \d+/.test(problem.reason) + ? ", applying the fixes in order with positions counted from 0 in the pattern as the earlier fixes leave it" + : ""; return ( - `the "exclude" input pattern "${problem.pattern}" can never match an owner/name repository: a ` + - `pattern takes at most one "/", with a non-empty glob on each side of it. Use ` + - `"" or "/", where "*" matches any characters` + `the "exclude" input pattern ${quote(shown)} is not a usable glob: ${problem.reason}${positioned}. Write ` + + `"" or "/", where a glob is letters, digits, ".", "-", "_", "*", "?", ` + + `and the classes "[abc]", "[!abc]", or "[^abc]" over those characters, with one optional leading "!" ` + + `negating the whole pattern` ); + } case "input-repository-with-multi": return 'the "repository" input cannot be combined with "repos" or "repos-dir"; multi-repo targets come from those inputs. Remove "repository", or remove the multi-repo inputs to stay in single-repo mode'; case "input-settings-file-with-multi": diff --git a/src/text.ts b/src/text.ts index b0fc8782d..7f7468f4c 100644 --- a/src/text.ts +++ b/src/text.ts @@ -7,3 +7,8 @@ export function agree(count: number, one: string, many: string): string { export function countNoun(count: number, one: string, many: string): string { return `${count} ${agree(count, one, many)}`; } + +/** A user-supplied value quoted for a message, so a quote or backslash inside it stays distinguishable. */ +export function quote(value: unknown): string { + return JSON.stringify(String(value)); +} diff --git a/test/action/run.test.ts b/test/action/run.test.ts index cb0bc529d..0a471da31 100644 --- a/test/action/run.test.ts +++ b/test/action/run.test.ts @@ -156,6 +156,7 @@ describe("run in multi-repo mode (env glue)", () => { const EXCLUDE = (pattern: string): Problem => ({ code: "input-exclude-pattern-invalid", pattern, + reason: 'it takes at most one "/", with a non-empty glob on each side of it', }); // A row names the whole Problem where that is the rule, and a message fragment where the whole message would restate // the input's option list or a library's own wording. @@ -180,6 +181,11 @@ describe("run in multi-repo mode (env glue)", () => { ["exclude: a/b/c", { INPUT_REPOS: "*", INPUT_EXCLUDE: "a/b/c" }, EXCLUDE("a/b/c")], ["exclude: octo/", { INPUT_REPOS: "*", INPUT_EXCLUDE: "octo/" }, EXCLUDE("octo/")], ["exclude: /repo", { INPUT_REPOS: "*", INPUT_EXCLUDE: "/repo" }, EXCLUDE("/repo")], + [ + "exclude: svc-[ab", + { INPUT_REPOS: "*", INPUT_EXCLUDE: "svc-[ab" }, + 'the "exclude" input pattern "svc-[ab" is not a usable glob: its "[" at 4 has no partner, so remove it', + ], [ "a filter with an explicit repos list", { INPUT_REPOS: "o/a", INPUT_FORKS: "exclude" }, diff --git a/test/cli/inputs.test.ts b/test/cli/inputs.test.ts index 7d6568c9e..75d3dc017 100644 --- a/test/cli/inputs.test.ts +++ b/test/cli/inputs.test.ts @@ -485,7 +485,8 @@ describe("the per-mode flag split", () => { exclude: { argv: discovery, pair: ["x/*", "tmp-*"], - kept: (cfg) => (cfg.kind === "multi" ? cfg.discoveryFilters.exclude : []), + kept: (cfg) => + cfg.kind === "multi" ? cfg.discoveryFilters.exclude.map((e) => e.pattern) : [], }, topics: { argv: discovery, diff --git a/test/discovery/discover.test.ts b/test/discovery/discover.test.ts index 8ad14066f..4a358da94 100644 --- a/test/discovery/discover.test.ts +++ b/test/discovery/discover.test.ts @@ -1,10 +1,13 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, spyOn, test } from "bun:test"; import { err } from "neverthrow"; +import picomatch from "picomatch"; import { + compileExcludePattern, DEFAULT_DISCOVERY_FILTERS, type DiscoveredRepoRef, type DiscoveryFilters, discoverRepos, + type ExcludePattern, excludeMatches, type FilteredRepoRef, formatSkipNotice, @@ -13,21 +16,348 @@ import { markPrivate } from "../../src/private.js"; import { describeProblem } from "../../src/problem.js"; import { MockApi } from "../mock-api.js"; -describe("excludeMatches", () => { +/** The compiled form of a pattern the test knows to be valid; a refusal is a test bug, named. */ +const compiled = (pattern: string): ExcludePattern => + compileExcludePattern(pattern).match( + (exclude) => exclude, + (problem) => { + throw new Error(describeProblem(problem)); + }, + ); + +describe("compileExcludePattern", () => { test.each<[string, string, string, boolean]>([ ["* spans any characters, anchored at both ends", "tmp-*", "o/tmp-x", true], ["* spans any characters, anchored at both ends", "tmp", "o/tmp-x", false], ["* spans any characters, anchored at both ends", "*-archive", "o/old-archive", true], - ["regex metacharacters are literal", "a.b", "o/a.b", true], - ["regex metacharacters are literal", "a.b", "o/axb", false], + ["* spans a leading dot, which names a repository, not a hidden file", "*", "o/.github", true], + ["a dot is literal", "a.b", "o/a.b", true], + ["a dot is literal", "a.b", "o/axb", false], ["matching is case-insensitive", "TMP-*", "o/tmp-x", true], + ["matching is case-insensitive", "octo/*", "OCTO/x", true], ["a pattern with a slash matches the full slug", "octo/*", "octo/anything", true], ["a pattern with a slash matches the full slug", "octo/*", "example-org/anything", false], ["a pattern without a slash matches the name only", "web*", "weborg/api", false], ["a pattern without a slash matches the name only", "web*", "anyowner/web-x", true], + ["? matches exactly one character", "tmp-?", "o/tmp-1", true], + ["? matches exactly one character", "tmp-?", "o/tmp-10", false], + ["? matches exactly one character", "tmp-?", "o/tmp-", false], + ["[abc] matches one of the set", "svc-[ab]", "o/svc-a", true], + ["[abc] matches one of the set", "svc-[ab]", "o/svc-c", false], + ["[!abc] matches one character outside the set", "svc-[!ab]", "o/svc-c", true], + ["[!abc] matches one character outside the set", "svc-[!ab]", "o/svc-a", false], + ["a leading ! negates the whole pattern", "!svc-*", "o/tooling", true], + ["a leading ! negates the whole pattern", "!svc-*", "o/svc-a", false], + ["a leading ! negates the whole pattern", "!octo/*", "octo/x", false], + ["a leading ! negates the whole pattern", "!octo/*", "other/x", true], + ["* around a literal dot spans an empty run too (no fast path)", "*.*", "o/archive.", true], + ["* around a literal dot spans an empty run too (no fast path)", "*.*", "o/.github", true], + ["* around a literal dot spans an empty run too (no fast path)", "*.*", "o/archive", false], ])("%s: %p against %p is %p", (_rule, pattern, slug, matches) => { - expect(excludeMatches(pattern, slug)).toBe(matches); + expect(excludeMatches(compiled(pattern), slug)).toBe(matches); + }); + + test("the compiled entry keeps the operator's text for the skip notice and the scope its slash chose", () => { + expect(compiled("TMP-*")).toMatchObject({ pattern: "TMP-*", scope: "name" }); + expect(compiled("octo/*")).toMatchObject({ pattern: "octo/*", scope: "slug" }); + }); + + test("an accepted pattern is compiled once: the engine step's regex is the one kept", () => { + const makeRe = spyOn(picomatch, "makeRe"); + try { + expect(compileExcludePattern("svc-[ab]").isOk()).toBe(true); + expect(makeRe).toHaveBeenCalledTimes(1); + } finally { + makeRe.mockRestore(); + } + }); + + // Every flaw of one pattern is named at once, each sentence one fix, in the order the fixes apply. + const DOT_SLASH = 'it starts with "./", which names no owner, so drop every leading "./"'; + const STARS = + 'it holds a run of stars, which matches no more than one "*" does, so write one "*"'; + test.each<[string, string]>([ + ["./tmp-**", `${DOT_SLASH}, and ${STARS}`], + ["!./", `${DOT_SLASH}, and "!" has nothing to negate`], + ["././x", DOT_SLASH], + [ + " ./x/. ", + `it is wrapped in whitespace, so trim it, and ${DOT_SLASH}, and "." and ".." are never names`, + ], + [" ", "it is wrapped in whitespace, so trim it, and it is empty"], + [ + "./***x y", + `it holds " ", which no owner or repository name contains, so remove it, and ${DOT_SLASH}, and ${STARS}`, + ], + ["a/**/c", `${STARS}, and it takes at most one "/", with a non-empty glob on each side of it`], + [ + "[**]", + 'the class "[**]" takes name characters only, so write it as "[]", and the class "[]" matches nothing, ' + + "so delete it, and it is empty", + ], + [ + "svc-(a)+", + 'it holds "(", ")", "+", which no owner or repository name contains, so remove them', + ], + [ + `!./${"a".repeat(65_536)}`, + `${DOT_SLASH}, and it is 65537 characters long, past the cap of 65536, so shorten it`, + ], + [ + `./${"a".repeat(65_536)}[z-a]`, + `the class "[z-a]" has the backwards range "z-a", so write it as "[a-z]", and ${DOT_SLASH}, and it is ` + + "65541 characters long, past the cap of 65536, so shorten it", + ], + [ + "./[z-*]", + `the class "[z-*]" takes name characters only, so write it as "[z-]", and ${DOT_SLASH}`, + ], + [ + "svc-[!z-a0-9]", + 'the class "[!z-a0-9]" has the backwards range "z-a", so write it as "[!a-z0-9]"', + ], + [ + "[0a--]", + 'the class "[0a--]" has a "-" where it reads as a range operator, so write it as "[0a-]"', + ], + ["[-z-a]", 'the class "[-z-a]" has the backwards range "z-a", so write it as "[-a-z]"'], + [ + "[9-0a--]", + 'the class "[9-0a--]" has a "-" where it reads as a range operator, so write it as "[9-0a-]", and ' + + 'the class "[9-0a-]" has the backwards range "9-0", so write it as "[0-9a-]"', + ], + [ + "[z-a_-.]", + 'the class "[z-a_-.]" has the backwards range "z-a" and "_-.", so write it as "[a-z.-_]"', + ], + [ + "!!./tmp-**", + `its "!" at 1 neither negates nor opens a class, so remove it, and ${DOT_SLASH}, and ${STARS}`, + ], + [ + "x/.!", + 'its "!" at 3 neither negates nor opens a class, so remove it, and "." and ".." are never names', + ], + ["[a/b]", 'its "[" at 0 and "]" at 4 have no partner, so remove them'], + ["*[]*", `the class "[]" matches nothing, so delete it, and ${STARS}`], + ["[]./x", `the class "[]" matches nothing, so delete it, and ${DOT_SLASH}`], + ["svc-[ab", 'its "[" at 4 has no partner, so remove it'], + [ + "x[!a", + 'its "[" at 1 has no partner, so remove it, and its "!" at 1 neither negates nor opens a class, so remove it', + ], + ])("%p is refused naming every flaw", (pattern, reason) => { + expect(compileExcludePattern(pattern)).toEqual( + err({ code: "input-exclude-pattern-invalid", pattern, reason }), + ); + }); + + test("a stray quote or backslash is quoted so it stays distinguishable from the punctuation", () => { + expect(compileExcludePattern('svc-"a"')).toEqual( + err({ + code: "input-exclude-pattern-invalid", + pattern: 'svc-"a"', + reason: 'it holds "\\"", which no owner or repository name contains, so remove it', + }), + ); + const slash = compileExcludePattern("svc-\\d"); + expect(slash.isErr() && slash.error.reason).toBe( + 'it holds "\\\\", which no owner or repository name contains, so remove it', + ); + const problem = compileExcludePattern('svc-"a"'); + expect(problem.isErr() && describeProblem(problem.error)).toContain( + 'the "exclude" input pattern "svc-\\"a\\"" is not a usable glob', + ); }); + + // A refusal is a program: each sentence names one fix a machine can apply (so a reader can too), in order, and + // the result is accepted or carries only what the user has to write (an empty body, the sides around a "/", a + // side that is only dots, a shorter pattern), every one of which the first refusal named already. If a + // sentence has no machine fix and is not one of those, it is not a fix, and the applier throws. + const REMAINDERS = [ + /^it is empty$/, + /^"!" has nothing to negate$/, + /^it takes at most one "\/"/, + /^"\." and "\.\." are never names$/, + /^it is \d+ characters long, past the cap/, + /^it compiles to a regular expression the runtime refuses/, + ]; + const isRemainder = (sentence: string): boolean => + REMAINDERS.some((remainder) => remainder.test(sentence)); + const quoted = (text: string): string[] => + [...text.matchAll(/"((?:\\.|[^"\\])*)"/g)].map((m) => JSON.parse(`"${m[1]}"`) as string); + const applyFix = (view: string, sentence: string): string => { + if (sentence.startsWith("it is wrapped in whitespace")) { + return view.trim(); + } + if (sentence.startsWith("it holds a run of stars")) { + return view.replace(/\*{2,}/g, "*"); + } + if (sentence.startsWith("it holds ")) { + const chars = quoted(sentence.slice(0, sentence.indexOf(", which"))); + return view + .split("") + .filter((char) => !chars.includes(char)) + .join(""); + } + if (sentence.startsWith("its ")) { + const where = [...sentence.matchAll(/ at (\d+)/g)].map((m) => Number(m[1])); + return view + .split("") + .filter((_, i) => !where.includes(i)) + .join(""); + } + if (sentence.startsWith('it starts with "./"')) { + return view.replace(/^(!?)(?:\.\/)+/, "$1"); + } + if (sentence.startsWith("the class ")) { + const parts = quoted(sentence); + const cls = parts[0] ?? ""; + return view.replace(cls, sentence.endsWith("so delete it") ? "" : (parts.at(-1) ?? "")); + } + if (isRemainder(sentence)) { + return view; + } + throw new Error(`no machine fix for: ${sentence}`); + }; + const oneRound = (pattern: string): void => { + const first = compileExcludePattern(pattern); + if (first.isOk()) { + return; + } + const sentences = first.error.reason.split(", and "); + const second = compileExcludePattern(sentences.reduce(applyFix, pattern)); + if (second.isErr()) { + for (const left of second.error.reason.split(", and ")) { + const shown = JSON.stringify(pattern).slice(0, 80); + const detail = `${shown} left "${left}" after: ${first.error.reason.slice(0, 400)}`; + expect(isRemainder(left), detail).toBe(true); + expect(sentences, detail).toContain(left); + } + } + }; + test.each([ + "!!./tmp-**", + "x/.!", + "./***x y", + "a/**/c", + "[**]", + "./[z-*]", + "[0a--]", + "[-a---b]", + "!./", + "./", + "x[!a", + "[a/b]", + "[[a]]", + `./${"a".repeat(65_536)}[z-a]`, + `!./${"a".repeat(65_536)}`, + "*[]*", + "[]./x", + `./${"?".repeat(60_000)}`, + ])("the fixes named for %p apply in one round", oneRound); + // Builder controls ran the same generator at 4000 patterns (seed 20261005, with a 65,540-character piece) and + // 50,000 patterns (seed 0x9e3779b9, 96 pieces, up to 12 per pattern); the committed set stays small. + test("the fixes named for 300 generated patterns apply in one round", () => { + const pieces = [ + "tmp", + "svc-a", + ".github", + "*", + "**", + "***", + "?", + "./", + "././", + "!", + "!!", + "^", + "/", + "..", + ".", + "(", + "+", + " ", + "\\", + '"', + "[", + "]", + "[ab]", + "[!ab]", + "[z-a]", + "[a*]", + "[]", + "[!]", + "[0a--]", + "[--a]", + "[-a-]", + "[a-z-9]", + "[[a]]", + "[9-0_-.]", + "-", + "_", + ]; + let seed = 20261005; + const next = (): number => { + seed = (seed * 1103515245 + 12345) % 2147483648; + return seed; + }; + for (let n = 0; n < 300; n++) { + const count = 1 + (next() % 5); + let pattern = ""; + for (let i = 0; i < count; i++) { + pattern += pieces[next() % pieces.length]; + } + oneRound(pattern); + } + }); + + // Every row is a pattern picomatch itself would accept or refuse with its own words, several compiled to a wrong + // or never-matching regex: "(a)+" a quantifier, "\d" a digit class, "{az..bz}" a range, a leading "./" dropped, + // "***a." with the dot unescaped, "!!" an empty body, "[a/b]" a class across the slash, "[z-a]" and the length + // cap thrown as regex errors. A pattern is the user's input, so each is refused naming the fix, never repaired. + test.each<[string, string]>([ + [" tmp-* ", "trim it"], + ["tmp-*\n", "trim it"], + ["svc-(a)+", 'it holds "("'], + ["svc-\\d", 'it holds "\\\\"'], + ["svc-{az..bz}", 'it holds "{"'], + ['svc-"a"', 'it holds "\\""'], + ["a|b", 'it holds "|"'], + ["tmp x", 'it holds " "'], + ["a/b/c", 'at most one "/"'], + ["octo/", 'at most one "/"'], + ["/repo", 'at most one "/"'], + ["./tmp-*", 'drop every leading "./"'], + ["!./svc-*", 'drop every leading "./"'], + ["./", 'drop every leading "./"'], + ["!", "nothing to negate"], + ["!/x", 'at most one "/"'], + ["x/..", '"." and ".." are never names'], + ["x/.", '"." and ".." are never names'], + ["tmp-**", 'write one "*"'], + ["***a.", 'write one "*"'], + ["o/**", 'write one "*"'], + ["!!svc-*", "neither negates nor opens a class"], + ["a!b", "neither negates nor opens a class"], + ["^a", 'its "^" at 0 neither negates nor opens a class'], + ["[a/b]", "have no partner"], + ["svc-[ab", "has no partner"], + ["[]", 'the class "[]" matches nothing, so delete it'], + ["[*]", 'the class "[*]" takes name characters only'], + ["[z-a]", 'has the backwards range "z-a"'], + ["a".repeat(70_000), "past the cap of 65536, so shorten it"], + ])( + "%p is refused with the reason, not thrown or compiled to something else", + (pattern, reason) => { + const result = compileExcludePattern(pattern); + expect(result.isErr()).toBe(true); + if (result.isErr()) { + expect(result.error).toMatchObject({ code: "input-exclude-pattern-invalid", pattern }); + expect(result.error).toHaveProperty("reason", expect.stringContaining(reason)); + } + }, + ); }); describe("discoverRepos", () => { @@ -206,7 +536,7 @@ describe("discoverRepos", () => { data: [{ full_name: "o/keep" }, { full_name: "o/tmp-1" }, { full_name: "octo/keep" }], }, }, - { exclude: ["tmp-*", "octo/*"] }, + { exclude: [compiled("tmp-*"), compiled("octo/*")] }, ); expect(slugs(discovered.repos)).toEqual(["o/keep"]); expect(filteredSlugs(discovered.filtered)).toEqual([ @@ -220,7 +550,7 @@ describe("discoverRepos", () => { { [OWNED]: { data: [{ full_name: "o/tmp-fork", archived: true, fork: true }] }, }, - { forks: "exclude", exclude: ["tmp-*"] }, + { forks: "exclude", exclude: [compiled("tmp-*")] }, ); expect(discovered.repos).toEqual([]); expect(filteredSlugs(discovered.filtered)).toEqual([ diff --git a/test/e2e/oracle.ts b/test/e2e/oracle.ts index 2ba6e1ac4..c7903e84b 100644 --- a/test/e2e/oracle.ts +++ b/test/e2e/oracle.ts @@ -614,11 +614,15 @@ export interface DiscoveryFilters { } /** - * An INDEPENDENT glob matcher, deliberately not src's excludeMatches (a RegExp compile), so a bug in - * either surfaces as a disagreement. A pattern with "/" matches the full slug, otherwise the name - * portion, mirroring the repos-dir .yml vs /.yml split. + * An INDEPENDENT glob matcher, deliberately not src's compileExcludePattern (a picomatch compile), so a bug in + * either surfaces as a disagreement; it covers the forms the scenarios write ("*", "?", a leading "!"), not + * the whole picomatch grammar. A pattern with "/" matches the full slug, otherwise the name portion, + * mirroring the repos-dir .yml vs /.yml split. */ function globMatches(pattern: string, slug: string): boolean { + if (pattern.startsWith("!")) { + return !globMatches(pattern.slice(1), slug); + } const target = (pattern.includes("/") ? slug : (slug.split("/")[1] ?? slug)).toLowerCase(); const pat = pattern.toLowerCase(); let p = 0; @@ -626,7 +630,7 @@ function globMatches(pattern: string, slug: string): boolean { let star = -1; let mark = 0; while (t < target.length) { - if (p < pat.length && (pat[p] === target[t] || pat[p] === "*")) { + if (p < pat.length && (pat[p] === target[t] || pat[p] === "*" || pat[p] === "?")) { if (pat[p] === "*") { star = p; mark = t; diff --git a/test/e2e/scenarios/multi-discovery-exclude-negation.yml b/test/e2e/scenarios/multi-discovery-exclude-negation.yml new file mode 100644 index 000000000..180457f63 --- /dev/null +++ b/test/e2e/scenarios/multi-discovery-exclude-negation.yml @@ -0,0 +1,45 @@ +# Multi-repo repos: "*" discovery with a negated exclude glob. "!svc-*" excludes +# every repository whose name does NOT match svc-*, so the exclude input doubles +# as an allowlist: svc-a and svc-b are kept, the other two are dropped under the +# one negated pattern, and the action issues zero requests to them. +name: multi-discovery-exclude-negation +settings: {} +discovery: + inputs: + exclude: "!svc-*" + pool: + - {slug: e2e-owner/svc-a} + - {slug: e2e-owner/svc-b} + - {slug: e2e-owner/tooling} + - {slug: e2e-owner/website} +repos: + e2e-owner/svc-a: + settings: + labels: + - name: managed + color: "00ff00" + expect: + result: applied + e2e-owner/svc-b: + settings: + labels: + - name: managed + color: "00ff00" + expect: + result: applied +token_permissions: + issues: write + contents: read +expect: + exit_code: 0 + result: applied + repos_result: + e2e-owner/svc-a: applied + e2e-owner/svc-b: applied + stdout_contains: + - 'skipped 2 repositories by exclude pattern "!svc-*": e2e-owner/tooling, e2e-owner/website' + never: + - "GET /repos/e2e-owner/tooling" + - "GET /repos/e2e-owner/website" + requests_contain: + - "GET /user/repos" diff --git a/test/e2e/scenarios/multi-discovery-exclude.yml b/test/e2e/scenarios/multi-discovery-exclude.yml new file mode 100644 index 000000000..833c6d821 --- /dev/null +++ b/test/e2e/scenarios/multi-discovery-exclude.yml @@ -0,0 +1,57 @@ +# Multi-repo repos: "*" discovery with exclude globs. The pool carries two +# managed services, tmp-1, tmp-20, and legacy-old. "tmp-?" matches one +# character after the dash, so it drops tmp-1 and keeps tmp-20, and +# "E2E-Owner/legacy-*" matches the whole owner/name case-insensitively. The +# action must issue zero requests to the two excluded repos, and the skip notice +# names the pattern that fired. +name: multi-discovery-exclude +settings: {} +discovery: + inputs: + exclude: "tmp-?,E2E-Owner/legacy-*" + pool: + - {slug: e2e-owner/svc-a} + - {slug: e2e-owner/svc-b} + - {slug: e2e-owner/tmp-1} + - {slug: e2e-owner/tmp-20} + - {slug: e2e-owner/legacy-old} +repos: + e2e-owner/svc-a: + settings: + labels: + - name: managed + color: "00ff00" + expect: + result: applied + e2e-owner/svc-b: + settings: + labels: + - name: managed + color: "00ff00" + expect: + result: applied + e2e-owner/tmp-20: + settings: + labels: + - name: managed + color: "00ff00" + expect: + result: applied +token_permissions: + issues: write + contents: read +expect: + exit_code: 0 + result: applied + repos_result: + e2e-owner/svc-a: applied + e2e-owner/svc-b: applied + e2e-owner/tmp-20: applied + stdout_contains: + - 'skipped 1 repository by exclude pattern "tmp-?": e2e-owner/tmp-1' + - 'skipped 1 repository by exclude pattern "E2E-Owner/legacy-*": e2e-owner/legacy-old' + never: + - "GET /repos/e2e-owner/tmp-1" + - "GET /repos/e2e-owner/legacy-old" + requests_contain: + - "GET /user/repos" diff --git a/test/private.test.ts b/test/private.test.ts index 239a331ad..cc67caf7a 100644 --- a/test/private.test.ts +++ b/test/private.test.ts @@ -33,6 +33,7 @@ const OPENER_IMPORTERS: Record = { "DEFAULT_DISCOVERY_FILTERS", "FORKS_FILTERS", "VISIBILITY_FILTERS", + "compileExcludePattern", "discoverRepos", "excludeMatches", "formatSkipNotice", diff --git a/test/problem.test.ts b/test/problem.test.ts index 8c1430e8a..b8b76d960 100644 --- a/test/problem.test.ts +++ b/test/problem.test.ts @@ -127,8 +127,15 @@ const SPECIMENS = { 'the "affiliation" input entry "member" is not a supported affiliation, so discovery cannot build the /user/repos query. Use a comma-separated list of "owner", "collaborator", "organization_member"', ], "input-exclude-pattern-invalid": [ - { code: "input-exclude-pattern-invalid", pattern: "a/b/c" }, - 'the "exclude" input pattern "a/b/c" can never match an owner/name repository: a pattern takes at most one "/", with a non-empty glob on each side of it. Use "" or "/", where "*" matches any characters', + { + code: "input-exclude-pattern-invalid", + pattern: "a/b/c", + reason: 'it takes at most one "/", with a non-empty glob on each side of it', + }, + 'the "exclude" input pattern "a/b/c" is not a usable glob: it takes at most one "/", with a non-empty glob ' + + 'on each side of it. Write "" or "/", where a glob is letters, digits, ' + + '".", "-", "_", "*", "?", and the classes "[abc]", "[!abc]", or "[^abc]" over those characters, with one ' + + 'optional leading "!" negating the whole pattern', ], "discovery-filters-without-wildcard": [ { code: "discovery-filters-without-wildcard", filters: ["forks"], targets: "single-repo" },