From c4c79e8fd6114d6ce4692d460e48548e3be18c45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Tue, 25 Aug 2026 23:13:02 +0200 Subject: [PATCH 1/3] Move the tracked source pins forward on a cron Upstream movement used to reach a nightly only when someone edited Components.cmake by hand, so a nightly was as old as the last time anybody remembered. A bot resolves the upstream branches the tracked leaves develop on, writes the exact revisions back into the file, and commits only what actually moved -- its own push can never be a reason for the next run to commit again. Which leaf follows which branch is declared in cmake/pin-tracking.json, and so is every pin the bot must leave alone and why: a released tag, a third-party tag, a revision whose packaging assumptions were reviewed. A pin that appears in neither list stops the bot before it resolves anything, so a source added to the build cannot quietly start or stop being followed. newlib develops on `vita`, not on master; the config says so per leaf rather than assuming. --- .github/workflows/build.yml | 2 +- .github/workflows/bump-pins.yml | 54 ++++++++ cmake/pin-tracking.json | 16 +++ scripts/bump-pins.py | 220 ++++++++++++++++++++++++++++++++ tests/ci/test-bump-pins.sh | 214 +++++++++++++++++++++++++++++++ 5 files changed, 505 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/bump-pins.yml create mode 100644 cmake/pin-tracking.json create mode 100755 scripts/bump-pins.py create mode 100755 tests/ci/test-bump-pins.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bcb1024..5bb89f7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,7 +40,7 @@ jobs: echo "== $test" "$test" done - - name: Run the reusable workflow tests + - name: Run the CI script tests run: | for test in tests/ci/test-*.sh; do echo "== $test" diff --git a/.github/workflows/bump-pins.yml b/.github/workflows/bump-pins.yml new file mode 100644 index 0000000..4b2ddb6 --- /dev/null +++ b/.github/workflows/bump-pins.yml @@ -0,0 +1,54 @@ +name: Bump source pins + +# Upstream movement enters the build as a commit here, never as a floating +# ref: what a nightly was built from is always readable in this history. +on: + schedule: + - cron: "0 3 * * *" + workflow_dispatch: + inputs: + dry_run: + description: Report what would move without committing it + required: false + default: false + type: boolean + +jobs: + bump: + runs-on: ubuntu-24.04 + steps: + - name: Check the push token is configured + env: + BUMP_TOKEN: ${{ secrets.REPO_DISPATCH_TOKEN }} + shell: bash + run: | + if [[ -z "$BUMP_TOKEN" ]]; then + echo "::error::REPO_DISPATCH_TOKEN is required: a push made with GITHUB_TOKEN fires no workflow, so the build would never be announced" + exit 1 + fi + + - uses: actions/checkout@v7 + with: + # Not GITHUB_TOKEN on purpose: GitHub fires no workflow for a push + # made with it, and this push is what triggers discovery. + token: ${{ secrets.REPO_DISPATCH_TOKEN }} + + - name: Resolve the tracked upstream branches + shell: bash + run: | + python3 scripts/bump-pins.py \ + ${{ inputs.dry_run && '--dry-run' || '' }} \ + --message-file "$RUNNER_TEMP/commit-message" + + - name: Commit and push what moved + if: ${{ !inputs.dry_run }} + shell: bash + run: | + if git diff --quiet; then + echo "no pin moved; nothing to push" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -a --file "$RUNNER_TEMP/commit-message" + git push origin HEAD:"$GITHUB_REF_NAME" diff --git a/cmake/pin-tracking.json b/cmake/pin-tracking.json new file mode 100644 index 0000000..568fdf9 --- /dev/null +++ b/cmake/pin-tracking.json @@ -0,0 +1,16 @@ +{ + "schema": 1, + "_comment": "Which git-pinned components the pin-bump bot moves forward, and which it leaves alone. Every component pinned by repository plus revision must appear in exactly one of the two maps: scripts/bump-pins.py refuses to run otherwise, so a new pin cannot go unnoticed in either direction.", + "tracked": { + "newlib": {"branch": "vita"}, + "samples": {"branch": "master"}, + "headers": {"branch": "master"}, + "toolchain": {"branch": "master"}, + "pthread": {"branch": "master"} + }, + "untracked": { + "isl": "a third-party release tag; moves with the GCC dependency set, by hand", + "vdpm": "a released tag; the vdpm release process moves it, and the core embeds the matching bundle", + "vita-makepkg": "a reviewed revision; its packaging assumptions are load-bearing for the core package" + } +} diff --git a/scripts/bump-pins.py b/scripts/bump-pins.py new file mode 100755 index 0000000..313a734 --- /dev/null +++ b/scripts/bump-pins.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Moves the tracked source pins to what their upstream branches hold now.""" + +import argparse +import json +import os +import re +import subprocess +import sys + +COMPONENTS_PATH = "cmake/Components.cmake" +TRACKING_PATH = "cmake/pin-tracking.json" +SCHEMA = 1 + +SET_PATTERN = re.compile(r'set\(\s*([A-Za-z0-9_]+)\s+"?([^")\s]+)"?') +VARIABLE_REFERENCE = re.compile(r"\$\{([A-Za-z0-9_]+)\}") + + +class BumpError(Exception): + pass + + +def component_name(variable): + return variable.lower().replace("_", "-") + + +def parse_components(text): + """Every component pinned by repository plus revision, by lock name.""" + + raw = dict(SET_PATTERN.findall(text)) + + def resolve(value, seen=()): + def substitute(match): + name = match.group(1) + if name in seen: + raise BumpError(f"{COMPONENTS_PATH}: circular reference on {name}") + if name not in raw: + raise BumpError( + f"{COMPONENTS_PATH} references undefined variable {name}" + ) + return resolve(raw[name], seen + (name,)) + + return VARIABLE_REFERENCE.sub(substitute, value) + + components = {} + for variable, value in raw.items(): + if not variable.endswith("_REPOSITORY"): + continue + prefix = variable[: -len("_REPOSITORY")] + if f"{prefix}_TAG" not in raw: + continue + components[component_name(prefix)] = { + "variable": prefix, + "repository": resolve(value), + "pin": resolve(raw[f"{prefix}_TAG"]), + } + if not components: + raise BumpError(f"{COMPONENTS_PATH} declares no git-pinned component") + return components + + +def load_tracking(path, components): + try: + with open(path, encoding="utf-8") as handle: + config = json.load(handle) + except OSError as exc: + raise BumpError(f"{path} cannot be read: {exc}") from exc + except json.JSONDecodeError as exc: + raise BumpError(f"{path} is not valid JSON: {exc}") from exc + + if config.get("schema") != SCHEMA: + raise BumpError(f"{path} declares schema {config.get('schema')}, expected {SCHEMA}") + tracked = config.get("tracked") + untracked = config.get("untracked") + if not isinstance(tracked, dict) or not isinstance(untracked, dict): + raise BumpError(f"{path} must declare 'tracked' and 'untracked' as objects") + + both = sorted(set(tracked) & set(untracked)) + if both: + raise BumpError(f"{path} lists as both tracked and untracked: {', '.join(both)}") + unknown = sorted((set(tracked) | set(untracked)) - set(components)) + if unknown: + raise BumpError( + f"{path} names components that are not pinned in {COMPONENTS_PATH}: " + f"{', '.join(unknown)}" + ) + # The point of the file: a pin added to the build cannot stay undeclared, + # in either direction, without this failing before anything is resolved. + undeclared = sorted(set(components) - set(tracked) - set(untracked)) + if undeclared: + raise BumpError( + f"{path} declares neither tracking nor a reason for: {', '.join(undeclared)}" + ) + + for name, entry in tracked.items(): + branch = entry.get("branch") if isinstance(entry, dict) else None + if not branch: + raise BumpError(f"{path}: tracked component {name} declares no branch") + for name, reason in untracked.items(): + if not isinstance(reason, str) or not reason.strip(): + raise BumpError(f"{path}: untracked component {name} gives no reason") + return {name: entry["branch"] for name, entry in tracked.items()} + + +def resolve_branch(repository, branch): + result = subprocess.run( + ["git", "ls-remote", "--exit-code", repository, f"refs/heads/{branch}"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise BumpError( + f"{repository} has no branch {branch}: " + f"{result.stderr.strip() or 'git ls-remote failed'}" + ) + return result.stdout.split()[0] + + +def rewrite_pin(text, variable, old, new): + pattern = re.compile( + r"(set\(\s*" + re.escape(variable) + r"_TAG\s+)" + re.escape(old) + r"(?=[\s)])" + ) + rewritten, count = pattern.subn(lambda match: match.group(1) + new, text) + if count != 1: + raise BumpError( + f"{COMPONENTS_PATH}: expected one {variable}_TAG assignment, found {count}" + ) + return rewritten + + +def commit_message(moves): + names = [move["component"] for move in moves] + if len(names) == 1: + subject = f"Move the {names[0]} pin to {moves[0]['new'][:9]}" + elif len(names) == 2: + subject = f"Move the {names[0]} and {names[1]} pins forward" + else: + subject = f"Move {len(names)} source pins forward" + body = "\n".join( + f"{move['component']} {move['branch']}: {move['old'][:9]} -> {move['new'][:9]}" + for move in moves + ) + return f"{subject}\n\n{body}\n" + + +def bump(components_path, tracking_path, overrides, dry_run): + try: + with open(components_path, encoding="utf-8") as handle: + text = handle.read() + except OSError as exc: + raise BumpError(f"{components_path} cannot be read: {exc}") from exc + + components = parse_components(text) + tracked = load_tracking(tracking_path, components) + + moves = [] + for name in sorted(tracked): + component = components[name] + repository = overrides.get(name, component["repository"]) + head = resolve_branch(repository, tracked[name]) + if head == component["pin"]: + continue + text = rewrite_pin(text, component["variable"], component["pin"], head) + moves.append( + { + "component": name, + "branch": tracked[name], + "old": component["pin"], + "new": head, + } + ) + + if moves and not dry_run: + with open(components_path, "w", encoding="utf-8") as handle: + handle.write(text) + return moves + + +def main(argv): + parser = argparse.ArgumentParser(prog="bump-pins") + parser.add_argument("--components", default=COMPONENTS_PATH) + parser.add_argument("--tracking", default=TRACKING_PATH) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--message-file") + parser.add_argument( + "--repository", + action="append", + default=[], + metavar="COMPONENT=URL", + help="resolve one component somewhere else than Components.cmake says", + ) + args = parser.parse_args(argv) + + overrides = {} + for override in args.repository: + name, separator, url = override.partition("=") + if not separator or not name or not url: + print(f"bump-pins: --repository wants COMPONENT=URL, got {override}", file=sys.stderr) + return 1 + overrides[name] = url + + try: + moves = bump(args.components, args.tracking, overrides, args.dry_run) + except BumpError as exc: + print(f"bump-pins: {exc}", file=sys.stderr) + return 1 + + for move in moves: + print(f"{move['component']} {move['branch']}: {move['old']} -> {move['new']}") + print(f"bump-pins: {len(moves)} pin(s) moved" if moves else "bump-pins: nothing moved") + + if args.message_file: + message = commit_message(moves) if moves else "" + with open(args.message_file, "w", encoding="utf-8") as handle: + handle.write(message) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/ci/test-bump-pins.sh b/tests/ci/test-bump-pins.sh new file mode 100755 index 0000000..275ad73 --- /dev/null +++ b/tests/ci/test-bump-pins.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repository_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P) +temporary_root=$(mktemp -d "${TMPDIR:-/tmp}/vitasdk-bump-pins.XXXXXXXX") +cleanup() { rm -rf -- "$temporary_root"; } +trap cleanup EXIT + +work="$temporary_root/work" +mkdir -p "$work/cmake" +cp "$repository_root/cmake/Components.cmake" "$work/cmake/Components.cmake" +cp "$repository_root/cmake/pin-tracking.json" "$work/cmake/pin-tracking.json" + +components="$work/cmake/Components.cmake" +tracking="$work/cmake/pin-tracking.json" + +bump() { + python3 "$repository_root/scripts/bump-pins.py" \ + --components "$components" --tracking "$tracking" "$@" +} + +# The upstreams the bot follows, straight from the file under test: a +# component added to the config is exercised here without editing this test. +mapfile -t tracked < <(python3 - "$tracking" <<'PYEOF' +import json, sys +config = json.load(open(sys.argv[1])) +for name, entry in sorted(config["tracked"].items()): + print(f"{name} {entry['branch']}") +PYEOF +) +[[ ${#tracked[@]} -ge 2 ]] || { + printf 'the tracking config declares fewer than two components\n' >&2 + exit 1 +} + +overrides=() +declare -A upstream_of=() +for entry in "${tracked[@]}"; do + read -r name branch <<< "$entry" + upstream="$temporary_root/upstream/$name" + mkdir -p "$upstream" + git init --quiet --initial-branch "$branch" "$upstream" + git -C "$upstream" config user.email bump-pins-tests@ci.invalid + git -C "$upstream" config user.name "bump-pins tests" + echo "first" > "$upstream/file" + git -C "$upstream" add file + git -C "$upstream" commit --quiet -m "first" + upstream_of[$name]=$upstream + overrides+=(--repository "$name=$upstream") +done + +pin_of() { + python3 - "$components" "$1" <<'PYEOF' +import re, sys +variable = sys.argv[2].upper().replace("-", "_") +text = open(sys.argv[1]).read() +match = re.search(r"set\(\s*" + variable + r"_TAG\s+(\S+)", text) +print(match.group(1) if match else "") +PYEOF +} + +# Everything moves once, to exactly what the upstream branches hold. +output=$(bump "${overrides[@]}") +grep -q "bump-pins: ${#tracked[@]} pin(s) moved" <<< "$output" || { + printf 'the first bump did not move every tracked pin: %s\n' "$output" >&2 + exit 1 +} +for entry in "${tracked[@]}"; do + read -r name _ <<< "$entry" + expected=$(git -C "${upstream_of[$name]}" rev-parse HEAD) + [[ $(pin_of "$name") == "$expected" ]] || { + printf 'pin of %s is not the upstream head\n' "$name" >&2 + exit 1 + } +done + +# Convergence: nothing upstream moved, so the bot writes nothing. Its own +# push must never be a reason for the next run to commit again. +cp "$components" "$temporary_root/before-idle" +output=$(bump "${overrides[@]}" --message-file "$temporary_root/idle-message") +grep -q 'bump-pins: nothing moved' <<< "$output" || { + printf 'an idle bump reported movement: %s\n' "$output" >&2 + exit 1 +} +cmp -s "$components" "$temporary_root/before-idle" || { + printf 'an idle bump rewrote Components.cmake\n' >&2 + exit 1 +} +[[ ! -s "$temporary_root/idle-message" ]] || { + printf 'an idle bump wrote a commit message\n' >&2 + exit 1 +} + +# One leaf moves: one pin changes, one line changes, and the message says +# which component it was. +read -r moved_name moved_branch <<< "${tracked[0]}" +moved_upstream=${upstream_of[$moved_name]} +echo "second" > "$moved_upstream/file" +git -C "$moved_upstream" commit --quiet -am "second" +moved_head=$(git -C "$moved_upstream" rev-parse HEAD) + +cp "$components" "$temporary_root/before-one" +output=$(bump "${overrides[@]}" --message-file "$temporary_root/one-message") +grep -q 'bump-pins: 1 pin(s) moved' <<< "$output" || { + printf 'moving one upstream did not move exactly one pin: %s\n' "$output" >&2 + exit 1 +} +[[ $(pin_of "$moved_name") == "$moved_head" ]] || { + printf 'pin of %s did not follow its upstream\n' "$moved_name" >&2 + exit 1 +} +changed=$(diff "$temporary_root/before-one" "$components" | grep '^[<>]' || true) +[[ $(grep -c . <<< "$changed") -eq 2 ]] || { + printf 'moving one pin changed more than one line:\n%s\n' "$changed" >&2 + exit 1 +} +[[ $(grep -c '_TAG' <<< "$changed") -eq 2 ]] || { + printf 'the bot changed something that is not a pin:\n%s\n' "$changed" >&2 + exit 1 +} +grep -q "$moved_name" "$temporary_root/one-message" || { + printf 'the commit message does not name %s\n' "$moved_name" >&2 + exit 1 +} +grep -q "$moved_branch" "$temporary_root/one-message" || { + printf 'the commit message does not name the branch followed\n' >&2 + exit 1 +} + +# A dry run reports what it would do and writes nothing. +echo "third" > "$moved_upstream/file" +git -C "$moved_upstream" commit --quiet -am "third" +cp "$components" "$temporary_root/before-dry" +output=$(bump "${overrides[@]}" --dry-run) +grep -q 'bump-pins: 1 pin(s) moved' <<< "$output" || { + printf 'the dry run did not report the pending move: %s\n' "$output" >&2 + exit 1 +} +cmp -s "$components" "$temporary_root/before-dry" || { + printf 'the dry run rewrote Components.cmake\n' >&2 + exit 1 +} + +# The pins the config leaves alone stay exactly as the file declares them, +# and are never resolved: no override is given for any of them here. +for name in $(python3 -c 'import json,sys; print(" ".join(sorted(json.load(open(sys.argv[1]))["untracked"])))' "$tracking"); do + before=$(python3 - "$repository_root/cmake/Components.cmake" "$name" <<'PYEOF' +import re, sys +variable = sys.argv[2].upper().replace("-", "_") +text = open(sys.argv[1]).read() +match = re.search(r"set\(\s*" + variable + r"_TAG\s+(\S+)", text) +print(match.group(1) if match else "") +PYEOF + ) + [[ $(pin_of "$name") == "$before" ]] || { + printf 'the bot moved the untracked pin %s\n' "$name" >&2 + exit 1 + } +done + +# A branch that is not there fails loudly, naming what could not be resolved. +missing_branch_overrides=("${overrides[@]}") +missing="$temporary_root/upstream/missing" +git init --quiet --initial-branch nowhere "$missing" +git -C "$missing" config user.email bump-pins-tests@ci.invalid +git -C "$missing" config user.name "bump-pins tests" +echo x > "$missing/file" +git -C "$missing" add file +git -C "$missing" commit --quiet -m x +if output=$(bump "${missing_branch_overrides[@]}" --repository "$moved_name=$missing" 2>&1); then + printf 'the bot accepted a branch that does not exist\n' >&2 + exit 1 +fi +grep -q "$moved_branch" <<< "$output" || { + printf 'the failure does not name the branch it could not resolve: %s\n' "$output" >&2 + exit 1 +} + +# A pin added to the build and to neither list stops the bot before it +# resolves anything, naming the component nobody decided about. +cp "$components" "$temporary_root/before-undeclared" +cat >> "$components" <<'CMAKEEOF' + +set(SOMETHING_REPOSITORY https://github.com/vitasdk/something) +set(SOMETHING_TAG 0000000000000000000000000000000000000000) +CMAKEEOF +if output=$(bump "${overrides[@]}" 2>&1); then + printf 'the bot ran with an undeclared pin\n' >&2 + exit 1 +fi +grep -q 'something' <<< "$output" || { + printf 'the failure does not name the undeclared pin: %s\n' "$output" >&2 + exit 1 +} +cp "$temporary_root/before-undeclared" "$components" + +# A tracked name that is not a pin at all is a typo, and says so. +python3 - "$tracking" <<'PYEOF' +import json, sys +config = json.load(open(sys.argv[1])) +config["tracked"]["bogus"] = {"branch": "master"} +json.dump(config, open(sys.argv[1], "w"), indent=2) +PYEOF +if output=$(bump "${overrides[@]}" 2>&1); then + printf 'the bot ran with a tracked component that is not pinned\n' >&2 + exit 1 +fi +grep -q 'bogus' <<< "$output" || { + printf 'the failure does not name the unknown component: %s\n' "$output" >&2 + exit 1 +} + +printf 'bump-pins: all checks passed\n' From 0fda7131fedd9697bc44af6d331f691a35c0a684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Tue, 25 Aug 2026 23:13:12 +0200 Subject: [PATCH 2/3] Ask autobuilds to build every revision that lands here Nothing connected a commit on this branch to a build of it: the nightly moved when a person went and started one. Every push to the development branch now announces its exact revision to autobuilds, which decides whether it is worth building -- it deduplicates by build_id, so a revision it already built costs one job and no runners. The `run_build` listeners this repository carried were the same idea from the other direction and no sender ever existed. Keeping them while the name means "autobuilds, build this" would leave a dispatch able to start a full build here by accident. --- .github/workflows/build.yml | 2 -- .github/workflows/dispatch-build.yml | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/dispatch-build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5bb89f7..fdd8c24 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,8 +3,6 @@ name: Build on: push: pull_request: - repository_dispatch: - types: [run_build] # A push to a branch with an open pull request fires both triggers; one # group per head commit keeps a single run of the pair alive. diff --git a/.github/workflows/dispatch-build.yml b/.github/workflows/dispatch-build.yml new file mode 100644 index 0000000..a3d0e5b --- /dev/null +++ b/.github/workflows/dispatch-build.yml @@ -0,0 +1,25 @@ +name: Announce a revision to autobuilds + +# Discovery starts here: every commit that lands on the development branch +# is a new build input, and autobuilds is told about it. It deduplicates by +# build_id, so announcing a revision it already built costs one job. +on: + push: + branches: [master] + +jobs: + announce: + runs-on: ubuntu-24.04 + steps: + - name: Ask autobuilds to build this revision + env: + GH_TOKEN: ${{ secrets.REPO_DISPATCH_TOKEN }} + shell: bash + run: | + if [[ -z "$GH_TOKEN" ]]; then + echo "::error::REPO_DISPATCH_TOKEN secret is required for cross-repo dispatch to vitasdk/autobuilds" + exit 1 + fi + gh api repos/vitasdk/autobuilds/dispatches \ + -f event_type=run_build \ + -f "client_payload[buildscripts_ref]=$GITHUB_SHA" From eef74067499217497ecd30fce9e6250ab6ff25ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Tue, 25 Aug 2026 23:44:03 +0200 Subject: [PATCH 3/3] Refuse a pin move that would roll the source back Rehearsing the bot against the real upstreams found newlib's `vita` branch 2445 commits behind the pinned revision, missing the 4.3, 4.4, 4.5 and 4.6 merges the SDK is actually built from. Following it would have undone the 4.6 upgrade on the first nightly, at 03:00, and the only sign would have been whatever broke afterwards. That is fixed where it was broken -- `vita` now holds what the SDK builds -- but it was luck that somebody looked. So a pin only moves to a revision that contains it. A force-push upstream, a branch that is not where the work happens, or a pin taken from somewhere else stops the bot and names both revisions. The check asks the remote for commits and nothing else: the whole history of the largest leaf is a few megabytes and about three seconds, and only a pin that is actually moving pays for it. --- scripts/bump-pins.py | 53 +++++++++++++++++++++++++++++++- tests/ci/test-bump-pins.sh | 62 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/scripts/bump-pins.py b/scripts/bump-pins.py index 313a734..9fbdeba 100755 --- a/scripts/bump-pins.py +++ b/scripts/bump-pins.py @@ -3,10 +3,11 @@ import argparse import json -import os import re +import shutil import subprocess import sys +import tempfile COMPONENTS_PATH = "cmake/Components.cmake" TRACKING_PATH = "cmake/pin-tracking.json" @@ -116,6 +117,50 @@ def resolve_branch(repository, branch): return result.stdout.split()[0] +def contains(repository, pin, branch): + """Whether what the branch holds now was built on top of the pin. + + A pin that moves to a revision not descended from it is a rollback: + a force-push upstream, a branch that is not where the work happens, + a pin taken from somewhere else. The bot refuses and a person looks. + """ + + probe = tempfile.mkdtemp(prefix="bump-pins-") + try: + subprocess.run(["git", "init", "--quiet", probe], check=True) + # tree:0 asks for commits and nothing else: the whole history of + # the largest leaf arrives in a few megabytes and a few seconds. + fetch = subprocess.run( + [ + "git", "-C", probe, "fetch", "--quiet", "--filter=tree:0", + "--no-tags", repository, + f"+refs/heads/{branch}:refs/heads/tracked", pin, + ], + capture_output=True, + text=True, + ) + if fetch.returncode != 0: + raise BumpError( + f"{repository} cannot serve {pin} and {branch} together: " + f"{fetch.stderr.strip() or 'git fetch failed'}" + ) + ancestor = subprocess.run( + ["git", "-C", probe, "merge-base", "--is-ancestor", pin, "refs/heads/tracked"], + capture_output=True, + text=True, + ) + # 1 is the answer "no"; anything else is git saying it could not + # work out the question, which must not read as a rollback. + if ancestor.returncode not in (0, 1): + raise BumpError( + f"{repository}: cannot tell whether {branch} contains {pin}: " + f"{ancestor.stderr.strip() or 'git merge-base failed'}" + ) + return ancestor.returncode == 0 + finally: + shutil.rmtree(probe, ignore_errors=True) + + def rewrite_pin(text, variable, old, new): pattern = re.compile( r"(set\(\s*" + re.escape(variable) + r"_TAG\s+)" + re.escape(old) + r"(?=[\s)])" @@ -160,6 +205,12 @@ def bump(components_path, tracking_path, overrides, dry_run): head = resolve_branch(repository, tracked[name]) if head == component["pin"]: continue + if not contains(repository, component["pin"], tracked[name]): + raise BumpError( + f"{name}: {tracked[name]} is at {head}, which does not contain " + f"the pinned {component['pin']}; moving it would roll the " + "source back" + ) text = rewrite_pin(text, component["variable"], component["pin"], head) moves.append( { diff --git a/tests/ci/test-bump-pins.sh b/tests/ci/test-bump-pins.sh index 275ad73..22c7887 100755 --- a/tests/ci/test-bump-pins.sh +++ b/tests/ci/test-bump-pins.sh @@ -20,6 +20,22 @@ bump() { --components "$components" --tracking "$tracking" "$@" } +seed_pin() { + python3 - "$components" "$1" "$2" <<'PYEOF' +import re, sys +path, name, revision = sys.argv[1], sys.argv[2], sys.argv[3] +variable = name.upper().replace("-", "_") +text, replaced = re.subn( + r"(set\(\s*" + variable + r"_TAG\s+)\S+", + lambda match: match.group(1) + revision, + open(path).read(), + count=1, +) +assert replaced == 1, f"no {variable}_TAG to seed" +open(path, "w").write(text) +PYEOF +} + # The upstreams the bot follows, straight from the file under test: a # component added to the config is exercised here without editing this test. mapfile -t tracked < <(python3 - "$tracking" <<'PYEOF' @@ -48,6 +64,10 @@ for entry in "${tracked[@]}"; do git -C "$upstream" commit --quiet -m "first" upstream_of[$name]=$upstream overrides+=(--repository "$name=$upstream") + # The pins start where these fixtures start: the real ones name + # revisions no fixture can serve, and a bump is a move from something + # the upstream actually holds. + seed_pin "$name" "$(git -C "$upstream" rev-parse HEAD)" done pin_of() { @@ -61,6 +81,11 @@ PYEOF } # Everything moves once, to exactly what the upstream branches hold. +for entry in "${tracked[@]}"; do + read -r name _ <<< "$entry" + echo "moved" > "${upstream_of[$name]}/file" + git -C "${upstream_of[$name]}" commit --quiet -am "moved" +done output=$(bump "${overrides[@]}") grep -q "bump-pins: ${#tracked[@]} pin(s) moved" <<< "$output" || { printf 'the first bump did not move every tracked pin: %s\n' "$output" >&2 @@ -142,6 +167,43 @@ cmp -s "$components" "$temporary_root/before-dry" || { exit 1 } +# A branch that no longer contains the pin is a rollback, whatever the +# reason -- a force-push, or a pin that lives on another branch. The bot +# refuses and names both revisions instead of quietly going backwards. +git -C "$moved_upstream" branch keep +git -C "$moved_upstream" reset --quiet --hard HEAD~2 +cp "$components" "$temporary_root/before-rollback" +if output=$(bump "${overrides[@]}" 2>&1); then + printf 'the bot accepted a branch that had gone backwards\n' >&2 + exit 1 +fi +grep -q "$moved_name" <<< "$output" || { + printf 'the refusal does not name the rolled back component: %s\n' "$output" >&2 + exit 1 +} +grep -q "$(pin_of "$moved_name")" <<< "$output" || { + printf 'the refusal does not name the pin it kept: %s\n' "$output" >&2 + exit 1 +} +cmp -s "$components" "$temporary_root/before-rollback" || { + printf 'a refused bump rewrote Components.cmake\n' >&2 + exit 1 +} +git -C "$moved_upstream" reset --quiet --hard keep + +# A pin git cannot resolve is git failing to answer, not an answer: it +# must not be reported as a source going backwards. +seed_pin "$moved_name" "not-a-revision" +if output=$(bump "${overrides[@]}" 2>&1); then + printf 'the bot ran with a pin that is not a revision\n' >&2 + exit 1 +fi +if grep -q 'roll the source back' <<< "$output"; then + printf 'an unresolvable pin was reported as a rollback: %s\n' "$output" >&2 + exit 1 +fi +seed_pin "$moved_name" "$moved_head" + # The pins the config leaves alone stay exactly as the file declares them, # and are never resolved: no override is given for any of them here. for name in $(python3 -c 'import json,sys; print(" ".join(sorted(json.load(open(sys.argv[1]))["untracked"])))' "$tracking"); do