diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bcb1024..fdd8c24 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,8 +3,6 @@ name: Build on: push: pull_request: - repository_dispatch: - types: [run_build] # A push to a branch with an open pull request fires both triggers; one # group per head commit keeps a single run of the pair alive. @@ -40,7 +38,7 @@ jobs: echo "== $test" "$test" done - - name: Run the reusable workflow tests + - name: Run the CI script tests run: | for test in tests/ci/test-*.sh; do echo "== $test" diff --git a/.github/workflows/bump-pins.yml b/.github/workflows/bump-pins.yml new file mode 100644 index 0000000..4b2ddb6 --- /dev/null +++ b/.github/workflows/bump-pins.yml @@ -0,0 +1,54 @@ +name: Bump source pins + +# Upstream movement enters the build as a commit here, never as a floating +# ref: what a nightly was built from is always readable in this history. +on: + schedule: + - cron: "0 3 * * *" + workflow_dispatch: + inputs: + dry_run: + description: Report what would move without committing it + required: false + default: false + type: boolean + +jobs: + bump: + runs-on: ubuntu-24.04 + steps: + - name: Check the push token is configured + env: + BUMP_TOKEN: ${{ secrets.REPO_DISPATCH_TOKEN }} + shell: bash + run: | + if [[ -z "$BUMP_TOKEN" ]]; then + echo "::error::REPO_DISPATCH_TOKEN is required: a push made with GITHUB_TOKEN fires no workflow, so the build would never be announced" + exit 1 + fi + + - uses: actions/checkout@v7 + with: + # Not GITHUB_TOKEN on purpose: GitHub fires no workflow for a push + # made with it, and this push is what triggers discovery. + token: ${{ secrets.REPO_DISPATCH_TOKEN }} + + - name: Resolve the tracked upstream branches + shell: bash + run: | + python3 scripts/bump-pins.py \ + ${{ inputs.dry_run && '--dry-run' || '' }} \ + --message-file "$RUNNER_TEMP/commit-message" + + - name: Commit and push what moved + if: ${{ !inputs.dry_run }} + shell: bash + run: | + if git diff --quiet; then + echo "no pin moved; nothing to push" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -a --file "$RUNNER_TEMP/commit-message" + git push origin HEAD:"$GITHUB_REF_NAME" diff --git a/.github/workflows/dispatch-build.yml b/.github/workflows/dispatch-build.yml new file mode 100644 index 0000000..a3d0e5b --- /dev/null +++ b/.github/workflows/dispatch-build.yml @@ -0,0 +1,25 @@ +name: Announce a revision to autobuilds + +# Discovery starts here: every commit that lands on the development branch +# is a new build input, and autobuilds is told about it. It deduplicates by +# build_id, so announcing a revision it already built costs one job. +on: + push: + branches: [master] + +jobs: + announce: + runs-on: ubuntu-24.04 + steps: + - name: Ask autobuilds to build this revision + env: + GH_TOKEN: ${{ secrets.REPO_DISPATCH_TOKEN }} + shell: bash + run: | + if [[ -z "$GH_TOKEN" ]]; then + echo "::error::REPO_DISPATCH_TOKEN secret is required for cross-repo dispatch to vitasdk/autobuilds" + exit 1 + fi + gh api repos/vitasdk/autobuilds/dispatches \ + -f event_type=run_build \ + -f "client_payload[buildscripts_ref]=$GITHUB_SHA" diff --git a/cmake/pin-tracking.json b/cmake/pin-tracking.json new file mode 100644 index 0000000..568fdf9 --- /dev/null +++ b/cmake/pin-tracking.json @@ -0,0 +1,16 @@ +{ + "schema": 1, + "_comment": "Which git-pinned components the pin-bump bot moves forward, and which it leaves alone. Every component pinned by repository plus revision must appear in exactly one of the two maps: scripts/bump-pins.py refuses to run otherwise, so a new pin cannot go unnoticed in either direction.", + "tracked": { + "newlib": {"branch": "vita"}, + "samples": {"branch": "master"}, + "headers": {"branch": "master"}, + "toolchain": {"branch": "master"}, + "pthread": {"branch": "master"} + }, + "untracked": { + "isl": "a third-party release tag; moves with the GCC dependency set, by hand", + "vdpm": "a released tag; the vdpm release process moves it, and the core embeds the matching bundle", + "vita-makepkg": "a reviewed revision; its packaging assumptions are load-bearing for the core package" + } +} diff --git a/scripts/bump-pins.py b/scripts/bump-pins.py new file mode 100755 index 0000000..9fbdeba --- /dev/null +++ b/scripts/bump-pins.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Moves the tracked source pins to what their upstream branches hold now.""" + +import argparse +import json +import re +import shutil +import subprocess +import sys +import tempfile + +COMPONENTS_PATH = "cmake/Components.cmake" +TRACKING_PATH = "cmake/pin-tracking.json" +SCHEMA = 1 + +SET_PATTERN = re.compile(r'set\(\s*([A-Za-z0-9_]+)\s+"?([^")\s]+)"?') +VARIABLE_REFERENCE = re.compile(r"\$\{([A-Za-z0-9_]+)\}") + + +class BumpError(Exception): + pass + + +def component_name(variable): + return variable.lower().replace("_", "-") + + +def parse_components(text): + """Every component pinned by repository plus revision, by lock name.""" + + raw = dict(SET_PATTERN.findall(text)) + + def resolve(value, seen=()): + def substitute(match): + name = match.group(1) + if name in seen: + raise BumpError(f"{COMPONENTS_PATH}: circular reference on {name}") + if name not in raw: + raise BumpError( + f"{COMPONENTS_PATH} references undefined variable {name}" + ) + return resolve(raw[name], seen + (name,)) + + return VARIABLE_REFERENCE.sub(substitute, value) + + components = {} + for variable, value in raw.items(): + if not variable.endswith("_REPOSITORY"): + continue + prefix = variable[: -len("_REPOSITORY")] + if f"{prefix}_TAG" not in raw: + continue + components[component_name(prefix)] = { + "variable": prefix, + "repository": resolve(value), + "pin": resolve(raw[f"{prefix}_TAG"]), + } + if not components: + raise BumpError(f"{COMPONENTS_PATH} declares no git-pinned component") + return components + + +def load_tracking(path, components): + try: + with open(path, encoding="utf-8") as handle: + config = json.load(handle) + except OSError as exc: + raise BumpError(f"{path} cannot be read: {exc}") from exc + except json.JSONDecodeError as exc: + raise BumpError(f"{path} is not valid JSON: {exc}") from exc + + if config.get("schema") != SCHEMA: + raise BumpError(f"{path} declares schema {config.get('schema')}, expected {SCHEMA}") + tracked = config.get("tracked") + untracked = config.get("untracked") + if not isinstance(tracked, dict) or not isinstance(untracked, dict): + raise BumpError(f"{path} must declare 'tracked' and 'untracked' as objects") + + both = sorted(set(tracked) & set(untracked)) + if both: + raise BumpError(f"{path} lists as both tracked and untracked: {', '.join(both)}") + unknown = sorted((set(tracked) | set(untracked)) - set(components)) + if unknown: + raise BumpError( + f"{path} names components that are not pinned in {COMPONENTS_PATH}: " + f"{', '.join(unknown)}" + ) + # The point of the file: a pin added to the build cannot stay undeclared, + # in either direction, without this failing before anything is resolved. + undeclared = sorted(set(components) - set(tracked) - set(untracked)) + if undeclared: + raise BumpError( + f"{path} declares neither tracking nor a reason for: {', '.join(undeclared)}" + ) + + for name, entry in tracked.items(): + branch = entry.get("branch") if isinstance(entry, dict) else None + if not branch: + raise BumpError(f"{path}: tracked component {name} declares no branch") + for name, reason in untracked.items(): + if not isinstance(reason, str) or not reason.strip(): + raise BumpError(f"{path}: untracked component {name} gives no reason") + return {name: entry["branch"] for name, entry in tracked.items()} + + +def resolve_branch(repository, branch): + result = subprocess.run( + ["git", "ls-remote", "--exit-code", repository, f"refs/heads/{branch}"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise BumpError( + f"{repository} has no branch {branch}: " + f"{result.stderr.strip() or 'git ls-remote failed'}" + ) + return result.stdout.split()[0] + + +def contains(repository, pin, branch): + """Whether what the branch holds now was built on top of the pin. + + A pin that moves to a revision not descended from it is a rollback: + a force-push upstream, a branch that is not where the work happens, + a pin taken from somewhere else. The bot refuses and a person looks. + """ + + probe = tempfile.mkdtemp(prefix="bump-pins-") + try: + subprocess.run(["git", "init", "--quiet", probe], check=True) + # tree:0 asks for commits and nothing else: the whole history of + # the largest leaf arrives in a few megabytes and a few seconds. + fetch = subprocess.run( + [ + "git", "-C", probe, "fetch", "--quiet", "--filter=tree:0", + "--no-tags", repository, + f"+refs/heads/{branch}:refs/heads/tracked", pin, + ], + capture_output=True, + text=True, + ) + if fetch.returncode != 0: + raise BumpError( + f"{repository} cannot serve {pin} and {branch} together: " + f"{fetch.stderr.strip() or 'git fetch failed'}" + ) + ancestor = subprocess.run( + ["git", "-C", probe, "merge-base", "--is-ancestor", pin, "refs/heads/tracked"], + capture_output=True, + text=True, + ) + # 1 is the answer "no"; anything else is git saying it could not + # work out the question, which must not read as a rollback. + if ancestor.returncode not in (0, 1): + raise BumpError( + f"{repository}: cannot tell whether {branch} contains {pin}: " + f"{ancestor.stderr.strip() or 'git merge-base failed'}" + ) + return ancestor.returncode == 0 + finally: + shutil.rmtree(probe, ignore_errors=True) + + +def rewrite_pin(text, variable, old, new): + pattern = re.compile( + r"(set\(\s*" + re.escape(variable) + r"_TAG\s+)" + re.escape(old) + r"(?=[\s)])" + ) + rewritten, count = pattern.subn(lambda match: match.group(1) + new, text) + if count != 1: + raise BumpError( + f"{COMPONENTS_PATH}: expected one {variable}_TAG assignment, found {count}" + ) + return rewritten + + +def commit_message(moves): + names = [move["component"] for move in moves] + if len(names) == 1: + subject = f"Move the {names[0]} pin to {moves[0]['new'][:9]}" + elif len(names) == 2: + subject = f"Move the {names[0]} and {names[1]} pins forward" + else: + subject = f"Move {len(names)} source pins forward" + body = "\n".join( + f"{move['component']} {move['branch']}: {move['old'][:9]} -> {move['new'][:9]}" + for move in moves + ) + return f"{subject}\n\n{body}\n" + + +def bump(components_path, tracking_path, overrides, dry_run): + try: + with open(components_path, encoding="utf-8") as handle: + text = handle.read() + except OSError as exc: + raise BumpError(f"{components_path} cannot be read: {exc}") from exc + + components = parse_components(text) + tracked = load_tracking(tracking_path, components) + + moves = [] + for name in sorted(tracked): + component = components[name] + repository = overrides.get(name, component["repository"]) + head = resolve_branch(repository, tracked[name]) + if head == component["pin"]: + continue + if not contains(repository, component["pin"], tracked[name]): + raise BumpError( + f"{name}: {tracked[name]} is at {head}, which does not contain " + f"the pinned {component['pin']}; moving it would roll the " + "source back" + ) + text = rewrite_pin(text, component["variable"], component["pin"], head) + moves.append( + { + "component": name, + "branch": tracked[name], + "old": component["pin"], + "new": head, + } + ) + + if moves and not dry_run: + with open(components_path, "w", encoding="utf-8") as handle: + handle.write(text) + return moves + + +def main(argv): + parser = argparse.ArgumentParser(prog="bump-pins") + parser.add_argument("--components", default=COMPONENTS_PATH) + parser.add_argument("--tracking", default=TRACKING_PATH) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--message-file") + parser.add_argument( + "--repository", + action="append", + default=[], + metavar="COMPONENT=URL", + help="resolve one component somewhere else than Components.cmake says", + ) + args = parser.parse_args(argv) + + overrides = {} + for override in args.repository: + name, separator, url = override.partition("=") + if not separator or not name or not url: + print(f"bump-pins: --repository wants COMPONENT=URL, got {override}", file=sys.stderr) + return 1 + overrides[name] = url + + try: + moves = bump(args.components, args.tracking, overrides, args.dry_run) + except BumpError as exc: + print(f"bump-pins: {exc}", file=sys.stderr) + return 1 + + for move in moves: + print(f"{move['component']} {move['branch']}: {move['old']} -> {move['new']}") + print(f"bump-pins: {len(moves)} pin(s) moved" if moves else "bump-pins: nothing moved") + + if args.message_file: + message = commit_message(moves) if moves else "" + with open(args.message_file, "w", encoding="utf-8") as handle: + handle.write(message) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/ci/test-bump-pins.sh b/tests/ci/test-bump-pins.sh new file mode 100755 index 0000000..22c7887 --- /dev/null +++ b/tests/ci/test-bump-pins.sh @@ -0,0 +1,276 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repository_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P) +temporary_root=$(mktemp -d "${TMPDIR:-/tmp}/vitasdk-bump-pins.XXXXXXXX") +cleanup() { rm -rf -- "$temporary_root"; } +trap cleanup EXIT + +work="$temporary_root/work" +mkdir -p "$work/cmake" +cp "$repository_root/cmake/Components.cmake" "$work/cmake/Components.cmake" +cp "$repository_root/cmake/pin-tracking.json" "$work/cmake/pin-tracking.json" + +components="$work/cmake/Components.cmake" +tracking="$work/cmake/pin-tracking.json" + +bump() { + python3 "$repository_root/scripts/bump-pins.py" \ + --components "$components" --tracking "$tracking" "$@" +} + +seed_pin() { + python3 - "$components" "$1" "$2" <<'PYEOF' +import re, sys +path, name, revision = sys.argv[1], sys.argv[2], sys.argv[3] +variable = name.upper().replace("-", "_") +text, replaced = re.subn( + r"(set\(\s*" + variable + r"_TAG\s+)\S+", + lambda match: match.group(1) + revision, + open(path).read(), + count=1, +) +assert replaced == 1, f"no {variable}_TAG to seed" +open(path, "w").write(text) +PYEOF +} + +# The upstreams the bot follows, straight from the file under test: a +# component added to the config is exercised here without editing this test. +mapfile -t tracked < <(python3 - "$tracking" <<'PYEOF' +import json, sys +config = json.load(open(sys.argv[1])) +for name, entry in sorted(config["tracked"].items()): + print(f"{name} {entry['branch']}") +PYEOF +) +[[ ${#tracked[@]} -ge 2 ]] || { + printf 'the tracking config declares fewer than two components\n' >&2 + exit 1 +} + +overrides=() +declare -A upstream_of=() +for entry in "${tracked[@]}"; do + read -r name branch <<< "$entry" + upstream="$temporary_root/upstream/$name" + mkdir -p "$upstream" + git init --quiet --initial-branch "$branch" "$upstream" + git -C "$upstream" config user.email bump-pins-tests@ci.invalid + git -C "$upstream" config user.name "bump-pins tests" + echo "first" > "$upstream/file" + git -C "$upstream" add file + git -C "$upstream" commit --quiet -m "first" + upstream_of[$name]=$upstream + overrides+=(--repository "$name=$upstream") + # The pins start where these fixtures start: the real ones name + # revisions no fixture can serve, and a bump is a move from something + # the upstream actually holds. + seed_pin "$name" "$(git -C "$upstream" rev-parse HEAD)" +done + +pin_of() { + python3 - "$components" "$1" <<'PYEOF' +import re, sys +variable = sys.argv[2].upper().replace("-", "_") +text = open(sys.argv[1]).read() +match = re.search(r"set\(\s*" + variable + r"_TAG\s+(\S+)", text) +print(match.group(1) if match else "") +PYEOF +} + +# Everything moves once, to exactly what the upstream branches hold. +for entry in "${tracked[@]}"; do + read -r name _ <<< "$entry" + echo "moved" > "${upstream_of[$name]}/file" + git -C "${upstream_of[$name]}" commit --quiet -am "moved" +done +output=$(bump "${overrides[@]}") +grep -q "bump-pins: ${#tracked[@]} pin(s) moved" <<< "$output" || { + printf 'the first bump did not move every tracked pin: %s\n' "$output" >&2 + exit 1 +} +for entry in "${tracked[@]}"; do + read -r name _ <<< "$entry" + expected=$(git -C "${upstream_of[$name]}" rev-parse HEAD) + [[ $(pin_of "$name") == "$expected" ]] || { + printf 'pin of %s is not the upstream head\n' "$name" >&2 + exit 1 + } +done + +# Convergence: nothing upstream moved, so the bot writes nothing. Its own +# push must never be a reason for the next run to commit again. +cp "$components" "$temporary_root/before-idle" +output=$(bump "${overrides[@]}" --message-file "$temporary_root/idle-message") +grep -q 'bump-pins: nothing moved' <<< "$output" || { + printf 'an idle bump reported movement: %s\n' "$output" >&2 + exit 1 +} +cmp -s "$components" "$temporary_root/before-idle" || { + printf 'an idle bump rewrote Components.cmake\n' >&2 + exit 1 +} +[[ ! -s "$temporary_root/idle-message" ]] || { + printf 'an idle bump wrote a commit message\n' >&2 + exit 1 +} + +# One leaf moves: one pin changes, one line changes, and the message says +# which component it was. +read -r moved_name moved_branch <<< "${tracked[0]}" +moved_upstream=${upstream_of[$moved_name]} +echo "second" > "$moved_upstream/file" +git -C "$moved_upstream" commit --quiet -am "second" +moved_head=$(git -C "$moved_upstream" rev-parse HEAD) + +cp "$components" "$temporary_root/before-one" +output=$(bump "${overrides[@]}" --message-file "$temporary_root/one-message") +grep -q 'bump-pins: 1 pin(s) moved' <<< "$output" || { + printf 'moving one upstream did not move exactly one pin: %s\n' "$output" >&2 + exit 1 +} +[[ $(pin_of "$moved_name") == "$moved_head" ]] || { + printf 'pin of %s did not follow its upstream\n' "$moved_name" >&2 + exit 1 +} +changed=$(diff "$temporary_root/before-one" "$components" | grep '^[<>]' || true) +[[ $(grep -c . <<< "$changed") -eq 2 ]] || { + printf 'moving one pin changed more than one line:\n%s\n' "$changed" >&2 + exit 1 +} +[[ $(grep -c '_TAG' <<< "$changed") -eq 2 ]] || { + printf 'the bot changed something that is not a pin:\n%s\n' "$changed" >&2 + exit 1 +} +grep -q "$moved_name" "$temporary_root/one-message" || { + printf 'the commit message does not name %s\n' "$moved_name" >&2 + exit 1 +} +grep -q "$moved_branch" "$temporary_root/one-message" || { + printf 'the commit message does not name the branch followed\n' >&2 + exit 1 +} + +# A dry run reports what it would do and writes nothing. +echo "third" > "$moved_upstream/file" +git -C "$moved_upstream" commit --quiet -am "third" +cp "$components" "$temporary_root/before-dry" +output=$(bump "${overrides[@]}" --dry-run) +grep -q 'bump-pins: 1 pin(s) moved' <<< "$output" || { + printf 'the dry run did not report the pending move: %s\n' "$output" >&2 + exit 1 +} +cmp -s "$components" "$temporary_root/before-dry" || { + printf 'the dry run rewrote Components.cmake\n' >&2 + exit 1 +} + +# A branch that no longer contains the pin is a rollback, whatever the +# reason -- a force-push, or a pin that lives on another branch. The bot +# refuses and names both revisions instead of quietly going backwards. +git -C "$moved_upstream" branch keep +git -C "$moved_upstream" reset --quiet --hard HEAD~2 +cp "$components" "$temporary_root/before-rollback" +if output=$(bump "${overrides[@]}" 2>&1); then + printf 'the bot accepted a branch that had gone backwards\n' >&2 + exit 1 +fi +grep -q "$moved_name" <<< "$output" || { + printf 'the refusal does not name the rolled back component: %s\n' "$output" >&2 + exit 1 +} +grep -q "$(pin_of "$moved_name")" <<< "$output" || { + printf 'the refusal does not name the pin it kept: %s\n' "$output" >&2 + exit 1 +} +cmp -s "$components" "$temporary_root/before-rollback" || { + printf 'a refused bump rewrote Components.cmake\n' >&2 + exit 1 +} +git -C "$moved_upstream" reset --quiet --hard keep + +# A pin git cannot resolve is git failing to answer, not an answer: it +# must not be reported as a source going backwards. +seed_pin "$moved_name" "not-a-revision" +if output=$(bump "${overrides[@]}" 2>&1); then + printf 'the bot ran with a pin that is not a revision\n' >&2 + exit 1 +fi +if grep -q 'roll the source back' <<< "$output"; then + printf 'an unresolvable pin was reported as a rollback: %s\n' "$output" >&2 + exit 1 +fi +seed_pin "$moved_name" "$moved_head" + +# The pins the config leaves alone stay exactly as the file declares them, +# and are never resolved: no override is given for any of them here. +for name in $(python3 -c 'import json,sys; print(" ".join(sorted(json.load(open(sys.argv[1]))["untracked"])))' "$tracking"); do + before=$(python3 - "$repository_root/cmake/Components.cmake" "$name" <<'PYEOF' +import re, sys +variable = sys.argv[2].upper().replace("-", "_") +text = open(sys.argv[1]).read() +match = re.search(r"set\(\s*" + variable + r"_TAG\s+(\S+)", text) +print(match.group(1) if match else "") +PYEOF + ) + [[ $(pin_of "$name") == "$before" ]] || { + printf 'the bot moved the untracked pin %s\n' "$name" >&2 + exit 1 + } +done + +# A branch that is not there fails loudly, naming what could not be resolved. +missing_branch_overrides=("${overrides[@]}") +missing="$temporary_root/upstream/missing" +git init --quiet --initial-branch nowhere "$missing" +git -C "$missing" config user.email bump-pins-tests@ci.invalid +git -C "$missing" config user.name "bump-pins tests" +echo x > "$missing/file" +git -C "$missing" add file +git -C "$missing" commit --quiet -m x +if output=$(bump "${missing_branch_overrides[@]}" --repository "$moved_name=$missing" 2>&1); then + printf 'the bot accepted a branch that does not exist\n' >&2 + exit 1 +fi +grep -q "$moved_branch" <<< "$output" || { + printf 'the failure does not name the branch it could not resolve: %s\n' "$output" >&2 + exit 1 +} + +# A pin added to the build and to neither list stops the bot before it +# resolves anything, naming the component nobody decided about. +cp "$components" "$temporary_root/before-undeclared" +cat >> "$components" <<'CMAKEEOF' + +set(SOMETHING_REPOSITORY https://github.com/vitasdk/something) +set(SOMETHING_TAG 0000000000000000000000000000000000000000) +CMAKEEOF +if output=$(bump "${overrides[@]}" 2>&1); then + printf 'the bot ran with an undeclared pin\n' >&2 + exit 1 +fi +grep -q 'something' <<< "$output" || { + printf 'the failure does not name the undeclared pin: %s\n' "$output" >&2 + exit 1 +} +cp "$temporary_root/before-undeclared" "$components" + +# A tracked name that is not a pin at all is a typo, and says so. +python3 - "$tracking" <<'PYEOF' +import json, sys +config = json.load(open(sys.argv[1])) +config["tracked"]["bogus"] = {"branch": "master"} +json.dump(config, open(sys.argv[1], "w"), indent=2) +PYEOF +if output=$(bump "${overrides[@]}" 2>&1); then + printf 'the bot ran with a tracked component that is not pinned\n' >&2 + exit 1 +fi +grep -q 'bogus' <<< "$output" || { + printf 'the failure does not name the unknown component: %s\n' "$output" >&2 + exit 1 +} + +printf 'bump-pins: all checks passed\n'