Move the pins by cron, and tell autobuilds when they move

.github/workflows/build.yml

@@ -3,8 +3,6 @@ name: Build
 on:
   push:
   pull_request:
-  repository_dispatch:
-    types: [run_build]
 
 # A push to a branch with an open pull request fires both triggers; one
 # group per head commit keeps a single run of the pair alive.
@@ -40,7 +38,7 @@ jobs:
           echo "== $test"
           "$test"
         done
-    - name: Run the reusable workflow tests
+    - name: Run the CI script tests
       run: |
         for test in tests/ci/test-*.sh; do
           echo "== $test"

.github/workflows/bump-pins.yml

@@ -0,0 +1,54 @@
+name: Bump source pins
+
+# Upstream movement enters the build as a commit here, never as a floating
+# ref: what a nightly was built from is always readable in this history.
+on:
+  schedule:
+    - cron: "0 3 * * *"
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: Report what would move without committing it
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  bump:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Check the push token is configured
+        env:
+          BUMP_TOKEN: ${{ secrets.REPO_DISPATCH_TOKEN }}
+        shell: bash
+        run: |
+          if [[ -z "$BUMP_TOKEN" ]]; then
+            echo "::error::REPO_DISPATCH_TOKEN is required: a push made with GITHUB_TOKEN fires no workflow, so the build would never be announced"
+            exit 1
+          fi
+
+      - uses: actions/checkout@v7
+        with:
+          # Not GITHUB_TOKEN on purpose: GitHub fires no workflow for a push
+          # made with it, and this push is what triggers discovery.
+          token: ${{ secrets.REPO_DISPATCH_TOKEN }}
+
+      - name: Resolve the tracked upstream branches
+        shell: bash
+        run: |
+          python3 scripts/bump-pins.py \
+            ${{ inputs.dry_run && '--dry-run' || '' }} \
+            --message-file "$RUNNER_TEMP/commit-message"
+
+      - name: Commit and push what moved
+        if: ${{ !inputs.dry_run }}
+        shell: bash
+        run: |
+          if git diff --quiet; then
+            echo "no pin moved; nothing to push"
+            exit 0
+          fi
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git commit -a --file "$RUNNER_TEMP/commit-message"
+          git push origin HEAD:"$GITHUB_REF_NAME"

.github/workflows/dispatch-build.yml

@@ -0,0 +1,25 @@
+name: Announce a revision to autobuilds
+
+# Discovery starts here: every commit that lands on the development branch
+# is a new build input, and autobuilds is told about it. It deduplicates by
+# build_id, so announcing a revision it already built costs one job.
+on:
+  push:
+    branches: [master]
+
+jobs:
+  announce:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Ask autobuilds to build this revision
+        env:
+          GH_TOKEN: ${{ secrets.REPO_DISPATCH_TOKEN }}
+        shell: bash
+        run: |
+          if [[ -z "$GH_TOKEN" ]]; then
+            echo "::error::REPO_DISPATCH_TOKEN secret is required for cross-repo dispatch to vitasdk/autobuilds"
+            exit 1
+          fi
+          gh api repos/vitasdk/autobuilds/dispatches \
+            -f event_type=run_build \
+            -f "client_payload[buildscripts_ref]=$GITHUB_SHA"

cmake/pin-tracking.json

@@ -0,0 +1,16 @@
+{
+  "schema": 1,
+  "_comment": "Which git-pinned components the pin-bump bot moves forward, and which it leaves alone. Every component pinned by repository plus revision must appear in exactly one of the two maps: scripts/bump-pins.py refuses to run otherwise, so a new pin cannot go unnoticed in either direction.",
+  "tracked": {
+    "newlib": {"branch": "vita"},
+    "samples": {"branch": "master"},
+    "headers": {"branch": "master"},
+    "toolchain": {"branch": "master"},
+    "pthread": {"branch": "master"}
+  },
+  "untracked": {
+    "isl": "a third-party release tag; moves with the GCC dependency set, by hand",
+    "vdpm": "a released tag; the vdpm release process moves it, and the core embeds the matching bundle",
+    "vita-makepkg": "a reviewed revision; its packaging assumptions are load-bearing for the core package"
+  }
+}

scripts/bump-pins.py

@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+"""Moves the tracked source pins to what their upstream branches hold now."""
+
+import argparse
+import json
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+COMPONENTS_PATH = "cmake/Components.cmake"
+TRACKING_PATH = "cmake/pin-tracking.json"
+SCHEMA = 1
+
+SET_PATTERN = re.compile(r'set\(\s*([A-Za-z0-9_]+)\s+"?([^")\s]+)"?')
+VARIABLE_REFERENCE = re.compile(r"\$\{([A-Za-z0-9_]+)\}")
+
+
+class BumpError(Exception):
+    pass
+
+
+def component_name(variable):
+    return variable.lower().replace("_", "-")
+
+
+def parse_components(text):
+    """Every component pinned by repository plus revision, by lock name."""
+
+    raw = dict(SET_PATTERN.findall(text))
+
+    def resolve(value, seen=()):
+        def substitute(match):
+            name = match.group(1)
+            if name in seen:
+                raise BumpError(f"{COMPONENTS_PATH}: circular reference on {name}")
+            if name not in raw:
+                raise BumpError(
+                    f"{COMPONENTS_PATH} references undefined variable {name}"
+                )
+            return resolve(raw[name], seen + (name,))
+
+        return VARIABLE_REFERENCE.sub(substitute, value)
+
+    components = {}
+    for variable, value in raw.items():
+        if not variable.endswith("_REPOSITORY"):
+            continue
+        prefix = variable[: -len("_REPOSITORY")]
+        if f"{prefix}_TAG" not in raw:
+            continue
+        components[component_name(prefix)] = {
+            "variable": prefix,
+            "repository": resolve(value),
+            "pin": resolve(raw[f"{prefix}_TAG"]),
+        }
+    if not components:
+        raise BumpError(f"{COMPONENTS_PATH} declares no git-pinned component")
+    return components
+
+
+def load_tracking(path, components):
+    try:
+        with open(path, encoding="utf-8") as handle:
+            config = json.load(handle)
+    except OSError as exc:
+        raise BumpError(f"{path} cannot be read: {exc}") from exc
+    except json.JSONDecodeError as exc:
+        raise BumpError(f"{path} is not valid JSON: {exc}") from exc
+
+    if config.get("schema") != SCHEMA:
+        raise BumpError(f"{path} declares schema {config.get('schema')}, expected {SCHEMA}")
+    tracked = config.get("tracked")
+    untracked = config.get("untracked")
+    if not isinstance(tracked, dict) or not isinstance(untracked, dict):
+        raise BumpError(f"{path} must declare 'tracked' and 'untracked' as objects")
+
+    both = sorted(set(tracked) & set(untracked))
+    if both:
+        raise BumpError(f"{path} lists as both tracked and untracked: {', '.join(both)}")
+    unknown = sorted((set(tracked) | set(untracked)) - set(components))
+    if unknown:
+        raise BumpError(
+            f"{path} names components that are not pinned in {COMPONENTS_PATH}: "
+            f"{', '.join(unknown)}"
+        )
+    # The point of the file: a pin added to the build cannot stay undeclared,
+    # in either direction, without this failing before anything is resolved.
+    undeclared = sorted(set(components) - set(tracked) - set(untracked))
+    if undeclared:
+        raise BumpError(
+            f"{path} declares neither tracking nor a reason for: {', '.join(undeclared)}"
+        )
+
+    for name, entry in tracked.items():
+        branch = entry.get("branch") if isinstance(entry, dict) else None
+        if not branch:
+            raise BumpError(f"{path}: tracked component {name} declares no branch")
+    for name, reason in untracked.items():
+        if not isinstance(reason, str) or not reason.strip():
+            raise BumpError(f"{path}: untracked component {name} gives no reason")
+    return {name: entry["branch"] for name, entry in tracked.items()}
+
+
+def resolve_branch(repository, branch):
+    result = subprocess.run(
+        ["git", "ls-remote", "--exit-code", repository, f"refs/heads/{branch}"],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        raise BumpError(
+            f"{repository} has no branch {branch}: "
+            f"{result.stderr.strip() or 'git ls-remote failed'}"
+        )
+    return result.stdout.split()[0]
+
+
+def contains(repository, pin, branch):
+    """Whether what the branch holds now was built on top of the pin.
+
+    A pin that moves to a revision not descended from it is a rollback:
+    a force-push upstream, a branch that is not where the work happens,
+    a pin taken from somewhere else. The bot refuses and a person looks.
+    """
+
+    probe = tempfile.mkdtemp(prefix="bump-pins-")
+    try:
+        subprocess.run(["git", "init", "--quiet", probe], check=True)
+        # tree:0 asks for commits and nothing else: the whole history of
+        # the largest leaf arrives in a few megabytes and a few seconds.
+        fetch = subprocess.run(
+            [
+                "git", "-C", probe, "fetch", "--quiet", "--filter=tree:0",
+                "--no-tags", repository,
+                f"+refs/heads/{branch}:refs/heads/tracked", pin,
+            ],
+            capture_output=True,
+            text=True,
+        )
+        if fetch.returncode != 0:
+            raise BumpError(
+                f"{repository} cannot serve {pin} and {branch} together: "
+                f"{fetch.stderr.strip() or 'git fetch failed'}"
+            )
+        ancestor = subprocess.run(
+            ["git", "-C", probe, "merge-base", "--is-ancestor", pin, "refs/heads/tracked"],
+            capture_output=True,
+            text=True,
+        )
+        # 1 is the answer "no"; anything else is git saying it could not
+        # work out the question, which must not read as a rollback.
+        if ancestor.returncode not in (0, 1):
+            raise BumpError(
+                f"{repository}: cannot tell whether {branch} contains {pin}: "
+                f"{ancestor.stderr.strip() or 'git merge-base failed'}"
+            )
+        return ancestor.returncode == 0
+    finally:
+        shutil.rmtree(probe, ignore_errors=True)
+
+
+def rewrite_pin(text, variable, old, new):
+    pattern = re.compile(
+        r"(set\(\s*" + re.escape(variable) + r"_TAG\s+)" + re.escape(old) + r"(?=[\s)])"
+    )
+    rewritten, count = pattern.subn(lambda match: match.group(1) + new, text)
+    if count != 1:
+        raise BumpError(
+            f"{COMPONENTS_PATH}: expected one {variable}_TAG assignment, found {count}"
+        )
+    return rewritten
+
+
+def commit_message(moves):
+    names = [move["component"] for move in moves]
+    if len(names) == 1:
+        subject = f"Move the {names[0]} pin to {moves[0]['new'][:9]}"
+    elif len(names) == 2:
+        subject = f"Move the {names[0]} and {names[1]} pins forward"
+    else:
+        subject = f"Move {len(names)} source pins forward"
+    body = "\n".join(
+        f"{move['component']} {move['branch']}: {move['old'][:9]} -> {move['new'][:9]}"
+        for move in moves
+    )
+    return f"{subject}\n\n{body}\n"
+
+
+def bump(components_path, tracking_path, overrides, dry_run):
+    try:
+        with open(components_path, encoding="utf-8") as handle:
+            text = handle.read()
+    except OSError as exc:
+        raise BumpError(f"{components_path} cannot be read: {exc}") from exc
+
+    components = parse_components(text)
+    tracked = load_tracking(tracking_path, components)
+
+    moves = []
+    for name in sorted(tracked):
+        component = components[name]
+        repository = overrides.get(name, component["repository"])
+        head = resolve_branch(repository, tracked[name])
+        if head == component["pin"]:
+            continue
+        if not contains(repository, component["pin"], tracked[name]):
+            raise BumpError(
+                f"{name}: {tracked[name]} is at {head}, which does not contain "
+                f"the pinned {component['pin']}; moving it would roll the "
+                "source back"
+            )
+        text = rewrite_pin(text, component["variable"], component["pin"], head)
+        moves.append(
+            {
+                "component": name,
+                "branch": tracked[name],
+                "old": component["pin"],
+                "new": head,
+            }
+        )
+
+    if moves and not dry_run:
+        with open(components_path, "w", encoding="utf-8") as handle:
+            handle.write(text)
+    return moves
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(prog="bump-pins")
+    parser.add_argument("--components", default=COMPONENTS_PATH)
+    parser.add_argument("--tracking", default=TRACKING_PATH)
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--message-file")
+    parser.add_argument(
+        "--repository",
+        action="append",
+        default=[],
+        metavar="COMPONENT=URL",
+        help="resolve one component somewhere else than Components.cmake says",
+    )
+    args = parser.parse_args(argv)
+
+    overrides = {}
+    for override in args.repository:
+        name, separator, url = override.partition("=")
+        if not separator or not name or not url:
+            print(f"bump-pins: --repository wants COMPONENT=URL, got {override}", file=sys.stderr)
+            return 1
+        overrides[name] = url
+
+    try:
+        moves = bump(args.components, args.tracking, overrides, args.dry_run)
+    except BumpError as exc:
+        print(f"bump-pins: {exc}", file=sys.stderr)
+        return 1
+
+    for move in moves:
+        print(f"{move['component']} {move['branch']}: {move['old']} -> {move['new']}")
+    print(f"bump-pins: {len(moves)} pin(s) moved" if moves else "bump-pins: nothing moved")
+
+    if args.message_file:
+        message = commit_message(moves) if moves else ""
+        with open(args.message_file, "w", encoding="utf-8") as handle:
+            handle.write(message)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))