Unsloth pin preflight #178
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-License-Identifier: AGPL-3.0-only | |
| # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. | |
| name: Unsloth pin preflight | |
| # Runs the nightly's merge a few hours early, on the same base tag it will | |
| # pick, and files the conflict before the schedule burns 39 build jobs on it. | |
| # | |
| # unsloth-pr-set-lint.yml checks that pins are well formed and belong to their | |
| # PR, which catches a bad edit. It cannot catch the failure that actually | |
| # recurs: a pin that was fine yesterday and stops merging today because the | |
| # base tag moved under it. That is what killed 08-02, and four nights in a | |
| # week between the two causes. | |
| on: | |
| schedule: | |
| - cron: '47 16 * * *' | |
| push: | |
| paths: | |
| - scripts/unsloth/pr-set.json | |
| workflow_dispatch: | |
| permissions: | |
| # write, not read: the mirror step pushes refs/pins/<sha>. With read it | |
| # failed every time with "Permission to unslothai/llama.cpp.git denied to | |
| # github-actions[bot]" (403), and the only refs that existed were ones | |
| # pushed by hand. | |
| contents: write | |
| issues: write | |
| # Two runs of the same ref probe the same pins against the same base, so the | |
| # second adds nothing and just competes for runners. On 08-04 a dispatch and the | |
| # schedule sat queued together for an hour. Newest wins: it sees the newest | |
| # pr-set.json. | |
| # | |
| # Per ref, though, not globally. This file also runs on any push that touches | |
| # pr-set.json, so with one shared group a push to a second branch cancelled the | |
| # first branch's run: observed on 09-03, where the run that would have said | |
| # whether a repin fixed the nightly was cancelled by an unrelated branch, and | |
| # the PR was left showing the failure from before the fix. | |
| concurrency: | |
| group: unsloth-pin-preflight-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| preflight: | |
| name: Dry-run the pin merges | |
| runs-on: ubuntu-24.04 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| REPIN_TOKEN: ${{ secrets.REPIN_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - name: Resolve base and dry-run the merges | |
| id: p | |
| run: | | |
| set -uo pipefail | |
| # Everything below reports through `status`/`details`, so a death | |
| # anywhere else leaves both empty and the alert blank: a red X on a | |
| # scheduled run nobody opens. Report the abort through the same | |
| # channel as a finding, so the repin bot sees a failure either way. | |
| trap 'rc=$?; if [ "$rc" != 0 ]; then { | |
| echo "status=failure" | |
| echo "details<<ALERT_EOF" | |
| echo "The preflight script exited ${rc} before it finished probing, so the pins were not fully checked. See the run log for the last command it reached." | |
| echo "ALERT_EOF"; } >> "$GITHUB_OUTPUT"; fi' EXIT | |
| AGE_H="${UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS:-6}" | |
| CUTOFF="$(date -u -d "-${AGE_H} hours" +%s)" | |
| # Same base tag the nightly resolves: newest aged b#### build. | |
| # Upstream marks those prerelease since 08-21, so match the tag shape. | |
| BASE="$(gh api 'repos/ggml-org/llama.cpp/releases?per_page=100' \ | |
| | jq -r --argjson cutoff "$CUTOFF" '[.[] | select(.draft==false) | select(.tag_name|test("^b[0-9]+$")) | select((.published_at|fromdateiso8601) <= $cutoff)] | max_by(.published_at|fromdateiso8601) | .tag_name')" | |
| if [ -z "$BASE" ] || [ "$BASE" = "null" ]; then | |
| echo "::warning::no aged upstream release found; skipping" | |
| echo "status=skip" >> "$GITHUB_OUTPUT"; exit 0 | |
| fi | |
| echo "base $BASE" | |
| git clone -q --filter=blob:none https://github.com/ggml-org/llama.cpp.git scratch | |
| cd scratch | |
| # GITHUB_TOKEN mirrors most pins, but GitHub refuses any ref that ADDS | |
| # a workflow file the repo does not already have. Observed on 08-03, | |
| # three pins mirrored and the fourth rejected: | |
| # ! [remote rejected] ... -> refs/pins/c3fb9724... | |
| # (refusing to allow a GitHub App to create or update workflow | |
| # `.github/workflows/build-self-hosted.yml` without `workflows` | |
| # permission) | |
| # The check applies to any ref, not just branches. REPIN_TOKEN has | |
| # workflow scope and covers those; without it we still mirror what we | |
| # can rather than nothing, and warn about the rest. | |
| MIRROR_TOKEN="${REPIN_TOKEN:-$GH_TOKEN}" | |
| MIRROR="https://x-access-token:${MIRROR_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" | |
| git fetch -q --no-tags origin "refs/tags/${BASE}:refs/tags/${BASE}" | |
| git checkout -q --detach "refs/tags/${BASE}" | |
| # Mirror every pin to refs/pins/<sha> before probing anything. The | |
| # 07-31 outage was a reviewed commit pruned out of its PR by a force | |
| # push, which no amount of checking can recover from after the fact. | |
| # A ref of our own keeps the object alive; resolve falls back to it. | |
| # All these repos are in one fork network, so this transfers nothing. | |
| # Done first, and past failures, so a conflict in an early pin does | |
| # not leave the later ones unmirrored. | |
| while read -r url; do | |
| SRC="$(sed -E 's|https://github.com/([^/]+)/llama.cpp/pull/.*|\1|' <<<"$url")/llama.cpp" | |
| SHA="$(sed -E 's|.*/commits/([0-9a-f]{40})/?$|\1|' <<<"$url")" | |
| if git ls-remote --exit-code "$MIRROR" "refs/pins/${SHA}" >/dev/null 2>&1; then | |
| echo "mirrored already: ${SHA:0:10}" | |
| continue | |
| fi | |
| if ! git fetch -q --no-tags "https://github.com/${SRC}.git" "$SHA" 2>/dev/null; then | |
| echo "::warning::cannot mirror ${SHA:0:10}; it is already unfetchable from ${SRC}" | |
| continue | |
| fi | |
| # Report git's own error. Guessing the cause hid a 403 behind a | |
| # workflow-scope message for a week. | |
| if ERR="$(git push -q "$MIRROR" "${SHA}:refs/pins/${SHA}" 2>&1)"; then | |
| echo "mirrored ${SHA:0:10}" | |
| else | |
| echo "::warning::could not mirror refs/pins/${SHA:0:10}: $(sed "s|${MIRROR_TOKEN}|***|g" <<<"$ERR" | tr '\n' ' '). A ref adding a workflow file needs REPIN_TOKEN (workflow scope); that pin stays deletable by a force-push." | |
| fi | |
| done < <(jq -r '.prs[] | if type == "string" then . else .url end' \ | |
| ../scripts/unsloth/pr-set.json) | |
| PROBLEMS="" | |
| MERGED="" | |
| while read -r url REQUIRED; do | |
| SRC="$(sed -E 's|https://github.com/([^/]+)/llama.cpp/pull/.*|\1|' <<<"$url")/llama.cpp" | |
| NUM="$(sed -E 's|.*/pull/([0-9]+)/commits/.*|\1|' <<<"$url")" | |
| SHA="$(sed -E 's|.*/commits/([0-9a-f]{40})/?$|\1|' <<<"$url")" | |
| STATE="$(gh api "repos/${SRC}/pulls/${NUM}" --jq .state 2>/dev/null || echo unknown)" | |
| if [ "$STATE" != "open" ]; then | |
| # Non-open required pins are still merged by the nightly, so keep | |
| # probing them here rather than reporting them as a problem; an | |
| # optional one is skipped there, so skip it here too. | |
| if [ "$REQUIRED" = "false" ]; then | |
| continue | |
| fi | |
| echo "note: ${SRC}#${NUM} is ${STATE}; still probing because required pins are merged regardless of state" | |
| fi | |
| if ! git fetch -q --no-tags "https://github.com/${SRC}.git" "$SHA" 2>/dev/null; then | |
| PROBLEMS="${PROBLEMS}- \`${SRC}#${NUM}\` pinned commit \`${SHA:0:10}\` cannot be fetched; it was probably force-pushed away.\n" | |
| continue | |
| fi | |
| if git -c user.name=preflight -c user.email=preflight@local \ | |
| -c merge.conflictStyle=diff3 \ | |
| merge --no-ff --no-edit -m "probe ${SRC}#${NUM}" "$SHA" >/dev/null 2>&1; then | |
| echo "ok ${SRC}#${NUM}" | |
| MERGED=1 | |
| continue | |
| fi | |
| # Mirror resolve: a pure add/add is what the nightly will merge | |
| # automatically, so reporting it as a conflict here is a false | |
| # alarm. Anything additive_merge.py refuses is still a conflict. | |
| if python3 ../scripts/unsloth/additive_merge.py >/dev/null 2>&1 \ | |
| && [ -z "$(git diff --name-only --diff-filter=U)" ]; then | |
| git -c user.name=preflight -c user.email=preflight@local commit -q --no-edit | |
| echo "ok ${SRC}#${NUM} (additive resolve)" | |
| MERGED=1 | |
| continue | |
| fi | |
| FILES="$(git diff --name-only --diff-filter=U | sed 's/^/ /')" | |
| # `|| true` is load-bearing, not tidying. GitHub runs a `run:` block | |
| # under `bash -e` whatever this script's own `set` line says, `head` | |
| # closes the pipe after 20 lines, and pipefail then makes the whole | |
| # assignment fail. So on 09-03 the step died right here, on the first | |
| # real conflict, with `grep: write error: Broken pipe` and no alert: | |
| # the one path this job exists to report was the one it could not | |
| # survive. It only fires when the conflict diff is bigger than the | |
| # 64 KiB pipe buffer, since a smaller one is written before `head` | |
| # ever closes it, which is why most conflicts got reported fine. | |
| HUNKS="$(git diff --diff-filter=U -U0 2>/dev/null | grep -E '^\+|^-' | grep -vE '^(\+\+\+|---)' | head -20 || true)" | |
| git merge --abort 2>/dev/null | |
| PROBLEMS="${PROBLEMS}- \`${SRC}#${NUM}\` (\`${SHA:0:10}\`) does not merge onto \`${BASE}\` + the pins before it.\n\n Conflicting files:\n\n\`\`\`\n${FILES}\n\`\`\`\n\n <details><summary>conflict hunks</summary>\n\n\`\`\`diff\n${HUNKS}\n\`\`\`\n\n </details>\n" | |
| # Stop here, like resolve does. Probing later pins against a tree | |
| # missing this one reports conflicts that are consequences of it. | |
| PROBLEMS="${PROBLEMS}\nLater pins were not probed; fix this one first.\n" | |
| break | |
| done < <(jq -r '.prs[] | if type == "string" then {url: ., required: true} else . end | |
| | "\(.url)\t\(if .required == null then true else .required end)"' \ | |
| ../scripts/unsloth/pr-set.json | tr '\t' ' ') | |
| # Only when a pin actually merged. With no pins, or only optional closed ones, this tree is pristine upstream, and a finding there is not a pin problem to alert on. The nightly gates the same check on MERGED_PINS. | |
| if [ -z "$PROBLEMS" ] && [ -n "$MERGED" ]; then | |
| # Merging cleanly is not the same as merging correctly. Two mistakes | |
| # made on 08-27 compiled fine and would have shipped: a tensor-map key | |
| # defined twice, which Python resolves silently by keeping the last, | |
| # and an arch arm made unreachable by the same arch appearing in an | |
| # earlier fallthrough condition. Both are checked here, on the tree the | |
| # pins just produced, because this is the first point it exists. | |
| if ! python3 ../scripts/unsloth/merge_checks.py --root . ; then | |
| PROBLEMS="${PROBLEMS}- the merged tree builds, but \`scripts/unsloth/merge_checks.py\` found a resolution that is silently wrong. See the run log for file and line.\n" | |
| fi | |
| # The other half of that question. merge_checks.py asks whether the | |
| # tree contains something wrong; this asks whether it still contains | |
| # what each pin carries. A pin that has rotted into a no-op, or an | |
| # arch registration a resolution quietly dropped, is invisible to | |
| # every other check here and to the compiler. | |
| if ! python3 ../scripts/unsloth/pin_contract.py --root . --base "$BASE" \ | |
| --pr-set ../scripts/unsloth/pr-set.json --report "${RUNNER_TEMP}/pin_contract.json" ; then | |
| PROBLEMS="${PROBLEMS}- the merged tree is missing code a pin carries. See the run log for the pin and file.\n" | |
| fi | |
| NOTES="$(jq -r '.notices[]?' "${RUNNER_TEMP}/pin_contract.json" 2>/dev/null || true)" | |
| if [ -n "$NOTES" ]; then | |
| PROBLEMS="${PROBLEMS}- pins upstream has taken over, safe to delete from \`pr-set.json\`:\n\n\`\`\`\n${NOTES}\n\`\`\`\n" | |
| fi | |
| # A clean merge is not a compiling tree. On 09-03 ggml-org#27754 | |
| # merged with no conflicts at all and did not compile: upstream had | |
| # added a parameter to build_attn_mha and the pin's new | |
| # build_attn_sparse still called the old signature. Nothing above | |
| # can see that. CPU only and the `llama` target only, which is where | |
| # that translation unit lives; 59s cold at -j4 with no ccache. | |
| GATE_OK=1 | |
| if ! cmake -B "${RUNNER_TEMP}/gate" -DCMAKE_BUILD_TYPE=Release \ | |
| -DGGML_CUDA=OFF -DLLAMA_BUILD_TESTS=ON -DLLAMA_BUILD_SERVER=OFF \ | |
| -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_CURL=OFF > /dev/null \ | |
| || ! cmake --build "${RUNNER_TEMP}/gate" -j "$(nproc)" \ | |
| --target llama test-llama-archs test-backend-ops test-mtmd-impl ; then | |
| GATE_OK= | |
| PROBLEMS="${PROBLEMS}- the pins merge cleanly and the merged tree does not compile. See the run log for the file and line; this is the failure that only shows up in the CUDA leg once the nightly has fanned out.\n" | |
| fi | |
| # The last question, and the only one that needs a binary: does each | |
| # feature we ship still work. Everything above is about the source. | |
| # CPU only, because no runner in this pipeline has a GPU -- see the | |
| # note in feature_matrix.py about what that does and does not prove. | |
| if [ -n "$GATE_OK" ]; then | |
| if ! python3 ../scripts/unsloth/feature_matrix.py \ | |
| --build-dir "${RUNNER_TEMP}/gate" \ | |
| --feature-checks ../scripts/unsloth/feature-checks.json \ | |
| --report "${RUNNER_TEMP}/feature_matrix.json" ; then | |
| PROBLEMS="${PROBLEMS}- the merged tree compiles and a feature we ship could not be shown to work. See the run log for which feature and which probe.\n" | |
| fi | |
| fi | |
| fi | |
| if [ -z "$PROBLEMS" ]; then | |
| echo "all pins merge cleanly onto ${BASE}" | |
| echo "status=success" >> "$GITHUB_OUTPUT" | |
| echo "details=" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| echo "status=failure" >> "$GITHUB_OUTPUT" | |
| { | |
| echo 'details<<ALERT_EOF' | |
| echo "Tonight's nightly will fail on base \`${BASE}\` unless \`scripts/unsloth/pr-set.json\` is repinned." | |
| echo | |
| printf '%b' "$PROBLEMS" | |
| echo | |
| echo "For a pin on a branch we control, merge \`${BASE}\` into it and repin. For a third-party PR, wait for the author to merge master or drop the pin." | |
| echo 'ALERT_EOF' | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Alert | |
| if: ${{ steps.p.outputs.status != 'skip' }} | |
| uses: ./.github/actions/prebuilt-alert | |
| with: | |
| status: ${{ steps.p.outputs.status }} | |
| key: llama-pin-preflight | |
| title: 'Pinned PRs no longer merge onto the current base tag' | |
| details: ${{ steps.p.outputs.details }} | |
| token: ${{ github.token }} | |
| # The probe reports through its output, so without this the run still | |
| # ends green and unsloth-repin-bot.yml, which waits for a failed | |
| # workflow_run, never fires. On 08-05 the Inkling pin stopped merging | |
| # onto b10280, the alert said so, the run said success, and the bot | |
| # skipped. Fails last so the alert is always posted first. | |
| - name: Fail the run when a pin does not merge | |
| if: ${{ steps.p.outputs.status == 'failure' }} | |
| run: | | |
| echo "::error::pins do not merge onto the current base tag; see the alert above" | |
| exit 1 |