Skip to content

Commit 1d3b40d

Browse files
authored
Add FastLanes Linux mixed-schema retry workflow (#168)
* Add FastLanes Linux mixed-schema retry workflow * Harden FastLanes retry evidence collection
1 parent 7b918d8 commit 1d3b40d

2 files changed

Lines changed: 299 additions & 0 deletions

File tree

Lines changed: 264 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,264 @@
1+
name: FastLanes mixed-schema retry
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
numeric_rows:
7+
description: Numeric control row count
8+
required: true
9+
default: "1000000"
10+
type: string
11+
timeout_seconds:
12+
description: Timeout per generated case
13+
required: true
14+
default: "120"
15+
type: string
16+
17+
permissions:
18+
contents: read
19+
20+
env:
21+
FASTLANES_SOURCE_COMMIT: f0edc1020a538f1f8098640fce8347c9ac247a0d
22+
RUN_DIR: runs/fastlanes-linux-${{ github.run_id }}
23+
ARCHIVE: outputs/release/data-format-lab-fastlanes-linux-x86_64-${{ github.run_id }}.tar.zst
24+
25+
jobs:
26+
retry:
27+
name: FastLanes official Python round trip
28+
runs-on: ubuntu-24.04
29+
timeout-minutes: 30
30+
31+
steps:
32+
- name: Check out public commit
33+
id: checkout
34+
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
35+
36+
- name: Initialize evidence directory
37+
if: always()
38+
run: |
39+
set -euo pipefail
40+
mkdir -p "$RUN_DIR"
41+
printf 'platform=linux-x86_64\nrun_id=%s\n' "$GITHUB_RUN_ID" \
42+
> "$RUN_DIR/context.txt"
43+
44+
- name: Install Nix
45+
uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22
46+
47+
- name: Synchronize frozen lab environment
48+
run: nix develop --command uv sync --frozen
49+
50+
- name: Check out pinned FastLanes source
51+
id: source
52+
continue-on-error: true
53+
run: |
54+
set -euo pipefail
55+
mkdir -p native
56+
git clone --filter=blob:none --no-tags \
57+
https://github.com/cwida/FastLanes.git native/fastlanes
58+
git -C native/fastlanes checkout --detach "$FASTLANES_SOURCE_COMMIT"
59+
git -C native/fastlanes submodule update --init --recursive
60+
git -C native/fastlanes rev-parse HEAD > "$RUN_DIR/source-commit.txt"
61+
62+
- name: Build official Python extension
63+
id: build
64+
if: always()
65+
continue-on-error: true
66+
run: |
67+
set -euo pipefail
68+
mkdir -p "$RUN_DIR"
69+
set +e
70+
(
71+
set -euo pipefail
72+
test -f native/fastlanes/CMakeLists.txt
73+
nix develop --command python -m venv native/fastlanes/.venv
74+
nix develop --command native/fastlanes/.venv/bin/python -m pip install \
75+
--disable-pip-version-check --no-cache-dir \
76+
scikit-build-core==0.11.6 \
77+
cmake==4.4.0 \
78+
ninja==1.13.0 \
79+
pybind11==2.12.1 \
80+
setuptools-scm==8.3.1
81+
CMAKE_BUILD_PARALLEL_LEVEL=4 \
82+
nix develop --command native/fastlanes/.venv/bin/python -m pip install \
83+
--no-build-isolation -e native/fastlanes
84+
native/fastlanes/.venv/bin/python -c \
85+
'import pyfastlanes; print(pyfastlanes.get_version())'
86+
) > "$RUN_DIR/build.log" 2>&1
87+
status=$?
88+
set -e
89+
printf '%s\n' "$status" > "$RUN_DIR/build-status.txt"
90+
exit "$status"
91+
92+
- name: Record reproducibility metadata
93+
if: always()
94+
run: |
95+
set -euo pipefail
96+
{
97+
printf 'source_commit=%s\n' "$FASTLANES_SOURCE_COMMIT"
98+
printf 'requested_numeric_rows=%s\n' '${{ inputs.numeric_rows }}'
99+
printf 'requested_timeout_seconds=%s\n' '${{ inputs.timeout_seconds }}'
100+
printf 'python='; nix develop --command python --version
101+
if test -x native/fastlanes/.venv/bin/python; then
102+
printf 'fastlanes_python='; native/fastlanes/.venv/bin/python --version
103+
fi
104+
printf 'clang='; nix develop --command clang --version
105+
printf 'cmake='; nix develop --command cmake --version
106+
printf 'ninja='; nix develop --command ninja --version
107+
printf 'uname='; uname -a
108+
} > "$RUN_DIR/environment.txt" 2>&1 || true
109+
110+
- name: Run fresh-process FastLanes claims
111+
id: measure
112+
if: steps.build.outcome == 'success'
113+
continue-on-error: true
114+
env:
115+
NUMERIC_ROWS: ${{ inputs.numeric_rows }}
116+
CASE_TIMEOUT_SECONDS: ${{ inputs.timeout_seconds }}
117+
run: |
118+
set -euo pipefail
119+
case "$CASE_TIMEOUT_SECONDS" in
120+
''|*[!0-9]*)
121+
echo "timeout_seconds must be an integer" >&2
122+
exit 2
123+
;;
124+
esac
125+
if test "$CASE_TIMEOUT_SECONDS" -lt 1 || \
126+
test "$CASE_TIMEOUT_SECONDS" -gt 120; then
127+
echo "timeout_seconds must be between 1 and 120" >&2
128+
exit 2
129+
fi
130+
set +e
131+
PYTHONPATH="$GITHUB_WORKSPACE/src" \
132+
native/fastlanes/.venv/bin/python - "$RUN_DIR/results.json" <<'PY'
133+
import json
134+
import os
135+
import sys
136+
from pathlib import Path
137+
138+
from format_bench.claims.fastlanes import run_fastlanes_claim
139+
140+
output = Path(sys.argv[1])
141+
evidence = run_fastlanes_claim(
142+
output.parent / "claims" / "fastlanes-official",
143+
numeric_rows=int(os.environ["NUMERIC_ROWS"]),
144+
timeout_seconds=float(os.environ["CASE_TIMEOUT_SECONDS"]),
145+
)
146+
payload = {
147+
"schema_version": "1",
148+
"contract_version": "1",
149+
"lane": "claims",
150+
"platform": "linux-x86_64",
151+
"source_commit": "f0edc1020a538f1f8098640fce8347c9ac247a0d",
152+
"evidence": evidence,
153+
}
154+
output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
155+
PY
156+
status=$?
157+
set -e
158+
printf '%s\n' "$status" > "$RUN_DIR/measure-status.txt"
159+
exit "$status"
160+
161+
- name: Preserve unavailable or failed evidence
162+
if: always()
163+
run: |
164+
set -euo pipefail
165+
if test -s "$RUN_DIR/results.json"; then
166+
exit 0
167+
fi
168+
nix develop --command python - "$RUN_DIR/results.json" \
169+
'${{ steps.source.outcome }}' '${{ steps.build.outcome }}' \
170+
'${{ steps.measure.outcome }}' <<'PY'
171+
import json
172+
import sys
173+
from pathlib import Path
174+
175+
output = Path(sys.argv[1])
176+
source, build, measure = sys.argv[2:]
177+
available = source == "success" and build == "success"
178+
output.write_text(
179+
json.dumps(
180+
{
181+
"schema_version": "1",
182+
"contract_version": "1",
183+
"lane": "claims",
184+
"platform": "linux-x86_64",
185+
"source_commit": "f0edc1020a538f1f8098640fce8347c9ac247a0d",
186+
"status": "UNSUPPORTED" if not available else "FAILED",
187+
"failure_reason": (
188+
f"source={source}; build={build}; measure={measure}"
189+
),
190+
},
191+
indent=2,
192+
sort_keys=True,
193+
)
194+
+ "\n"
195+
)
196+
PY
197+
198+
- name: Write concise evidence report
199+
if: always()
200+
run: |
201+
set -euo pipefail
202+
nix develop --command python - "$RUN_DIR/results.json" "$RUN_DIR/report.md" <<'PY'
203+
import json
204+
import sys
205+
from pathlib import Path
206+
207+
results = json.loads(Path(sys.argv[1]).read_text())
208+
lines = [
209+
"# FastLanes Linux x86_64 retry",
210+
"",
211+
f"- Source commit: `{results['source_commit']}`",
212+
f"- Platform: `{results['platform']}`",
213+
]
214+
evidence = results.get("evidence")
215+
if evidence:
216+
lines.extend(["", f"- Status: `{evidence.get('status', 'UNKNOWN')}`"])
217+
lines.append(f"- Summary: {evidence.get('summary', 'not available')}")
218+
else:
219+
lines.extend(["", f"- Status: `{results.get('status', 'UNKNOWN')}`"])
220+
lines.append(f"- Failure: {results.get('failure_reason', 'not available')}")
221+
Path(sys.argv[2]).write_text("\n".join(lines) + "\n")
222+
PY
223+
224+
- name: Package evidence
225+
if: always()
226+
run: |
227+
set -euo pipefail
228+
mkdir -p "$(dirname "$ARCHIVE")"
229+
tar --zstd --sort=name --mtime='UTC 2026-07-17' \
230+
-cf "$ARCHIVE" -C "$RUN_DIR" .
231+
sha256sum "$ARCHIVE" > "$ARCHIVE.sha256"
232+
233+
- name: Upload FastLanes evidence
234+
if: always()
235+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
236+
with:
237+
name: data-format-lab-fastlanes-linux-x86_64-${{ github.run_id }}
238+
path: |
239+
${{ env.RUN_DIR }}
240+
${{ env.ARCHIVE }}
241+
${{ env.ARCHIVE }}.sha256
242+
retention-days: 14
243+
if-no-files-found: error
244+
245+
- name: Preserve retry failure status
246+
if: always()
247+
run: |
248+
set -euo pipefail
249+
test '${{ steps.checkout.outcome }}' = success
250+
test '${{ steps.source.outcome }}' = success
251+
test '${{ steps.build.outcome }}' = success
252+
nix develop --command python - "$RUN_DIR/results.json" <<'PY'
253+
import json
254+
import sys
255+
from pathlib import Path
256+
257+
results = json.loads(Path(sys.argv[1]).read_text())
258+
evidence = results.get("evidence", {})
259+
if evidence.get("status") != "MEASURED":
260+
raise SystemExit(f"FastLanes claim status: {evidence.get('status')}")
261+
mixed = evidence.get("mixed", {})
262+
if mixed.get("outcome") != "ROUNDTRIP_EQUAL":
263+
raise SystemExit(f"mixed 13-column outcome: {mixed.get('outcome')}")
264+
PY

‎tests/test_fastlanes_workflow.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
from pathlib import Path
2+
3+
4+
WORKFLOW = Path(__file__).parents[1] / ".github/workflows/benchmark-fastlanes.yml"
5+
6+
7+
def test_fastlanes_retry_workflow_is_pinned_and_linux_only() -> None:
8+
workflow = WORKFLOW.read_text(encoding="utf-8")
9+
10+
assert "runs-on: ubuntu-24.04" in workflow
11+
assert "f0edc1020a538f1f8098640fce8347c9ac247a0d" in workflow
12+
assert "uv sync --frozen" in workflow
13+
assert "native/fastlanes/.venv/bin/python -m pip install" in workflow
14+
assert "run_fastlanes_claim" in workflow
15+
assert "mixed 13-column outcome" in workflow
16+
assert "timeout-minutes: 30" in workflow
17+
assert 'default: "120"' in workflow
18+
assert "timeout_seconds must be between 1 and 120" in workflow
19+
assert "tar --zstd" in workflow
20+
assert "retention-days: 14" in workflow
21+
22+
assert workflow.index("name: Check out public commit") < workflow.index(
23+
"name: Initialize evidence directory"
24+
)
25+
26+
27+
def test_fastlanes_retry_workflow_preserves_build_and_measurement_logs() -> None:
28+
workflow = WORKFLOW.read_text(encoding="utf-8")
29+
30+
assert '"$RUN_DIR/build.log"' in workflow
31+
assert '"$RUN_DIR/build-status.txt"' in workflow
32+
assert '"$RUN_DIR/measure-status.txt"' in workflow
33+
assert "steps.measure.outcome" in workflow
34+
assert '"status": "UNSUPPORTED" if not available else "FAILED"' in workflow
35+
assert "if: always()" in workflow

0 commit comments

Comments
 (0)