Skip to content

Commit 8a1fadc

Browse files
aryamantodkararyamantodkar
authored
Stage 7: certify v0.1.0 platform compatibility (#86)
* ci: add compatibility certification matrix * ci: run platform certification on release changes * fix: validate workflow context boundaries * ci: isolate model downloads from certification * ci: use larger runner for MPS certification * docs: retain local MPS compatibility evidence * docs: publish Stage 7 compatibility evidence * docs: clarify platform certification hosts * ci: certify publishable manylinux wheels * fix: use portable Cargo dependency syntax * docs: retain manylinux certification evidence * style: format platform certification tests
1 parent a803cbe commit 8a1fadc

17 files changed

Lines changed: 801 additions & 18 deletions

‎.github/workflows/ci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ jobs:
4949
- name: Verify frozen release boundaries
5050
run: uv run python scripts/check_frozen_boundaries.py
5151

52+
- name: Validate GitHub Actions workflows
53+
run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.7
54+
5255
rust:
5356
name: Rust
5457
runs-on: ubuntu-22.04

‎.github/workflows/platform-validation.yml‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
name: Platform validation
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
branches:
7+
- main
8+
paths:
9+
- ".github/workflows/platform-validation.yml"
10+
- "pyproject.toml"
11+
- "uv.lock"
12+
- "rust/**"
13+
- "src/**"
14+
- "scripts/certify_*.py"
15+
- "eval/voice_agent_itn/**"
16+
17+
permissions:
18+
contents: read
19+
20+
concurrency:
21+
group: platform-validation-${{ github.ref }}
22+
cancel-in-progress: false
23+
24+
jobs:
25+
package:
26+
name: ${{ matrix.name }} / Python ${{ matrix.python }}
27+
runs-on: ${{ matrix.runner }}
28+
strategy:
29+
fail-fast: false
30+
matrix:
31+
include:
32+
- { name: macOS arm64, runner: macos-14, os: Darwin, arch: arm64, python: "3.11" }
33+
- { name: macOS arm64, runner: macos-14, os: Darwin, arch: arm64, python: "3.12" }
34+
- { name: macOS arm64, runner: macos-14, os: Darwin, arch: arm64, python: "3.13" }
35+
- { name: Linux x86_64, runner: ubuntu-22.04, os: Linux, arch: x86_64, python: "3.11" }
36+
- { name: Linux x86_64, runner: ubuntu-22.04, os: Linux, arch: x86_64, python: "3.12" }
37+
- { name: Linux x86_64, runner: ubuntu-22.04, os: Linux, arch: x86_64, python: "3.13" }
38+
steps:
39+
- name: Check out repository
40+
uses: actions/checkout@v4
41+
42+
- name: Set up Python ${{ matrix.python }}
43+
uses: actions/setup-python@v5
44+
with:
45+
python-version: ${{ matrix.python }}
46+
47+
- name: Set up uv
48+
uses: astral-sh/setup-uv@v5
49+
with:
50+
enable-cache: true
51+
52+
- name: Build macOS release wheel for the matrix interpreter
53+
if: matrix.os == 'Darwin'
54+
run: uv build --wheel --python ${{ matrix.python }} --out-dir wheelhouse
55+
56+
- name: Build manylinux release wheel for the matrix interpreter
57+
if: matrix.os == 'Linux'
58+
uses: PyO3/maturin-action@86b9d133d34bc1b40018696f782949dac11bd380
59+
with:
60+
command: build
61+
maturin-version: v1.11.5
62+
target: x86_64-unknown-linux-gnu
63+
manylinux: 2_28
64+
args: >-
65+
--release --locked --compatibility pypi
66+
--interpreter python${{ matrix.python }}
67+
--out wheelhouse
68+
69+
- name: Inspect release wheel
70+
shell: bash
71+
run: python scripts/inspect_release_artifact.py wheelhouse/*.whl
72+
73+
- name: Create clean install environment
74+
shell: bash
75+
run: |
76+
set -euo pipefail
77+
uv venv --python ${{ matrix.python }} "$RUNNER_TEMP/certification-venv"
78+
uv pip install --only-binary :all: \
79+
--python "$RUNNER_TEMP/certification-venv/bin/python" wheelhouse/*.whl
80+
81+
- name: Certify installed package and CLI
82+
shell: bash
83+
run: |
84+
"$RUNNER_TEMP/certification-venv/bin/python" -m \
85+
scripts.certify_installed_platform \
86+
--wheel "$GITHUB_WORKSPACE"/wheelhouse/*.whl \
87+
--expected-os "${{ matrix.os }}" \
88+
--expected-arch "${{ matrix.arch }}" \
89+
--expected-python "${{ matrix.python }}" \
90+
--expected-wheel-platform "${{ matrix.os == 'Linux' && 'manylinux_2_28_x86_64' || 'macosx_11_0_arm64' }}" \
91+
--source-commit "$GITHUB_SHA" \
92+
--output "$GITHUB_WORKSPACE/evidence/package-${{ matrix.os }}-${{ matrix.arch }}-py${{ matrix.python }}.json"
93+
94+
- name: Retain package evidence and wheel
95+
uses: actions/upload-artifact@v4
96+
with:
97+
name: package-${{ matrix.os }}-${{ matrix.arch }}-py${{ matrix.python }}
98+
path: |
99+
evidence/*.json
100+
wheelhouse/*.whl
101+
if-no-files-found: error
102+
103+
inference:
104+
name: ${{ matrix.name }} / ${{ matrix.device }} inference
105+
runs-on: ${{ matrix.runner }}
106+
timeout-minutes: 180
107+
strategy:
108+
fail-fast: false
109+
matrix:
110+
include:
111+
- { name: Linux x86_64, runner: ubuntu-22.04, os: Linux, arch: x86_64, device: cpu, unavailable: mps }
112+
steps:
113+
- name: Check out repository
114+
uses: actions/checkout@v4
115+
116+
- name: Set up Python 3.11
117+
uses: actions/setup-python@v5
118+
with:
119+
python-version: "3.11"
120+
121+
- name: Set up uv
122+
uses: astral-sh/setup-uv@v5
123+
with:
124+
enable-cache: true
125+
126+
- name: Build manylinux release wheel
127+
uses: PyO3/maturin-action@86b9d133d34bc1b40018696f782949dac11bd380
128+
with:
129+
command: build
130+
maturin-version: v1.11.5
131+
target: x86_64-unknown-linux-gnu
132+
manylinux: 2_28
133+
args: >-
134+
--release --locked --compatibility pypi
135+
--interpreter python3.11
136+
--out wheelhouse
137+
138+
- name: Clean-install manylinux release wheel
139+
shell: bash
140+
run: |
141+
set -euo pipefail
142+
python scripts/inspect_release_artifact.py wheelhouse/*.whl
143+
uv venv --python 3.11 "$RUNNER_TEMP/inference-venv"
144+
uv pip install --only-binary :all: \
145+
--python "$RUNNER_TEMP/inference-venv/bin/python" wheelhouse/*.whl
146+
147+
- name: Download frozen model with bounded retries
148+
env:
149+
HF_HOME: ${{ runner.temp }}/huggingface
150+
HF_HUB_CACHE: ${{ runner.temp }}/huggingface/hub
151+
HF_HUB_DOWNLOAD_TIMEOUT: "120"
152+
HF_HUB_ETAG_TIMEOUT: "30"
153+
shell: bash
154+
run: |
155+
set -euo pipefail
156+
for attempt in 1 2 3; do
157+
if "$RUNNER_TEMP/inference-venv/bin/python" - <<'PY'
158+
from huggingface_hub import snapshot_download
159+
from premove_itn.contextual import DEFAULT_MODEL_ID, DEFAULT_REVISION
160+
161+
snapshot_download(repo_id=DEFAULT_MODEL_ID, revision=DEFAULT_REVISION)
162+
PY
163+
then
164+
exit 0
165+
fi
166+
if [ "$attempt" -eq 3 ]; then
167+
exit 1
168+
fi
169+
sleep 5
170+
done
171+
172+
- name: Certify frozen prediction equivalence
173+
shell: bash
174+
env:
175+
HF_HOME: ${{ runner.temp }}/huggingface
176+
HF_HUB_CACHE: ${{ runner.temp }}/huggingface/hub
177+
HF_HUB_OFFLINE: "1"
178+
run: |
179+
"$RUNNER_TEMP/inference-venv/bin/python" -m \
180+
scripts.certify_inference_backend \
181+
--device "${{ matrix.device }}" \
182+
--dataset "$GITHUB_WORKSPACE/eval/voice_agent_itn/voice_agent_eval.jsonl" \
183+
--frozen-digest "$GITHUB_WORKSPACE/eval/voice_agent_itn/FROZEN.sha256" \
184+
--reference-records "$GITHUB_WORKSPACE/eval/voice_agent_itn/results/first-evaluation/premove-itn/records.jsonl" \
185+
--wheel "$GITHUB_WORKSPACE"/wheelhouse/*.whl \
186+
--expected-wheel-platform manylinux_2_28_x86_64 \
187+
--source-commit "$GITHUB_SHA" \
188+
--unavailable-device "${{ matrix.unavailable }}" \
189+
--output "$GITHUB_WORKSPACE/evidence/inference-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.device }}.json"
190+
191+
- name: Retain inference evidence
192+
if: always()
193+
uses: actions/upload-artifact@v4
194+
with:
195+
name: inference-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.device }}
196+
path: evidence/*.json
197+
if-no-files-found: warn

‎.github/workflows/release-smoke.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,6 @@ jobs:
1616
if: github.event_name == 'workflow_dispatch'
1717
runs-on: macos-14
1818
timeout-minutes: 90
19-
env:
20-
HF_HOME: ${{ runner.temp }}/huggingface
21-
HF_HUB_CACHE: ${{ runner.temp }}/huggingface/hub
2219
steps:
2320
- name: Check out repository
2421
uses: actions/checkout@v4
@@ -70,12 +67,17 @@ jobs:
7067
7168
- name: Download and validate the frozen model
7269
working-directory: ${{ runner.temp }}
70+
env:
71+
HF_HOME: ${{ runner.temp }}/huggingface
72+
HF_HUB_CACHE: ${{ runner.temp }}/huggingface/hub
7373
run: |
7474
"$RUNNER_TEMP/premove-release-venv/bin/python" "$GITHUB_WORKSPACE/scripts/smoke_installed_release.py" --device auto
7575
7676
- name: Restart and validate from the cached model
7777
working-directory: ${{ runner.temp }}
7878
env:
79+
HF_HOME: ${{ runner.temp }}/huggingface
80+
HF_HUB_CACHE: ${{ runner.temp }}/huggingface/hub
7981
HF_HUB_OFFLINE: "1"
8082
run: |
8183
"$RUNNER_TEMP/premove-release-venv/bin/python" "$GITHUB_WORKSPACE/scripts/smoke_installed_release.py" --device auto

‎README.md‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -224,10 +224,11 @@ users can pass `device="cpu"`, `device="mps"`, or `device="cuda"` to
224224
`PremoveITN.from_pretrained()`. The CLI exposes the same choices through
225225
`--device`.
226226

227-
The current release candidate has been validated end-to-end on macOS Apple
228-
Silicon with Python 3.11 and MPS. CPU, CUDA, other operating systems, and other
229-
Python versions will be validated before the public release. API availability
230-
does not imply that a platform has completed release validation.
227+
Release wheels are validated on macOS 14+ arm64 and `manylinux_2_28` x86_64
228+
for Python 3.11–3.13. Real frozen-model inference is validated on Apple
229+
Silicon MPS and Linux CPU, with exact output equivalence across 1,500 inputs. See the
230+
[platform support matrix](docs/platform-support.md) and [Stage 7 evidence](docs/platform-evidence/README.md).
231+
API availability does not imply support for an unlisted platform or device.
231232

232233
## Limitations
233234

@@ -242,8 +243,8 @@ does not imply that a platform has completed release validation.
242243
REFERENCE_ID, and VERSION. See the report for exact per-category results.
243244
- Independent human gold adjudication and broader contamination checks remain
244245
incomplete.
245-
- End-to-end release validation currently covers only the environment stated
246-
above.
246+
- CUDA, Windows, macOS Intel, Linux ARM64, and other accelerators are not
247+
validated v0.1.0 support claims.
247248

248249
## Model and weights
249250

‎docs/model-card.md‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,10 @@ repository. Create one `PremoveITN` instance and reuse it; model initialization
5555
is expensive compared with warm normalization.
5656

5757
`device="auto"` selects CUDA when available, then Apple MPS, then CPU. The
58-
current release candidate has been validated end-to-end only on macOS Apple
59-
Silicon with Python 3.11 and MPS. Other environments require release
60-
validation.
58+
Release wheels are validated on macOS 14+ arm64 and `manylinux_2_28` x86_64
59+
for Python 3.11–3.13. Real frozen-model inference is validated on Apple
60+
Silicon MPS and Linux CPU. Other environments require release validation. The source repository retains the current
61+
[platform support matrix](https://github.com/premove-ai/premove-itn/blob/main/docs/platform-support.md).
6162

6263
## Architecture
6364

@@ -144,8 +145,8 @@ Candidate metadata is generated internally.
144145
REFERENCE_ID, and VERSION.
145146
- Independent human gold adjudication and broader contamination checks remain
146147
incomplete. The backend evaluation itself was blind.
147-
- End-to-end release validation currently covers macOS Apple Silicon, Python
148-
3.11, and MPS only.
148+
- CUDA, Windows, macOS Intel, Linux ARM64, and other accelerators are not
149+
validated v0.1.0 support claims.
149150

150151
## Release identity and provenance
151152

‎docs/platform-evidence/README.md‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Stage 7 certification evidence
2+
3+
This directory records the v0.1.0 compatibility certification.
4+
5+
- All six release-wheel cells passed: macOS 14+ arm64 and Linux x86_64 on
6+
Python 3.11, 3.12, and 3.13. Linux release wheels use the
7+
`manylinux_2_28` platform contract.
8+
- MPS and CPU each produced **1,500/1,500 exact output matches** against the
9+
retained First Evaluation Premove predictions.
10+
- `device="auto"` selected MPS on Apple Silicon and CPU on Linux.
11+
- Explicit unavailable-device checks passed for CUDA on macOS and MPS on the
12+
CPU-only Linux runner.
13+
- The MPS run used the same clean release wheel on an Apple Silicon host. The
14+
standard public macOS runner cannot fit this model in its MPS memory limit,
15+
and the larger macOS runner tier is not available to this repository.
16+
17+
The JSON files contain the environment and native-build details recorded by the
18+
certification scripts. The refreshed package matrix records the source commit
19+
and exact wheel digests. The refreshed Linux inference evidence also freezes
20+
the accepted reference-prediction digest. The earlier local MPS record does not
21+
retroactively claim artifact digests that were not captured during that run.
22+
This is execution-equivalence evidence, not a new model accuracy evaluation and
23+
not a tuning input.

‎docs/platform-evidence/linux-x86_64-cpu-py3.11.json‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"schema_version": 1,
3+
"result": "pass",
4+
"os": "Linux",
5+
"os_release": "6.8.0-1064-azure",
6+
"architecture": "x86_64",
7+
"python": "3.11.16",
8+
"torch": "2.13.0+cu130",
9+
"device": "cpu",
10+
"auto_device": "cpu",
11+
"unavailable_devices": ["mps"],
12+
"package_version": "0.1.0",
13+
"model_repository": "premove-ai/premove-itn",
14+
"model_revision": "80bda5e2e1fe9542aa628597090242df57c1a157",
15+
"dataset_sha256": "782b14d0291e4e176e4ba22d0d756019e906a5ea4c7d267d9703e80b082cbd61",
16+
"reference": "first-evaluation/premove-itn/records.jsonl",
17+
"reference_predictions_sha256": "8048bbebdb4854ff6c3449b80198d3a515b47573a737168a82a5754591473d79",
18+
"wheel": "premove_itn-0.1.0-cp311-cp311-manylinux_2_28_x86_64.whl",
19+
"wheel_sha256": "27b7d0bbfd5b7a2e8f7fd3da9d51fb79645eac299d80917885018520f24fb318",
20+
"source_commit": "53bac6c29b55612f402cfeade1ab5c5da270244e",
21+
"rows": 1500,
22+
"identical_rows": 1500,
23+
"mismatch_count": 0,
24+
"mismatches": [],
25+
"peak_rss_bytes": 4434644992,
26+
"rust_target": "x86_64-unknown-linux-gnu",
27+
"rust_profile": "release",
28+
"rust_debug_assertions": false
29+
}

‎docs/platform-evidence/macos-arm64-mps-py3.11.json‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"schema_version": 1,
3+
"result": "pass",
4+
"os": "Darwin",
5+
"os_release": "24.6.0",
6+
"architecture": "arm64",
7+
"python": "3.11.15",
8+
"torch": "2.13.0",
9+
"device": "mps",
10+
"auto_device": "mps",
11+
"unavailable_devices": ["cuda"],
12+
"package_version": "0.1.0",
13+
"model_repository": "premove-ai/premove-itn",
14+
"model_revision": "80bda5e2e1fe9542aa628597090242df57c1a157",
15+
"dataset_sha256": "782b14d0291e4e176e4ba22d0d756019e906a5ea4c7d267d9703e80b082cbd61",
16+
"reference": "first-evaluation/premove-itn/records.jsonl",
17+
"rows": 1500,
18+
"identical_rows": 1500,
19+
"mismatch_count": 0,
20+
"mismatches": [],
21+
"peak_rss_bytes": 3860430848,
22+
"rust_target": "aarch64-apple-darwin",
23+
"rust_profile": "release",
24+
"rust_debug_assertions": false
25+
}

0 commit comments

Comments
 (0)