Repository navigation
199 lines (181 loc) · 7.89 KB
/
Copy pathcoreai.yml
File metadata and controls
199 lines (181 loc) · 7.89 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
name: Core AI
on:
push:
branches:
- main
- release/*
tags:
- ciflow/coreai/*
pull_request:
paths:
- .github/workflows/coreai.yml
- .clang-format
- CMakeLists.txt
- backends/apple/coreai/**
- backends/test/**
- extension/module/**
- extension/pybindings/**
- extension/tensor/**
- setup.py
- runtime/**
- tools/cmake/**
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
cancel-in-progress: true
permissions:
contents: read
jobs:
run-decision:
name: CI run decision
uses: ./.github/workflows/_ci-run-decision.yml
test-coreai:
needs: run-decision
if: |
github.event_name == 'pull_request' ||
needs.run-decision.outputs.is-full-run == 'true'
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
with:
default-packages: ""
job-name: test-coreai
# GitHub-hosted macOS 27 with Xcode 27 (public preview).
runner: xcode-27
# coreai-opt supports Python < 3.14.
python-version: "3.12"
submodules: recursive
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
timeout: 90
script: |
set -eux
echo "::group::Toolchain"
sw_vers
xcodebuild -version
xcrun --show-sdk-version
echo "::endgroup::"
echo "::group::Install the Metal Toolchain"
# coreai-build ships in the Metal Toolchain; without it the AOT compile
# tests skip.
xcodebuild -downloadComponent MetalToolchain
xcrun --find coreai-build
echo "::endgroup::"
echo "::group::Install ExecuTorch and Core AI requirements"
${CONDA_RUN} python install_executorch.py --editable > /dev/null
${CONDA_RUN} pip install -r backends/apple/coreai/requirements.txt "cmake>=3.31,<4" ninja clang-format==18.1.3 pytest-xdist
${CONDA_RUN} pip list
echo "::endgroup::"
echo "::group::Check Core AI formatting"
${CONDA_RUN} python - <<'PY'
from pathlib import Path
import subprocess
import yaml
root = yaml.safe_load(Path(".clang-format").read_text())
coreai = yaml.safe_load(Path("backends/apple/coreai/.clang-format").read_text())
root.pop("Language")
assert coreai == root, "Sync the Core AI clang-format configuration with the root style (without Language)"
sources = sorted(Path("backends/apple/coreai").rglob("*.mm"))
subprocess.run(["clang-format", "--dry-run", "--Werror", *map(str, sources)], check=True)
PY
echo "::endgroup::"
echo "::group::Run Core AI export tests"
${CONDA_RUN} python -m pytest backends/apple/coreai -n auto
echo "::endgroup::"
echo "::group::Build and run Core AI runtime tests"
${CONDA_RUN} cmake -S . -B cmake-out-coreai -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_OSX_DEPLOYMENT_TARGET=27.0 \
-DEXECUTORCH_BUILD_COREAI=ON \
-DEXECUTORCH_BUILD_TESTS=ON \
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
-DEXECUTORCH_BUILD_PORTABLE_OPS=ON
${CONDA_RUN} cmake --build cmake-out-coreai --target backends/apple/coreai/all -j$(sysctl -n hw.ncpu)
COREAI_STRESS_THREADS=50 COREAI_STRESS_ITERATIONS=100 \
${CONDA_RUN} ctest --test-dir cmake-out-coreai/backends/apple/coreai --output-on-failure --no-tests=error
echo "::endgroup::"
backend-tester:
needs: run-decision
if: |
github.event_name == 'pull_request' ||
needs.run-decision.outputs.is-full-run == 'true'
strategy:
fail-fast: false
max-parallel: 8
matrix:
suite: [models, operators]
flow: [coreai_fp32, coreai_fp16, coreai_w8a8, coreai_w8]
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
with:
default-packages: ""
job-name: test-${{ matrix.flow }}-${{ matrix.suite }}
runner: xcode-27
python-version: "3.12"
submodules: recursive
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
timeout: 120
script: |
set -euxo pipefail
echo "::group::Install the Core AI toolchain and Python runtime"
xcodebuild -version
xcodebuild -downloadComponent MetalToolchain
xcrun --find coreai-build
export CMAKE_GENERATOR=Ninja
export CMAKE_ARGS="-DEXECUTORCH_BUILD_COREAI=ON -DEXECUTORCH_BUILD_COREML=OFF -DEXECUTORCH_BUILD_MLX=OFF -DCMAKE_OSX_DEPLOYMENT_TARGET=27.0"
${CONDA_RUN} pip install "cmake>=3.31,<4" ninja
${CONDA_RUN} python install_executorch.py --editable > /dev/null
${CONDA_RUN} pip install -r backends/apple/coreai/requirements.txt pytest-xdist pytest-json-report pytest-timeout
${CONDA_RUN} pip list
${CONDA_RUN} python - <<'PY'
from executorch.extension.pybindings.portable_lib import _get_registered_backend_names, _is_available
assert "CoreAIBackend" in _get_registered_backend_names()
assert _is_available("CoreAIBackend")
PY
echo "::endgroup::"
echo "::group::Run shared backend suite (${{ matrix.flow }} / ${{ matrix.suite }})"
export GOLDEN_ARTIFACTS_DIR="${RUNNER_TEMP}/coreai-${{ matrix.flow }}-${{ matrix.suite }}"
# Allowed failures for gating suites; unlisted suites are informational.
case "${{ matrix.flow }}/${{ matrix.suite }}" in
coreai_fp32/operators | coreai_fp16/operators | coreai_w8/operators | \
coreai_fp32/models | coreai_fp16/models | coreai_w8/models) MAX_FAILED=0 ;;
# Known failures; lower this as they're fixed.
coreai_w8a8/operators) MAX_FAILED=23 ;;
# 7 fail locally (accuracy). On the xcode-27 runners, 19 w8a8
# dynamic_shapes models also fail execution with Core AI
# inferenceFailed(-1) (3 overlap the local 7), while their static
# variants and fp32 dynamic_shapes pass. TODO: report to Apple and
# lower back to 7.
coreai_w8a8/models) MAX_FAILED=23 ;;
*) MAX_FAILED="" ;;
esac
FAIL_FAST=""
if [[ "${MAX_FAILED}" == "0" ]]; then FAIL_FAST="-x"; fi
status=0
${CONDA_RUN} python -m pytest -c /dev/null backends/test/suite/${{ matrix.suite }}/ \
-m flow_${{ matrix.flow }} -n auto ${FAIL_FAST} \
--json-report --json-report-file=coreai-results.json \
--junitxml=coreai-results.xml 2>&1 | tee coreai-pytest.log || status=$?
echo "::endgroup::"
if [[ -z "${MAX_FAILED}" ]]; then
if [[ ${status} -ne 0 ]]; then
echo "::warning::${{ matrix.flow }} / ${{ matrix.suite }} is not gating; pytest exited with ${status}"
fi
exit 0
fi
# 1 means tests failed (checked against MAX_FAILED below); anything else
# is a pytest crash, interrupt or usage error.
if [[ ${status} -ne 0 && ${status} -ne 1 ]]; then exit ${status}; fi
echo "::group::Check ${{ matrix.flow }} / ${{ matrix.suite }} results"
export MAX_FAILED
${CONDA_RUN} python - <<'PY'
import json
import os
summary = json.load(open("coreai-results.json"))["summary"]
max_failed = int(os.environ["MAX_FAILED"])
print(summary)
failed = summary.get("failed", 0)
assert failed <= max_failed, f"{failed} tests failed; at most {max_failed} allowed"
assert summary.get("error", 0) == 0, "tests errored"
assert summary.get("passed", 0) > 0, "no tests passed"
PY
echo "::endgroup::"