Skip to content

Commit d189b45

Browse files
authored
Arm backend: Xfail RIFE VGF tests on old converter (pytorch#21755)
Keep the VGF dependency pins at 0.9.0 and xfail the copied RIFE VGF quant tests when the installed model-converter is below version 0.10.0. Only query model-converter during collection when matching RIFE VGF quant tests are present, so unrelated Arm tests do not pay the probe cost. This avoids the known converter partitioning crash seen in the Nightly RIFE test without pulling in the broader VGF runtime regressions triggered by the 0.10.0 converter package. cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils @Sebastian-Larsson @robell @rascani
1 parent bc2833f commit d189b45

9 files changed

Lines changed: 274 additions & 60 deletions

‎backends/arm/test/conftest.py‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#
33
# This source code is licensed under the BSD-style license found in the
44
# LICENSE file in the root directory of this source tree.
5+
"""Pytest hooks and fixtures for the Arm test suite."""
56

67
import logging
78
import os
@@ -10,10 +11,11 @@
1011
from typing import Any
1112

1213
import pytest
13-
14-
"""
15-
This file contains the pytest hooks, fixtures etc. for the Arm test suite.
16-
"""
14+
from executorch.backends.arm.vgf.model_converter import (
15+
get_model_converter_minimum_version_failure_reason,
16+
get_model_converter_version_text,
17+
MIN_MODEL_CONVERTER_VERSION_FOR_VGF_TESTS,
18+
)
1719

1820

1921
# ==== Pytest hooks ====
@@ -41,8 +43,39 @@ def pytest_report_header(config):
4143
return config._test_seed_label
4244

4345

46+
def _mark_rife_vgf_xfails_for_model_converter_below_minimum_version(
47+
items, reason: str
48+
) -> None:
49+
for item in items:
50+
nodeid = item.nodeid.lower()
51+
if "test_rife.py" not in nodeid or "vgf_quant" not in nodeid:
52+
continue
53+
item.add_marker(pytest.mark.xfail(reason=reason, strict=False))
54+
55+
56+
def _has_rife_vgf_quant_tests(items) -> bool:
57+
return any(
58+
"test_rife.py" in item.nodeid.lower() and "vgf_quant" in item.nodeid.lower()
59+
for item in items
60+
)
61+
62+
4463
def pytest_collection_modifyitems(config, items):
45-
pass
64+
if not _has_rife_vgf_quant_tests(items):
65+
return
66+
67+
version_text = get_model_converter_version_text()
68+
if version_text is None:
69+
return
70+
71+
reason = get_model_converter_minimum_version_failure_reason(
72+
version_text,
73+
MIN_MODEL_CONVERTER_VERSION_FOR_VGF_TESTS,
74+
requirement_name="the copied RIFE VGF quant tests",
75+
)
76+
if reason is None:
77+
return
78+
_mark_rife_vgf_xfails_for_model_converter_below_minimum_version(items, reason)
4679

4780

4881
def pytest_addoption(parser):

‎backends/arm/test/misc/test_vgf_check_env.py‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,13 @@
77

88
import stat
99
from pathlib import Path
10+
from typing import Any
1011

1112
import executorch.backends.arm.vgf.check_env as check_env
1213
import executorch.backends.arm.vgf.model_converter as model_converter
1314

1415
import pytest
16+
from executorch.backends.arm.test import conftest as arm_conftest
1517
from executorch.backends.arm.vgf import backend as vgf_backend
1618
from executorch.backends.arm.vgf.compile_spec import VgfCompileSpec
1719

@@ -151,6 +153,137 @@ def test_model_converter_check_reports_version(monkeypatch, tmp_path):
151153
assert "0.9.0" in result.detail
152154

153155

156+
def test_get_model_converter_version_text(monkeypatch, tmp_path):
157+
converter = _make_executable(
158+
tmp_path / "model-converter",
159+
"#!/usr/bin/env python3\n"
160+
"import sys\n"
161+
"if '--version' in sys.argv:\n"
162+
" print('model-converter d8c1b8e')\n"
163+
" raise SystemExit(0)\n"
164+
"raise SystemExit(1)\n",
165+
)
166+
monkeypatch.setattr(
167+
model_converter, "find_model_converter_binary", lambda: str(converter)
168+
)
169+
170+
assert model_converter.get_model_converter_version_text() == (
171+
"model-converter d8c1b8e"
172+
)
173+
174+
175+
def test_parse_model_converter_version_uses_known_build_alias(monkeypatch, tmp_path):
176+
converter = _make_executable(
177+
tmp_path / "model-converter",
178+
"#!/usr/bin/env python3\n"
179+
"import sys\n"
180+
"if '--version' in sys.argv:\n"
181+
" print('model-converter d8c1b8e')\n"
182+
" raise SystemExit(0)\n"
183+
"raise SystemExit(1)\n",
184+
)
185+
monkeypatch.setattr(
186+
model_converter, "find_model_converter_binary", lambda: str(converter)
187+
)
188+
189+
version_text = model_converter.get_model_converter_version_text()
190+
191+
assert version_text is not None
192+
assert model_converter.parse_model_converter_version(version_text) == (
193+
model_converter.Version("0.9.0")
194+
)
195+
196+
197+
def test_below_minimum_model_converter_reason(monkeypatch, tmp_path):
198+
converter = _make_executable(
199+
tmp_path / "model-converter",
200+
"#!/usr/bin/env python3\n"
201+
"import sys\n"
202+
"if '--version' in sys.argv:\n"
203+
" print('model-converter d8c1b8e')\n"
204+
" raise SystemExit(0)\n"
205+
"raise SystemExit(1)\n",
206+
)
207+
monkeypatch.setattr(
208+
model_converter, "find_model_converter_binary", lambda: str(converter)
209+
)
210+
211+
version_text = model_converter.get_model_converter_version_text()
212+
213+
assert version_text is not None
214+
assert model_converter.get_model_converter_minimum_version_failure_reason(
215+
version_text,
216+
model_converter.MIN_MODEL_CONVERTER_VERSION_FOR_VGF_TESTS,
217+
requirement_name="the copied RIFE VGF quant tests",
218+
) == (
219+
"model-converter d8c1b8e is below the minimum supported version "
220+
"0.10.0 required for the copied RIFE VGF quant tests"
221+
)
222+
223+
224+
def test_mark_rife_vgf_xfails_marks_only_rife_vgf_quant():
225+
class DummyItem:
226+
def __init__(self, nodeid: str):
227+
self.nodeid = nodeid
228+
self.markers: list[Any] = []
229+
230+
def add_marker(self, marker: Any) -> None:
231+
self.markers.append(marker)
232+
233+
matching = DummyItem("backends/arm/test/models/test_RIFE.py::test_vgf_quant")
234+
non_matching = DummyItem("backends/arm/test/models/test_RIFE.py::test_tosa")
235+
236+
arm_conftest._mark_rife_vgf_xfails_for_model_converter_below_minimum_version(
237+
[matching, non_matching], "below minimum version"
238+
)
239+
240+
assert len(matching.markers) == 1
241+
assert matching.markers[0].name == "xfail"
242+
assert not non_matching.markers
243+
244+
245+
def test_has_rife_vgf_quant_tests_matches_only_rife_vgf_quant():
246+
class DummyItem:
247+
__slots__ = ("nodeid",)
248+
249+
def __init__(self, nodeid: str):
250+
self.nodeid = nodeid
251+
252+
assert arm_conftest._has_rife_vgf_quant_tests(
253+
[
254+
DummyItem("backends/arm/test/models/test_RIFE.py::test_tosa"),
255+
DummyItem("backends/arm/test/models/test_RIFE.py::test_vgf_quant"),
256+
]
257+
)
258+
assert not arm_conftest._has_rife_vgf_quant_tests(
259+
[
260+
DummyItem("backends/arm/test/models/test_RIFE.py::test_tosa"),
261+
DummyItem("backends/arm/test/ops/test_add.py::test_basic"),
262+
]
263+
)
264+
265+
266+
def test_collection_hook_skips_converter_probe_when_no_rife_vgf_quant(monkeypatch):
267+
class DummyItem:
268+
__slots__ = ("nodeid",)
269+
270+
def __init__(self, nodeid: str):
271+
self.nodeid = nodeid
272+
273+
def fail_probe():
274+
raise AssertionError("converter probe should not run")
275+
276+
monkeypatch.setattr(arm_conftest, "get_model_converter_version_text", fail_probe)
277+
278+
arm_conftest.pytest_collection_modifyitems(
279+
None,
280+
[
281+
DummyItem("backends/arm/test/models/test_RIFE.py::test_tosa"),
282+
DummyItem("backends/arm/test/ops/test_add.py::test_basic"),
283+
],
284+
)
285+
286+
154287
def test_model_converter_lib_dir_fails_when_invalid(monkeypatch, tmp_path):
155288
missing = tmp_path / "missing"
156289
monkeypatch.setenv("MODEL_CONVERTER_LIB_DIR", str(missing))

‎backends/arm/test/runtime/_vgf_runtime_test_utils.py‎

Lines changed: 24 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,10 @@
4242
)
4343
from executorch.backends.arm.vgf import VgfCompileSpec, VgfPartitioner
4444
from executorch.backends.arm.vgf.model_converter import (
45-
find_model_converter_binary,
46-
model_converter_env,
45+
get_model_converter_minimum_version_failure_reason,
46+
get_model_converter_version_text,
47+
MIN_MODEL_CONVERTER_VERSION_FOR_VGF_TESTS,
48+
parse_model_converter_version,
4749
)
4850
from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower
4951
from executorch.exir.pass_base import ExportPass
@@ -67,60 +69,37 @@ def ensure_glslc() -> None:
6769

6870

6971
@functools.lru_cache(maxsize=1)
70-
def _model_converter_is_legacy_release() -> tuple[bool, str]:
71-
model_converter = find_model_converter_binary()
72-
if model_converter is None:
72+
def _model_converter_supports_vgf_tests() -> tuple[bool, str]:
73+
version_text = get_model_converter_version_text()
74+
if version_text is None:
7375
warnings.warn(
7476
"Could not find model-converter while evaluating the VGF runtime "
75-
"legacy-version xfail gate; assuming a newer/custom build.",
77+
"minimum-version xfail gate; assuming a newer/custom build.",
7678
stacklevel=2,
7779
)
78-
return False, ""
80+
return True, ""
7981

80-
try:
81-
result = subprocess.run( # nosec B603 - trusted local tool
82-
[model_converter, "--version"],
83-
check=True,
84-
capture_output=True,
85-
text=True,
86-
env=model_converter_env(),
87-
)
88-
except Exception as exc:
89-
warnings.warn(
90-
"Failed to query model-converter --version while evaluating the VGF "
91-
f"runtime legacy-version xfail gate ({exc}); assuming a newer/custom "
92-
"build.",
93-
stacklevel=2,
94-
)
95-
return False, ""
82+
reason = get_model_converter_minimum_version_failure_reason(
83+
version_text,
84+
MIN_MODEL_CONVERTER_VERSION_FOR_VGF_TESTS,
85+
requirement_name="these VGF runtime tests",
86+
)
87+
if reason is not None:
88+
return False, reason
9689

97-
version_text = (result.stdout or result.stderr).strip()
98-
if not version_text:
90+
if parse_model_converter_version(version_text) is None:
9991
warnings.warn(
100-
"model-converter --version returned no output while evaluating the VGF "
101-
"runtime legacy-version xfail gate; assuming a newer/custom build.",
92+
"Could not map model-converter version output to a comparable "
93+
f"release while evaluating the VGF runtime minimum-version xfail "
94+
f"gate; detected {version_text!r}. Assuming a newer/custom build.",
10295
stacklevel=2,
10396
)
104-
return False, ""
105-
106-
if "d8c1b8e" in version_text:
107-
return (
108-
True,
109-
"released model-converter build d8c1b8e predates required VGF custom "
110-
"shader features; use a newer source build",
111-
)
112-
113-
warnings.warn(
114-
"model-converter legacy-version xfail gate expected d8c1b8e; detected "
115-
f"{version_text!r}. Assuming a newer/custom build.",
116-
stacklevel=2,
117-
)
118-
return False, ""
97+
return True, ""
11998

12099

121-
def xfail_if_legacy_model_converter_release() -> pytest.MarkDecorator:
122-
is_legacy_release, reason = _model_converter_is_legacy_release()
123-
return pytest.mark.xfail(is_legacy_release, reason=reason, strict=False)
100+
def xfail_if_model_converter_below_minimum_version() -> pytest.MarkDecorator:
101+
supports_vgf_tests, reason = _model_converter_supports_vgf_tests()
102+
return pytest.mark.xfail(not supports_vgf_tests, reason=reason, strict=False)
124103

125104

126105
def find_single_vgf_json(output_dir: Path) -> Path:

‎backends/arm/test/runtime/test_vgf_aliasing_runtime.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@
1616
lower_sampler_vgf,
1717
lower_threes_vgf,
1818
make_sampler_probe_inputs,
19-
xfail_if_legacy_model_converter_release,
19+
xfail_if_model_converter_below_minimum_version,
2020
)
2121
from executorch.backends.arm.test import common
2222

23-
pytestmark = xfail_if_legacy_model_converter_release()
23+
pytestmark = xfail_if_model_converter_below_minimum_version()
2424

2525

2626
class _ThreesModule(torch.nn.Module):

‎backends/arm/test/runtime/test_vgf_combinations_runtime.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@
1818
lower_threes_vgf,
1919
make_sampler_probe_inputs,
2020
segment_types,
21-
xfail_if_legacy_model_converter_release,
21+
xfail_if_model_converter_below_minimum_version,
2222
)
2323
from executorch.backends.arm.test import common
2424

25-
pytestmark = xfail_if_legacy_model_converter_release()
25+
pytestmark = xfail_if_model_converter_below_minimum_version()
2626

2727

2828
def _has_alias_pair(vgf_json: dict, lhs: str, rhs: str) -> bool:

‎backends/arm/test/runtime/test_vgf_multi_segment_runtime.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@
1717
make_identity_grid,
1818
make_input_tensor,
1919
make_sampler_probe_inputs,
20-
xfail_if_legacy_model_converter_release,
20+
xfail_if_model_converter_below_minimum_version,
2121
)
2222
from executorch.backends.arm.test import common
2323

24-
pytestmark = xfail_if_legacy_model_converter_release()
24+
pytestmark = xfail_if_model_converter_below_minimum_version()
2525

2626

2727
class _GraphThenShader(torch.nn.Module):

‎backends/arm/test/runtime/test_vgf_sampler_image_runtime.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@
1616
make_identity_grid,
1717
make_input_tensor,
1818
make_sampler_probe_inputs,
19-
xfail_if_legacy_model_converter_release,
19+
xfail_if_model_converter_below_minimum_version,
2020
)
2121
from executorch.backends.arm.test import common
2222

23-
pytestmark = xfail_if_legacy_model_converter_release()
23+
pytestmark = xfail_if_model_converter_below_minimum_version()
2424

2525

2626
class _IdentitySampler(torch.nn.Module):

‎backends/arm/test/runtime/test_vgf_tensor_buffer_runtime.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@
1818
lower_in_tree_vgf,
1919
make_identity_grid,
2020
make_input_tensor,
21-
xfail_if_legacy_model_converter_release,
21+
xfail_if_model_converter_below_minimum_version,
2222
)
2323
from executorch.backends.arm.test import common
2424

25-
pytestmark = xfail_if_legacy_model_converter_release()
25+
pytestmark = xfail_if_model_converter_below_minimum_version()
2626

2727

2828
class _IdentityGridSample(torch.nn.Module):

0 commit comments

Comments
 (0)