Skip to content

Commit d92a619

Browse files
authored
Revert "Arm backend: Xfail RIFE VGF tests on old converter" (pytorch#21783)
Reverts pytorch#21755 Failing internal CI cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils @Sebastian-Larsson @robell @rascani
1 parent fdae61d commit d92a619

9 files changed

Lines changed: 60 additions & 274 deletions

‎backends/arm/test/conftest.py‎

Lines changed: 5 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
#
33
# This source code is licensed under the BSD-style license found in the
44
# LICENSE file in the root directory of this source tree.
5-
"""Pytest hooks and fixtures for the Arm test suite."""
65

76
import logging
87
import os
@@ -11,11 +10,10 @@
1110
from typing import Any
1211

1312
import pytest
14-
from executorch.backends.arm.vgf.model_converter import (
15-
get_model_converter_minimum_version_failure_reason,
16-
get_model_converter_version_text,
17-
MIN_MODEL_CONVERTER_VERSION_FOR_VGF_TESTS,
18-
)
13+
14+
"""
15+
This file contains the pytest hooks, fixtures etc. for the Arm test suite.
16+
"""
1917

2018

2119
# ==== Pytest hooks ====
@@ -43,39 +41,8 @@ def pytest_report_header(config):
4341
return config._test_seed_label
4442

4543

46-
def _mark_rife_vgf_xfails_for_model_converter_below_minimum_version(
47-
items, reason: str
48-
) -> None:
49-
for item in items:
50-
nodeid = item.nodeid.lower()
51-
if "test_rife.py" not in nodeid or "vgf_quant" not in nodeid:
52-
continue
53-
item.add_marker(pytest.mark.xfail(reason=reason, strict=False))
54-
55-
56-
def _has_rife_vgf_quant_tests(items) -> bool:
57-
return any(
58-
"test_rife.py" in item.nodeid.lower() and "vgf_quant" in item.nodeid.lower()
59-
for item in items
60-
)
61-
62-
6344
def pytest_collection_modifyitems(config, items):
64-
if not _has_rife_vgf_quant_tests(items):
65-
return
66-
67-
version_text = get_model_converter_version_text()
68-
if version_text is None:
69-
return
70-
71-
reason = get_model_converter_minimum_version_failure_reason(
72-
version_text,
73-
MIN_MODEL_CONVERTER_VERSION_FOR_VGF_TESTS,
74-
requirement_name="the copied RIFE VGF quant tests",
75-
)
76-
if reason is None:
77-
return
78-
_mark_rife_vgf_xfails_for_model_converter_below_minimum_version(items, reason)
45+
pass
7946

8047

8148
def pytest_addoption(parser):

‎backends/arm/test/misc/test_vgf_check_env.py‎

Lines changed: 0 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,11 @@
77

88
import stat
99
from pathlib import Path
10-
from typing import Any
1110

1211
import executorch.backends.arm.vgf.check_env as check_env
1312
import executorch.backends.arm.vgf.model_converter as model_converter
1413

1514
import pytest
16-
from executorch.backends.arm.test import conftest as arm_conftest
1715
from executorch.backends.arm.vgf import backend as vgf_backend
1816
from executorch.backends.arm.vgf.compile_spec import VgfCompileSpec
1917

@@ -153,137 +151,6 @@ def test_model_converter_check_reports_version(monkeypatch, tmp_path):
153151
assert "0.9.0" in result.detail
154152

155153

156-
def test_get_model_converter_version_text(monkeypatch, tmp_path):
157-
converter = _make_executable(
158-
tmp_path / "model-converter",
159-
"#!/usr/bin/env python3\n"
160-
"import sys\n"
161-
"if '--version' in sys.argv:\n"
162-
" print('model-converter d8c1b8e')\n"
163-
" raise SystemExit(0)\n"
164-
"raise SystemExit(1)\n",
165-
)
166-
monkeypatch.setattr(
167-
model_converter, "find_model_converter_binary", lambda: str(converter)
168-
)
169-
170-
assert model_converter.get_model_converter_version_text() == (
171-
"model-converter d8c1b8e"
172-
)
173-
174-
175-
def test_parse_model_converter_version_uses_known_build_alias(monkeypatch, tmp_path):
176-
converter = _make_executable(
177-
tmp_path / "model-converter",
178-
"#!/usr/bin/env python3\n"
179-
"import sys\n"
180-
"if '--version' in sys.argv:\n"
181-
" print('model-converter d8c1b8e')\n"
182-
" raise SystemExit(0)\n"
183-
"raise SystemExit(1)\n",
184-
)
185-
monkeypatch.setattr(
186-
model_converter, "find_model_converter_binary", lambda: str(converter)
187-
)
188-
189-
version_text = model_converter.get_model_converter_version_text()
190-
191-
assert version_text is not None
192-
assert model_converter.parse_model_converter_version(version_text) == (
193-
model_converter.Version("0.9.0")
194-
)
195-
196-
197-
def test_below_minimum_model_converter_reason(monkeypatch, tmp_path):
198-
converter = _make_executable(
199-
tmp_path / "model-converter",
200-
"#!/usr/bin/env python3\n"
201-
"import sys\n"
202-
"if '--version' in sys.argv:\n"
203-
" print('model-converter d8c1b8e')\n"
204-
" raise SystemExit(0)\n"
205-
"raise SystemExit(1)\n",
206-
)
207-
monkeypatch.setattr(
208-
model_converter, "find_model_converter_binary", lambda: str(converter)
209-
)
210-
211-
version_text = model_converter.get_model_converter_version_text()
212-
213-
assert version_text is not None
214-
assert model_converter.get_model_converter_minimum_version_failure_reason(
215-
version_text,
216-
model_converter.MIN_MODEL_CONVERTER_VERSION_FOR_VGF_TESTS,
217-
requirement_name="the copied RIFE VGF quant tests",
218-
) == (
219-
"model-converter d8c1b8e is below the minimum supported version "
220-
"0.10.0 required for the copied RIFE VGF quant tests"
221-
)
222-
223-
224-
def test_mark_rife_vgf_xfails_marks_only_rife_vgf_quant():
225-
class DummyItem:
226-
def __init__(self, nodeid: str):
227-
self.nodeid = nodeid
228-
self.markers: list[Any] = []
229-
230-
def add_marker(self, marker: Any) -> None:
231-
self.markers.append(marker)
232-
233-
matching = DummyItem("backends/arm/test/models/test_RIFE.py::test_vgf_quant")
234-
non_matching = DummyItem("backends/arm/test/models/test_RIFE.py::test_tosa")
235-
236-
arm_conftest._mark_rife_vgf_xfails_for_model_converter_below_minimum_version(
237-
[matching, non_matching], "below minimum version"
238-
)
239-
240-
assert len(matching.markers) == 1
241-
assert matching.markers[0].name == "xfail"
242-
assert not non_matching.markers
243-
244-
245-
def test_has_rife_vgf_quant_tests_matches_only_rife_vgf_quant():
246-
class DummyItem:
247-
__slots__ = ("nodeid",)
248-
249-
def __init__(self, nodeid: str):
250-
self.nodeid = nodeid
251-
252-
assert arm_conftest._has_rife_vgf_quant_tests(
253-
[
254-
DummyItem("backends/arm/test/models/test_RIFE.py::test_tosa"),
255-
DummyItem("backends/arm/test/models/test_RIFE.py::test_vgf_quant"),
256-
]
257-
)
258-
assert not arm_conftest._has_rife_vgf_quant_tests(
259-
[
260-
DummyItem("backends/arm/test/models/test_RIFE.py::test_tosa"),
261-
DummyItem("backends/arm/test/ops/test_add.py::test_basic"),
262-
]
263-
)
264-
265-
266-
def test_collection_hook_skips_converter_probe_when_no_rife_vgf_quant(monkeypatch):
267-
class DummyItem:
268-
__slots__ = ("nodeid",)
269-
270-
def __init__(self, nodeid: str):
271-
self.nodeid = nodeid
272-
273-
def fail_probe():
274-
raise AssertionError("converter probe should not run")
275-
276-
monkeypatch.setattr(arm_conftest, "get_model_converter_version_text", fail_probe)
277-
278-
arm_conftest.pytest_collection_modifyitems(
279-
None,
280-
[
281-
DummyItem("backends/arm/test/models/test_RIFE.py::test_tosa"),
282-
DummyItem("backends/arm/test/ops/test_add.py::test_basic"),
283-
],
284-
)
285-
286-
287154
def test_model_converter_lib_dir_fails_when_invalid(monkeypatch, tmp_path):
288155
missing = tmp_path / "missing"
289156
monkeypatch.setenv("MODEL_CONVERTER_LIB_DIR", str(missing))

‎backends/arm/test/runtime/_vgf_runtime_test_utils.py‎

Lines changed: 45 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,8 @@
4242
)
4343
from executorch.backends.arm.vgf import VgfCompileSpec, VgfPartitioner
4444
from executorch.backends.arm.vgf.model_converter import (
45-
get_model_converter_minimum_version_failure_reason,
46-
get_model_converter_version_text,
47-
MIN_MODEL_CONVERTER_VERSION_FOR_VGF_TESTS,
48-
parse_model_converter_version,
45+
find_model_converter_binary,
46+
model_converter_env,
4947
)
5048
from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower
5149
from executorch.exir.pass_base import ExportPass
@@ -69,37 +67,60 @@ def ensure_glslc() -> None:
6967

7068

7169
@functools.lru_cache(maxsize=1)
72-
def _model_converter_supports_vgf_tests() -> tuple[bool, str]:
73-
version_text = get_model_converter_version_text()
74-
if version_text is None:
70+
def _model_converter_is_legacy_release() -> tuple[bool, str]:
71+
model_converter = find_model_converter_binary()
72+
if model_converter is None:
7573
warnings.warn(
7674
"Could not find model-converter while evaluating the VGF runtime "
77-
"minimum-version xfail gate; assuming a newer/custom build.",
75+
"legacy-version xfail gate; assuming a newer/custom build.",
7876
stacklevel=2,
7977
)
80-
return True, ""
78+
return False, ""
8179

82-
reason = get_model_converter_minimum_version_failure_reason(
83-
version_text,
84-
MIN_MODEL_CONVERTER_VERSION_FOR_VGF_TESTS,
85-
requirement_name="these VGF runtime tests",
86-
)
87-
if reason is not None:
88-
return False, reason
80+
try:
81+
result = subprocess.run( # nosec B603 - trusted local tool
82+
[model_converter, "--version"],
83+
check=True,
84+
capture_output=True,
85+
text=True,
86+
env=model_converter_env(),
87+
)
88+
except Exception as exc:
89+
warnings.warn(
90+
"Failed to query model-converter --version while evaluating the VGF "
91+
f"runtime legacy-version xfail gate ({exc}); assuming a newer/custom "
92+
"build.",
93+
stacklevel=2,
94+
)
95+
return False, ""
8996

90-
if parse_model_converter_version(version_text) is None:
97+
version_text = (result.stdout or result.stderr).strip()
98+
if not version_text:
9199
warnings.warn(
92-
"Could not map model-converter version output to a comparable "
93-
f"release while evaluating the VGF runtime minimum-version xfail "
94-
f"gate; detected {version_text!r}. Assuming a newer/custom build.",
100+
"model-converter --version returned no output while evaluating the VGF "
101+
"runtime legacy-version xfail gate; assuming a newer/custom build.",
95102
stacklevel=2,
96103
)
97-
return True, ""
104+
return False, ""
105+
106+
if "d8c1b8e" in version_text:
107+
return (
108+
True,
109+
"released model-converter build d8c1b8e predates required VGF custom "
110+
"shader features; use a newer source build",
111+
)
112+
113+
warnings.warn(
114+
"model-converter legacy-version xfail gate expected d8c1b8e; detected "
115+
f"{version_text!r}. Assuming a newer/custom build.",
116+
stacklevel=2,
117+
)
118+
return False, ""
98119

99120

100-
def xfail_if_model_converter_below_minimum_version() -> pytest.MarkDecorator:
101-
supports_vgf_tests, reason = _model_converter_supports_vgf_tests()
102-
return pytest.mark.xfail(not supports_vgf_tests, reason=reason, strict=False)
121+
def xfail_if_legacy_model_converter_release() -> pytest.MarkDecorator:
122+
is_legacy_release, reason = _model_converter_is_legacy_release()
123+
return pytest.mark.xfail(is_legacy_release, reason=reason, strict=False)
103124

104125

105126
def find_single_vgf_json(output_dir: Path) -> Path:

‎backends/arm/test/runtime/test_vgf_aliasing_runtime.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@
1616
lower_sampler_vgf,
1717
lower_threes_vgf,
1818
make_sampler_probe_inputs,
19-
xfail_if_model_converter_below_minimum_version,
19+
xfail_if_legacy_model_converter_release,
2020
)
2121
from executorch.backends.arm.test import common
2222

23-
pytestmark = xfail_if_model_converter_below_minimum_version()
23+
pytestmark = xfail_if_legacy_model_converter_release()
2424

2525

2626
class _ThreesModule(torch.nn.Module):

‎backends/arm/test/runtime/test_vgf_combinations_runtime.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@
1818
lower_threes_vgf,
1919
make_sampler_probe_inputs,
2020
segment_types,
21-
xfail_if_model_converter_below_minimum_version,
21+
xfail_if_legacy_model_converter_release,
2222
)
2323
from executorch.backends.arm.test import common
2424

25-
pytestmark = xfail_if_model_converter_below_minimum_version()
25+
pytestmark = xfail_if_legacy_model_converter_release()
2626

2727

2828
def _has_alias_pair(vgf_json: dict, lhs: str, rhs: str) -> bool:

‎backends/arm/test/runtime/test_vgf_multi_segment_runtime.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@
1717
make_identity_grid,
1818
make_input_tensor,
1919
make_sampler_probe_inputs,
20-
xfail_if_model_converter_below_minimum_version,
20+
xfail_if_legacy_model_converter_release,
2121
)
2222
from executorch.backends.arm.test import common
2323

24-
pytestmark = xfail_if_model_converter_below_minimum_version()
24+
pytestmark = xfail_if_legacy_model_converter_release()
2525

2626

2727
class _GraphThenShader(torch.nn.Module):

‎backends/arm/test/runtime/test_vgf_sampler_image_runtime.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@
1616
make_identity_grid,
1717
make_input_tensor,
1818
make_sampler_probe_inputs,
19-
xfail_if_model_converter_below_minimum_version,
19+
xfail_if_legacy_model_converter_release,
2020
)
2121
from executorch.backends.arm.test import common
2222

23-
pytestmark = xfail_if_model_converter_below_minimum_version()
23+
pytestmark = xfail_if_legacy_model_converter_release()
2424

2525

2626
class _IdentitySampler(torch.nn.Module):

‎backends/arm/test/runtime/test_vgf_tensor_buffer_runtime.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@
1818
lower_in_tree_vgf,
1919
make_identity_grid,
2020
make_input_tensor,
21-
xfail_if_model_converter_below_minimum_version,
21+
xfail_if_legacy_model_converter_release,
2222
)
2323
from executorch.backends.arm.test import common
2424

25-
pytestmark = xfail_if_model_converter_below_minimum_version()
25+
pytestmark = xfail_if_legacy_model_converter_release()
2626

2727

2828
class _IdentityGridSample(torch.nn.Module):

0 commit comments

Comments
 (0)