Skip to content

Commit a36a252

Browse files
Arm backend: Fix artifact collation in collection
Compile specs created during pytest collection cannot resolve artifact directories because PYTEST_CURRENT_TEST is not yet set. Allow compile-spec factories to explicitly defer path resolution, then resolve missing paths strictly in ArmTester during test execution. Copy shared specs before assigning per-test directories and preserve explicit artifact paths. Add regression coverage for deferred resolution, missing test context, shared-spec isolation, explicit paths, and actual TOSA artifact output. Signed-off-by: Sangwon Ha <sangwon.ha@arm.com> Change-Id: I7c0261d5db54178bd528811488f5d0626acfde41
1 parent 4b1d027 commit a36a252

3 files changed

Lines changed: 113 additions & 9 deletions

File tree

‎backends/arm/test/common.py‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -57,12 +57,14 @@ def get_time_formatted_path(path: str, log_prefix: str) -> str:
5757
)
5858

5959

60-
def maybe_get_tosa_artifact_path() -> str | None:
60+
def maybe_get_tosa_artifact_path(*, allow_unresolved: bool = False) -> str | None:
6161
"""Return the configured artifact directory for the current test."""
6262
artifact_base_path = getattr(pytest, "_test_options", {}).get("dump_artifacts")
6363
if artifact_base_path:
6464
current_test = os.environ.get("PYTEST_CURRENT_TEST")
6565
if current_test is None:
66+
if allow_unresolved:
67+
return None
6668
raise RuntimeError("Could not determine the current pytest test name")
6769
test_name = (
6870
current_test.split(" (")[0]
@@ -72,16 +74,20 @@ def maybe_get_tosa_artifact_path() -> str | None:
7274
)
7375
return os.path.join(artifact_base_path, test_name)
7476

75-
return maybe_get_tosa_collate_path()
77+
return maybe_get_tosa_collate_path(allow_unresolved=allow_unresolved)
7678

7779

78-
def maybe_get_tosa_collate_path() -> str | None:
80+
def maybe_get_tosa_collate_path(*, allow_unresolved: bool = False) -> str | None:
7981
"""Return the current test's TOSA collation directory, when configured."""
8082
tosa_test_base = os.environ.get("TOSA_TESTCASES_BASE_PATH")
8183
if tosa_test_base:
8284
current_test = os.environ.get("PYTEST_CURRENT_TEST")
85+
if current_test is None:
86+
if allow_unresolved:
87+
return None
88+
raise RuntimeError("Could not determine the current pytest test name")
8389
# '::test_collate_tosa_INT_tests[randn] (call)'
84-
test_name = current_test.split("::")[1].split(" ")[0] # type: ignore[union-attr]
90+
test_name = current_test.split("::")[1].split(" ")[0]
8591
if "INT" in test_name:
8692
tosa_test_base = os.path.join(tosa_test_base, "tosa-int")
8793
elif "FP" in test_name:
@@ -100,7 +106,7 @@ def get_tosa_compile_spec(
100106
) -> TosaCompileSpec:
101107
"""Get the compile spec for default TOSA tests."""
102108
if not custom_path:
103-
custom_path = maybe_get_tosa_artifact_path()
109+
custom_path = maybe_get_tosa_artifact_path(allow_unresolved=True)
104110
if custom_path is not None:
105111
os.makedirs(custom_path, exist_ok=True)
106112

@@ -123,7 +129,7 @@ def get_u55_compile_spec(
123129
) -> EthosUCompileSpec:
124130
"""Default compile spec for Ethos-U55 tests."""
125131
if not custom_path:
126-
custom_path = maybe_get_tosa_artifact_path()
132+
custom_path = maybe_get_tosa_artifact_path(allow_unresolved=True)
127133
if custom_path is not None:
128134
os.makedirs(custom_path, exist_ok=True)
129135

@@ -167,7 +173,7 @@ def get_u85_compile_spec(
167173
) -> EthosUCompileSpec:
168174
"""Default compile spec for Ethos-U85 tests."""
169175
if not custom_path:
170-
custom_path = maybe_get_tosa_artifact_path()
176+
custom_path = maybe_get_tosa_artifact_path(allow_unresolved=True)
171177
if custom_path is not None:
172178
os.makedirs(custom_path, exist_ok=True)
173179

@@ -203,7 +209,7 @@ def get_u65_compile_spec(
203209
) -> EthosUCompileSpec:
204210
"""Default compile spec for Ethos-U65 tests."""
205211
if not custom_path:
206-
custom_path = maybe_get_tosa_artifact_path()
212+
custom_path = maybe_get_tosa_artifact_path(allow_unresolved=True)
207213
if custom_path is not None:
208214
os.makedirs(custom_path, exist_ok=True)
209215

@@ -239,7 +245,7 @@ def get_vgf_compile_spec(
239245
spec before calling .build() to finalize it.
240246
"""
241247
if not custom_path:
242-
custom_path = maybe_get_tosa_artifact_path()
248+
custom_path = maybe_get_tosa_artifact_path(allow_unresolved=True)
243249
if custom_path is not None:
244250
os.makedirs(custom_path, exist_ok=True)
245251

‎backends/arm/test/misc/test_artifact_dumping.py‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,15 @@
33
# This source code is licensed under the BSD-style license found in the
44
# LICENSE file in the root directory of this source tree.
55

6+
from functools import partial
67
from types import SimpleNamespace
78

89
import executorch.backends.arm.test.conftest as arm_conftest
910

1011
import pytest
12+
import torch
1113
from executorch.backends.arm.test import common
14+
from executorch.backends.arm.test.tester.arm_tester import ArmTester
1215

1316

1417
def _report(nodeid, outcome, when, **attributes):
@@ -180,3 +183,90 @@ def test_custom_path_overrides_dump_artifacts(monkeypatch, tmp_path) -> None:
180183
)
181184

182185
assert compile_spec._get_intermediate_path() == str(tmp_path / "custom")
186+
187+
188+
@pytest.fixture(params=["collation", "dump_artifacts"])
189+
def artifact_base(request, monkeypatch, tmp_path):
190+
monkeypatch.delenv("TOSA_TESTCASES_BASE_PATH", raising=False)
191+
options = {}
192+
monkeypatch.setattr(pytest, "_test_options", options, raising=False)
193+
if request.param == "collation":
194+
monkeypatch.setenv("TOSA_TESTCASES_BASE_PATH", str(tmp_path))
195+
return tmp_path / "tosa-int"
196+
options["dump_artifacts"] = str(tmp_path)
197+
return tmp_path
198+
199+
200+
@pytest.fixture(
201+
params=[
202+
partial(common.get_tosa_compile_spec, "TOSA-1.0+FP"),
203+
common.get_u55_compile_spec,
204+
common.get_u65_compile_spec,
205+
common.get_u85_compile_spec,
206+
partial(common.get_vgf_compile_spec, "TOSA-1.0+FP"),
207+
],
208+
ids=["tosa", "u55", "u65", "u85", "vgf"],
209+
)
210+
def compile_spec_factory(request):
211+
return request.param
212+
213+
214+
def test_artifact_path_requires_explicit_deferral(
215+
monkeypatch, artifact_base, compile_spec_factory
216+
):
217+
monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)
218+
compile_spec = compile_spec_factory()
219+
220+
assert compile_spec._get_intermediate_path() is None
221+
assert common.maybe_get_tosa_artifact_path(allow_unresolved=True) is None
222+
with pytest.raises(RuntimeError, match="current pytest test name"):
223+
common.maybe_get_tosa_artifact_path()
224+
with pytest.raises(RuntimeError, match="current pytest test name"):
225+
ArmTester(torch.nn.Identity(), (torch.ones(1),), compile_spec)
226+
227+
228+
def test_shared_compile_spec_gets_each_tests_artifact_path(
229+
monkeypatch, artifact_base, compile_spec_factory
230+
):
231+
monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)
232+
compile_spec = compile_spec_factory()
233+
testers = []
234+
235+
for name in ("test_first_INT[case]", "test_second_INT[case]"):
236+
monkeypatch.setenv("PYTEST_CURRENT_TEST", f"test_artifacts.py::{name} (call)")
237+
tester = ArmTester(torch.nn.Identity(), (torch.ones(1),), compile_spec)
238+
testers.append(tester)
239+
expected = artifact_base / name
240+
assert tester.compile_spec._get_intermediate_path() == str(expected)
241+
serialized = {spec.key: spec.value for spec in tester.compile_spec._to_list()}
242+
assert serialized["debug_artifact_path"] == str(expected).encode()
243+
assert expected.is_dir()
244+
assert compile_spec._get_intermediate_path() is None
245+
246+
assert testers[0].compile_spec._get_intermediate_path() == str(
247+
artifact_base / "test_first_INT[case]"
248+
)
249+
250+
251+
def test_explicit_artifact_path_takes_precedence_without_current_test(
252+
monkeypatch, tmp_path, artifact_base, compile_spec_factory
253+
):
254+
monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)
255+
explicit_path = tmp_path / "custom"
256+
compile_spec = compile_spec_factory(custom_path=str(explicit_path))
257+
tester = ArmTester(torch.nn.Identity(), (torch.ones(1),), compile_spec)
258+
259+
assert tester.compile_spec._get_intermediate_path() == str(explicit_path)
260+
261+
262+
def test_deferred_artifact_path_writes_tosa_output(monkeypatch, artifact_base):
263+
monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)
264+
compile_spec = common.get_tosa_compile_spec("TOSA-1.0+FP")
265+
monkeypatch.setenv(
266+
"PYTEST_CURRENT_TEST", "test_artifacts.py::test_lower_INT (call)"
267+
)
268+
tester = ArmTester(torch.nn.ReLU(), (torch.randn(1, 2, 3),), compile_spec)
269+
tester.export().to_edge_transform_and_lower()
270+
271+
assert list((artifact_base / "test_lower_INT").rglob("*.tosa"))
272+
assert compile_spec._get_intermediate_path() is None

‎backends/arm/test/tester/arm_tester.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import platform
1212

1313
from collections import Counter, defaultdict
14+
from pathlib import Path
1415
from pprint import pformat
1516
from typing import (
1617
Any,
@@ -41,6 +42,7 @@
4142
from executorch.backends.arm.common.arm_compile_spec import ArmCompileSpec
4243
from executorch.backends.arm.ethosu import EthosUCompileSpec
4344
from executorch.backends.arm.quantizer import get_symmetric_quantization_config
45+
from executorch.backends.arm.test.common import maybe_get_tosa_artifact_path
4446
from executorch.backends.arm.test.runner_utils import (
4547
dbg_tosa_fb_to_json,
4648
get_output_quantization_params,
@@ -370,6 +372,12 @@ def __init__(
370372
self.transform_passes = transform_passes
371373
self.constant_methods = constant_methods
372374
self.compile_spec = compile_spec
375+
if compile_spec._get_intermediate_path() is None:
376+
artifact_path = maybe_get_tosa_artifact_path()
377+
if artifact_path is not None:
378+
Path(artifact_path).mkdir(parents=True, exist_ok=True)
379+
self.compile_spec = copy.deepcopy(compile_spec)
380+
self.compile_spec.dump_intermediate_artifacts_to(artifact_path)
373381
stage_classes = tester.Tester.default_stage_classes() | {
374382
StageType.PARTITION: Partition,
375383
StageType.TO_EDGE_TRANSFORM_AND_LOWER: ToEdgeTransformAndLower,

0 commit comments

Comments
 (0)