Skip to content

Commit bd98e2e

Browse files
Neutron Backend: Add model test (Cifar10) with eIQ NSYS (pytorch#17416)
### Summary Add end-to-end model test with small CNN trained on Cifar10 dataset with eIQ NSYS (Neutron functional simulator). ### Test plan Introducing end-to-end model test
1 parent 1c88c23 commit bd98e2e

18 files changed

Lines changed: 1494 additions & 1 deletion

‎.github/workflows/pull.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1051,6 +1051,10 @@ jobs:
10511051
10521052
# Run e2e example with Simulator:
10531053
PYTHON_EXECUTABLE=python bash examples/nxp/run.sh cifar10
1054+
1055+
# Run lightweight model tests:
1056+
PYTHON_EXECUTABLE=python pytest -c /dev/null backends/nxp/tests_models/ \
1057+
--nxp_runner_path "./examples/nxp/executor_runner/build/nxp_executor_runner"
10541058
10551059
test-samsung-quantmodels-linux:
10561060
name: test-samsung-quantmodels-linux
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
tensorflow==2.18.0
22
pytest-mock
33
tflite
4-
GvGen
4+
GvGen
5+
polars
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Copyright 2026 NXP
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Copyright 2025 NXP
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
import lzma
7+
import pickle
8+
9+
import torch
10+
11+
from torch.utils.data.dataset import Dataset
12+
13+
14+
class CalibrationDataset(Dataset):
15+
def __init__(self, data_path):
16+
if data_path.endswith(".xz"):
17+
with lzma.open(data_path) as f:
18+
self.examples = pickle.load(f)
19+
elif data_path.endswith(".pt"):
20+
self.examples = torch.load(
21+
data_path, map_location=torch.device("cpu"), weights_only=False
22+
)
23+
else:
24+
raise ValueError("Invalid file format, supported formats are .xz, .pt.")
25+
26+
def __len__(self):
27+
return len(self.examples)
28+
29+
def __getitem__(self, i):
30+
return self.examples[i]
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Copyright 2026 NXP
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
import os
7+
import pathlib
8+
import shutil
9+
10+
import eiq_neutron_sdk
11+
12+
# The PROJECT_DIR env variable is set by the conftest.py in backends.nxp.tests_models.conftest.
13+
# It is supposed to point at ExecuTorch Project directory (not install folder) to derive path to artefacts (config files,
14+
# dataset, model weight) located in the project directory structure, but not installed.
15+
# TODO(Robert Kalmar) In accordance with the "TODO(dbort): Prune /test[s]/ dirs, /third-party/ dirs" in pyproject.toml,
16+
# once the test folders are not installed we can derive the path from current file location: `pathlib.Path(__file__)`
17+
PROJECT_DIR = os.environ.get("PROJECT_DIR")
18+
assert PROJECT_DIR and os.path.exists(PROJECT_DIR)
19+
20+
OUTPUTS_DIR = pathlib.Path(os.getcwd()) / ".outputs"
21+
22+
NSYS_PATH = pathlib.Path(shutil.which("nsys"))
23+
NSYS_CONFIG_PATH = os.path.join(
24+
PROJECT_DIR, "backends", "nxp", "tests_models", "neutron-imxrt700.ini"
25+
)
26+
NSYS_FIRMWARE_PATH = os.path.join(
27+
os.path.dirname(eiq_neutron_sdk.__file__),
28+
"target",
29+
"imxrt700",
30+
"cmodel",
31+
"NeutronFirmware.elf",
32+
)
33+
34+
# The NXP_RUNNER_PATH env variable is either defined by pytest when using the CLI argument --nxp_executor_path or
35+
# a standard environment variable.
36+
NEUTRON_TEST_PATH = os.environ.get("NXP_RUNNER_PATH")
37+
assert NEUTRON_TEST_PATH and os.path.exists(NEUTRON_TEST_PATH)
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Copyright 2026 NXP
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
import logging
7+
8+
logger = logging.getLogger(__name__)
9+
10+
try:
11+
import test.python.config as test_config # noqa F401
12+
13+
logger.debug("Importing from executorch-integration")
14+
except ImportError:
15+
import executorch.backends.nxp.tests_models.config as test_config # noqa F401
16+
17+
logger.debug("Importing from executorch")
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Copyright 2026 NXP
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
7+
import os
8+
import pathlib
9+
import shutil
10+
11+
from executorch.backends.nxp.tests_models.outputs_dir_importer import outputs_dir
12+
13+
14+
def pytest_addoption(parser):
15+
parser.addoption(
16+
"--nxp_runner_path",
17+
action="store",
18+
default=None,
19+
help="Path to the nxp_executor_runner executable",
20+
)
21+
22+
23+
def pytest_configure(config):
24+
nxp_runner_path = config.getoption("--nxp_runner_path")
25+
if nxp_runner_path:
26+
os.environ["NXP_RUNNER_PATH"] = nxp_runner_path
27+
28+
os.environ["PROJECT_DIR"] = str(pathlib.Path(__file__).parent.parent.parent.parent)
29+
30+
31+
# noinspection SpellCheckingInspection
32+
def pytest_sessionstart(session):
33+
import executorch.extension.pybindings.portable_lib
34+
import executorch.kernels.quantized # noqa F401
35+
36+
# Remove all cached test files
37+
shutil.rmtree(outputs_dir.OUTPUTS_DIR, ignore_errors=True)
38+
os.mkdir(outputs_dir.OUTPUTS_DIR)

0 commit comments

Comments
 (0)