Skip to content

Commit 3ca32af

Browse files
authored
Arm backend: Arm evaluator - Support bounded dataset evaluation (pytorch#21916)
Make it possible to limit the numbers of tested samples Signed-off-by: Zingo Andersen <Zingo.Andersen@arm.com>
1 parent b45e85d commit 3ca32af

3 files changed

Lines changed: 67 additions & 8 deletions

File tree

‎backends/arm/scripts/evaluate_model.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,12 @@ def _get_args():
141141
default=1,
142142
help="Batch size to use for ImageNet evaluation. (only applicable for ImageNet evaluation).",
143143
)
144+
parser.add_argument(
145+
"--max_samples",
146+
type=int,
147+
default=None,
148+
help="Limit the number of samples evaluated.",
149+
)
144150
parser.add_argument(
145151
"-s",
146152
"--so_library",
@@ -272,6 +278,7 @@ def _evaluate(
272278
batch_size=args.batch_size,
273279
validation_dataset_path=args.evaluation_dataset,
274280
eval_dtype=_DTYPE_MAP.get(args.dtype, None),
281+
max_samples=args.max_samples,
275282
)
276283
case _:
277284
raise AssertionError(f"Unknown evaluator {evaluator_name}")

‎backends/arm/test/misc/test_model_evaluator.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,10 @@
99
import torch
1010
from executorch.backends.arm.util.arm_model_evaluator import (
1111
FileCompressionEvaluator,
12+
ImageNetEvaluator,
1213
NumericalModelEvaluator,
1314
)
15+
from torch.utils.data import TensorDataset
1416

1517
# Create an input that is hard to compress
1618
COMPRESSION_RATIO_TEST = torch.rand([1024, 1024])
@@ -52,3 +54,30 @@ def test_get_compression_ratio(self):
5254

5355
ratio = evaluator.evaluate()["compression_ratio"]
5456
self.assertAlmostEqual(ratio, 1.1, places=1)
57+
58+
def test_imagenet_evaluator_limits_samples(self):
59+
predictions = torch.eye(5)[[0, 1, 2, 3]]
60+
labels = torch.tensor([0, 1, 2, 3])
61+
evaluator = ImageNetEvaluator(
62+
"dummy_model",
63+
torch.nn.Identity(),
64+
batch_size=2,
65+
validation_dataset_path="unused",
66+
max_samples=2,
67+
validation_dataset=TensorDataset(predictions, labels),
68+
)
69+
70+
metrics = evaluator.evaluate()
71+
72+
self.assertEqual(metrics["num_samples"], 2)
73+
self.assertEqual(metrics["accuracy"], {"top-1": 1.0, "top-5": 1.0})
74+
75+
def test_imagenet_evaluator_rejects_invalid_sample_limit(self):
76+
with self.assertRaisesRegex(ValueError, "max_samples"):
77+
ImageNetEvaluator(
78+
"dummy_model",
79+
torch.nn.Identity(),
80+
batch_size=1,
81+
validation_dataset_path="unused",
82+
max_samples=0,
83+
)

‎backends/arm/util/arm_model_evaluator.py‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,15 @@
88
import os
99
import tempfile
1010
import zipfile
11+
from collections.abc import Sized
1112

1213
from pathlib import Path
1314
from typing import Any, Optional, Tuple
1415

1516
import torch
1617
from torch.nn.modules import Module
1718
from torch.utils._pytree import tree_flatten
18-
from torch.utils.data import DataLoader
19+
from torch.utils.data import DataLoader, Dataset, Subset
1920
from torchvision import datasets, transforms # type: ignore[import-untyped]
2021

2122

@@ -109,32 +110,49 @@ def __init__(
109110
batch_size: int,
110111
validation_dataset_path: str,
111112
eval_dtype: Optional[torch.dtype] = None,
113+
max_samples: Optional[int] = None,
114+
validation_dataset: Optional[Dataset[Any]] = None,
112115
) -> None:
113116
self._model_name = model_name
114117
self._eval_model = eval_model
115118
self._batch_size = batch_size
116119
self._validation_set_path = validation_dataset_path
117120
self._eval_dtype = eval_dtype
121+
if max_samples is not None and max_samples <= 0:
122+
raise ValueError("max_samples must be greater than zero")
123+
self._max_samples = max_samples
124+
self._validation_dataset = validation_dataset
118125

119126
def evaluate(self) -> dict[str, Any]:
120127
metrics: dict[str, Any] = {}
121128

122-
dataset = ImageNetEvaluator.load_imagenet_folder(self._validation_set_path)
129+
dataset: Dataset[Any]
130+
if self._validation_dataset is None:
131+
dataset = ImageNetEvaluator.load_imagenet_folder(self._validation_set_path)
132+
else:
133+
dataset = self._validation_dataset
134+
if not isinstance(dataset, Sized):
135+
raise TypeError("validation dataset must define __len__")
136+
dataset_size = len(dataset)
137+
if self._max_samples is not None:
138+
dataset_size = min(self._max_samples, dataset_size)
139+
dataset = Subset(dataset, range(dataset_size))
123140
logger.debug(
124141
"Starting ImageNet evaluation for model '%s' on dataset '%s' with %d samples.",
125142
self._model_name,
126143
self._validation_set_path,
127-
len(dataset),
144+
dataset_size,
128145
)
129146

130147
top1, top5 = self._evaluate_topk(dataset, topk=5)
148+
metrics["num_samples"] = dataset_size
131149
metrics["accuracy"] = {"top-1": top1, "top-5": top5}
132150

133151
return metrics
134152

135153
def _evaluate_topk(
136154
self,
137-
dataset: datasets.ImageFolder,
155+
dataset: Dataset[Any],
138156
topk: int = 5,
139157
log_every: int = 50,
140158
) -> Tuple[float, float]:
@@ -170,6 +188,11 @@ def _evaluate_topk(
170188
logger.debug(
171189
"Model eval() unsupported and torchao allow_exported_model_train_eval not available; proceeding."
172190
)
191+
if not isinstance(dataset, Sized):
192+
raise TypeError("validation dataset must define __len__")
193+
dataset_size = len(dataset)
194+
if dataset_size == 0:
195+
raise ValueError("validation dataset must contain at least one sample")
173196
loaded_dataset = DataLoader(dataset, batch_size=self._batch_size, shuffle=False)
174197
top1_correct = 0
175198
topk_correct = 0
@@ -187,17 +210,17 @@ def _evaluate_topk(
187210
topk_correct += (topk_indices == target_view).sum().item()
188211
batch_sz = image.size(0)
189212
total += batch_sz
190-
if (i + 1) % log_every == 0 or total == len(dataset):
213+
if (i + 1) % log_every == 0 or total == dataset_size:
191214
logger.info(
192215
"Eval progress: %d / %d top1=%.4f top%d=%.4f",
193216
total,
194-
len(dataset),
217+
dataset_size,
195218
top1_correct / total,
196219
topk,
197220
topk_correct / total,
198221
)
199-
top1_accuracy = top1_correct / len(dataset)
200-
topk_accuracy = topk_correct / len(dataset)
222+
top1_accuracy = top1_correct / dataset_size
223+
topk_accuracy = topk_correct / dataset_size
201224

202225
return top1_accuracy, topk_accuracy
203226

0 commit comments

Comments
 (0)