Skip to content

Commit e458023

Browse files
authored
Arm backend: Remove evaluation from aot_arm_compiler.py (pytorch#18037)
Model evaluation, which means to compute and present a model's top-1/top-5 accuracy on ImageNet, as well as some other useful metrics, is to be moved into a new python program. Remove the evaluation feature from aot_arm_compiler.py, which is part of the migration. Signed-off-by: Martin Lindström <Martin.Lindstroem@arm.com>
1 parent 569e7a9 commit e458023

1 file changed

Lines changed: 18 additions & 117 deletions

File tree

‎examples/arm/aot_arm_compiler.py‎

Lines changed: 18 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
if _EXECUTORCH_DIR_STR not in sys.path:
2222
sys.path.insert(0, _EXECUTORCH_DIR_STR)
2323

24-
from typing import Any, Dict, List, Optional, Tuple
24+
from typing import Any, List, Optional, Tuple
2525

2626
import torch
2727
from examples.devtools.scripts.export_bundled_program import save_bundled_program
@@ -57,7 +57,6 @@
5757
from tabulate import tabulate
5858
from torch.export import ExportedProgram
5959
from torch.fx import GraphModule
60-
from torch.utils.data import DataLoader
6160

6261
# Quantize model if required using the standard export quantizaion flow.
6362
from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e
@@ -66,28 +65,6 @@
6665
from ..models.model_factory import EagerModelFactory
6766

6867

69-
_arm_model_evaluator = None
70-
71-
72-
def _load_arm_model_evaluator() -> Any:
73-
"""Lazily import arm_model_evaluator to avoid heavy deps when not evaluating."""
74-
global _arm_model_evaluator
75-
if _arm_model_evaluator is not None:
76-
return _arm_model_evaluator
77-
78-
try:
79-
from executorch.backends.arm.util import arm_model_evaluator as arm_eval
80-
except Exception as exc:
81-
raise RuntimeError(
82-
"Unable to run evaluation because arm_model_evaluator could not be imported. "
83-
"You probably need to install torchvision or rerun without --evaluate. "
84-
f"Original import error: {exc}"
85-
)
86-
87-
_arm_model_evaluator = arm_eval
88-
return arm_eval
89-
90-
9168
def _load_example_inputs(model_input: str | None) -> Any: # nosec B614
9269
"""Load example inputs from a `.pt` file when a path is provided."""
9370
if model_input is None:
@@ -138,7 +115,7 @@ def _load_registered_model(
138115
return None
139116

140117
logging.warning(
141-
"Using a model from examples/models not all of these are currently supported"
118+
"Using a model from examples/models. Not all of these are currently supported."
142119
)
143120
logging.info(
144121
f"Load {model_name} -> {MODEL_NAME_TO_MODEL[model_name]} from examples/models"
@@ -254,8 +231,6 @@ def quantize(
254231
model_name: str,
255232
compile_specs: ArmCompileSpec,
256233
example_inputs: Tuple[torch.Tensor],
257-
evaluator_name: str | None,
258-
evaluator_config: Dict[str, Any] | None,
259234
is_int16x8: bool = False,
260235
) -> GraphModule:
261236
"""This is the official recommended flow for quantization in pytorch 2.0
@@ -282,17 +257,9 @@ def quantize(
282257
quantizer.set_global(operator_config)
283258
m = prepare_pt2e(model, quantizer)
284259

285-
dataset = get_calibration_data(
286-
model_name, example_inputs, evaluator_name, evaluator_config
287-
)
288-
289-
# The dataset could be a tuple of tensors or a DataLoader
290-
# These two cases need to be accounted for
291-
if isinstance(dataset, DataLoader):
292-
for sample, _ in dataset:
293-
m(sample)
294-
else:
295-
m(*dataset)
260+
# Calibrate model using example inputs
261+
# TODO: Add support for using a calibration dataset
262+
m(*example_inputs)
296263

297264
m = convert_pt2e(m)
298265
logging.debug(f"Quantized model: {m}")
@@ -369,20 +336,6 @@ def forward(self, x):
369336
"qlinear": QuantLinearTest,
370337
}
371338

372-
CALIBRATION_DATA = {
373-
"qadd": (torch.randn(32, 2, 1),),
374-
"qadd2": (
375-
torch.randn(32, 2, 1),
376-
torch.randn(32, 2, 1),
377-
),
378-
"qops": (
379-
torch.randn(32, 2, 1),
380-
torch.randn(32, 2, 1),
381-
torch.randn(32, 2, 1) * -0.000001,
382-
torch.randn(32, 2, 1) * 1000,
383-
),
384-
}
385-
386339
TARGETS = [
387340
"ethos-u55-32",
388341
"ethos-u55-64",
@@ -401,30 +354,6 @@ def forward(self, x):
401354
]
402355

403356

404-
def get_calibration_data(
405-
model_name: str,
406-
example_inputs: Tuple[torch.Tensor],
407-
evaluator_name: str | None,
408-
evaluator_config: str | None,
409-
):
410-
# Firstly, if the model is being evaluated, take the evaluators calibration function if it has one
411-
if evaluator_name is not None:
412-
arm_eval = _load_arm_model_evaluator()
413-
evaluator_data = arm_eval.evaluator_calibration_data(
414-
evaluator_name, evaluator_config
415-
)
416-
if evaluator_data is not None:
417-
return evaluator_data
418-
419-
# If the model is in the CALIBRATION_DATA dictionary, get the data from there
420-
# This is used for the simple model examples provided
421-
if model_name in CALIBRATION_DATA:
422-
return CALIBRATION_DATA[model_name]
423-
424-
# As a last resort, fallback to the scripts previous behavior and return the example inputs
425-
return example_inputs
426-
427-
428357
def get_compile_spec(args) -> ArmCompileSpec:
429358
compile_spec = None
430359
if args.target.startswith("TOSA"):
@@ -523,21 +452,22 @@ def get_args():
523452
choices=TARGETS,
524453
help=f"Target backend. For delegated models: Ethos-U/VGF/TOSA variants. For non-delegated: cortex-m55+int8 (CMSIS-NN portable kernels). Valid targets: {TARGETS}",
525454
)
455+
# TODO: Remove --evaluate and --evaluate_config completely after a suitable time.
456+
# They are deprecated and no longer functional in this script.
526457
parser.add_argument(
527458
"-e",
528459
"--evaluate",
529460
required=False,
530461
nargs="?",
531462
const="generic",
532463
choices=["generic", "mv2", "deit_tiny", "resnet18"],
533-
help="Flag for running evaluation of the model.",
464+
help=argparse.SUPPRESS,
534465
)
535466
parser.add_argument(
536467
"-c",
537468
"--evaluate_config",
538469
required=False,
539-
default=None,
540-
help="Provide path to evaluator config, if it is required.",
470+
help=argparse.SUPPRESS,
541471
)
542472
parser.add_argument(
543473
"-q",
@@ -562,7 +492,7 @@ def get_args():
562492
"--intermediates",
563493
action="store",
564494
required=False,
565-
help="Store intermediate output (like TOSA artefacts) somewhere.",
495+
help="Store intermediate output (like TOSA artifacts) somewhere.",
566496
)
567497
parser.add_argument(
568498
"-o",
@@ -616,13 +546,6 @@ def get_args():
616546
)
617547
args = parser.parse_args()
618548

619-
if args.evaluate and (
620-
(not args.quantize) or args.intermediates is None or (not args.delegate)
621-
):
622-
raise RuntimeError(
623-
"--evaluate requires --quantize, --intermediates and --delegate to be enabled."
624-
)
625-
626549
LOGGING_FORMAT = "[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s"
627550
logging_level = logging.DEBUG if args.debug else logging.WARNING
628551
logging.basicConfig(level=logging_level, format=LOGGING_FORMAT, force=True)
@@ -639,6 +562,11 @@ def get_args():
639562
):
640563
raise RuntimeError(f"Model {args.model_name} cannot be delegated.")
641564

565+
if args.evaluate is not None or args.evaluate_config is not None:
566+
logging.error(
567+
"Model evaluation is no longer supported in this script. Ignore and continue."
568+
)
569+
642570
return args
643571

644572

@@ -733,8 +661,6 @@ def quantize_model(
733661
args.model_name,
734662
compile_spec,
735663
example_inputs,
736-
args.evaluate,
737-
args.evaluate_config,
738664
is_int16x8,
739665
)
740666
# Wrap quantized model back into an exported_program
@@ -812,15 +738,9 @@ def _to_channels_last(x):
812738
quantizer = CortexMQuantizer()
813739
prepared = prepare_pt2e(model, quantizer)
814740

815-
dataset = get_calibration_data(
816-
args.model_name, example_inputs, args.evaluate, args.evaluate_config
817-
)
818-
819-
if isinstance(dataset, DataLoader):
820-
for sample, _ in dataset:
821-
prepared(_to_channels_last(sample))
822-
else:
823-
prepared(*tuple(_to_channels_last(x) for x in dataset))
741+
# Calibrate model using example inputs
742+
# TODO: Add support for using a calibration dataset
743+
prepared(*example_inputs)
824744

825745
model_quant = convert_pt2e(prepared)
826746

@@ -919,12 +839,6 @@ def to_edge_no_delegate(
919839
model_quant = None
920840
if args.target == "cortex-m55+int8":
921841
# Cortex-M path: CMSIS-NN portable kernels, no delegation
922-
if getattr(args, "evaluate", False):
923-
logging.error(
924-
"--evaluate is not supported for target 'cortex-m55+int8' "
925-
"because this path does not use a TOSA delegate."
926-
)
927-
sys.exit(1)
928842
if args.delegate:
929843
logging.warning(
930844
"--delegate is ignored for target 'cortex-m55+int8' "
@@ -1001,16 +915,3 @@ def to_edge_no_delegate(
1001915
else:
1002916
save_pte_program(exec_prog, output_file_name)
1003917
print(f"PTE file saved as {output_file_name}")
1004-
1005-
if args.evaluate:
1006-
arm_eval = _load_arm_model_evaluator()
1007-
arm_eval.evaluate_model(
1008-
args.model_name,
1009-
args.intermediates,
1010-
args.target,
1011-
model_fp32,
1012-
model_quant,
1013-
example_inputs,
1014-
args.evaluate,
1015-
args.evaluate_config,
1016-
)

0 commit comments

Comments
 (0)