2121if _EXECUTORCH_DIR_STR not in sys .path :
2222 sys .path .insert (0 , _EXECUTORCH_DIR_STR )
2323
24- from typing import Any , Dict , List , Optional , Tuple
24+ from typing import Any , List , Optional , Tuple
2525
2626import torch
2727from examples .devtools .scripts .export_bundled_program import save_bundled_program
5757from tabulate import tabulate
5858from torch .export import ExportedProgram
5959from torch .fx import GraphModule
60- from torch .utils .data import DataLoader
6160
6261# Quantize model if required using the standard export quantizaion flow.
6362from torchao .quantization .pt2e .quantize_pt2e import convert_pt2e , prepare_pt2e
6665from ..models .model_factory import EagerModelFactory
6766
6867
69- _arm_model_evaluator = None
70-
71-
72- def _load_arm_model_evaluator () -> Any :
73- """Lazily import arm_model_evaluator to avoid heavy deps when not evaluating."""
74- global _arm_model_evaluator
75- if _arm_model_evaluator is not None :
76- return _arm_model_evaluator
77-
78- try :
79- from executorch .backends .arm .util import arm_model_evaluator as arm_eval
80- except Exception as exc :
81- raise RuntimeError (
82- "Unable to run evaluation because arm_model_evaluator could not be imported. "
83- "You probably need to install torchvision or rerun without --evaluate. "
84- f"Original import error: { exc } "
85- )
86-
87- _arm_model_evaluator = arm_eval
88- return arm_eval
89-
90-
9168def _load_example_inputs (model_input : str | None ) -> Any : # nosec B614
9269 """Load example inputs from a `.pt` file when a path is provided."""
9370 if model_input is None :
@@ -138,7 +115,7 @@ def _load_registered_model(
138115 return None
139116
140117 logging .warning (
141- "Using a model from examples/models not all of these are currently supported"
118+ "Using a model from examples/models. Not all of these are currently supported. "
142119 )
143120 logging .info (
144121 f"Load { model_name } -> { MODEL_NAME_TO_MODEL [model_name ]} from examples/models"
@@ -254,8 +231,6 @@ def quantize(
254231 model_name : str ,
255232 compile_specs : ArmCompileSpec ,
256233 example_inputs : Tuple [torch .Tensor ],
257- evaluator_name : str | None ,
258- evaluator_config : Dict [str , Any ] | None ,
259234 is_int16x8 : bool = False ,
260235) -> GraphModule :
261236 """This is the official recommended flow for quantization in pytorch 2.0
@@ -282,17 +257,9 @@ def quantize(
282257 quantizer .set_global (operator_config )
283258 m = prepare_pt2e (model , quantizer )
284259
285- dataset = get_calibration_data (
286- model_name , example_inputs , evaluator_name , evaluator_config
287- )
288-
289- # The dataset could be a tuple of tensors or a DataLoader
290- # These two cases need to be accounted for
291- if isinstance (dataset , DataLoader ):
292- for sample , _ in dataset :
293- m (sample )
294- else :
295- m (* dataset )
260+ # Calibrate model using example inputs
261+ # TODO: Add support for using a calibration dataset
262+ m (* example_inputs )
296263
297264 m = convert_pt2e (m )
298265 logging .debug (f"Quantized model: { m } " )
@@ -369,20 +336,6 @@ def forward(self, x):
369336 "qlinear" : QuantLinearTest ,
370337}
371338
372- CALIBRATION_DATA = {
373- "qadd" : (torch .randn (32 , 2 , 1 ),),
374- "qadd2" : (
375- torch .randn (32 , 2 , 1 ),
376- torch .randn (32 , 2 , 1 ),
377- ),
378- "qops" : (
379- torch .randn (32 , 2 , 1 ),
380- torch .randn (32 , 2 , 1 ),
381- torch .randn (32 , 2 , 1 ) * - 0.000001 ,
382- torch .randn (32 , 2 , 1 ) * 1000 ,
383- ),
384- }
385-
386339TARGETS = [
387340 "ethos-u55-32" ,
388341 "ethos-u55-64" ,
@@ -401,30 +354,6 @@ def forward(self, x):
401354]
402355
403356
404- def get_calibration_data (
405- model_name : str ,
406- example_inputs : Tuple [torch .Tensor ],
407- evaluator_name : str | None ,
408- evaluator_config : str | None ,
409- ):
410- # Firstly, if the model is being evaluated, take the evaluators calibration function if it has one
411- if evaluator_name is not None :
412- arm_eval = _load_arm_model_evaluator ()
413- evaluator_data = arm_eval .evaluator_calibration_data (
414- evaluator_name , evaluator_config
415- )
416- if evaluator_data is not None :
417- return evaluator_data
418-
419- # If the model is in the CALIBRATION_DATA dictionary, get the data from there
420- # This is used for the simple model examples provided
421- if model_name in CALIBRATION_DATA :
422- return CALIBRATION_DATA [model_name ]
423-
424- # As a last resort, fallback to the scripts previous behavior and return the example inputs
425- return example_inputs
426-
427-
428357def get_compile_spec (args ) -> ArmCompileSpec :
429358 compile_spec = None
430359 if args .target .startswith ("TOSA" ):
@@ -523,21 +452,22 @@ def get_args():
523452 choices = TARGETS ,
524453 help = f"Target backend. For delegated models: Ethos-U/VGF/TOSA variants. For non-delegated: cortex-m55+int8 (CMSIS-NN portable kernels). Valid targets: { TARGETS } " ,
525454 )
455+ # TODO: Remove --evaluate and --evaluate_config completely after a suitable time.
456+ # They are deprecated and no longer functional in this script.
526457 parser .add_argument (
527458 "-e" ,
528459 "--evaluate" ,
529460 required = False ,
530461 nargs = "?" ,
531462 const = "generic" ,
532463 choices = ["generic" , "mv2" , "deit_tiny" , "resnet18" ],
533- help = "Flag for running evaluation of the model." ,
464+ help = argparse . SUPPRESS ,
534465 )
535466 parser .add_argument (
536467 "-c" ,
537468 "--evaluate_config" ,
538469 required = False ,
539- default = None ,
540- help = "Provide path to evaluator config, if it is required." ,
470+ help = argparse .SUPPRESS ,
541471 )
542472 parser .add_argument (
543473 "-q" ,
@@ -562,7 +492,7 @@ def get_args():
562492 "--intermediates" ,
563493 action = "store" ,
564494 required = False ,
565- help = "Store intermediate output (like TOSA artefacts ) somewhere." ,
495+ help = "Store intermediate output (like TOSA artifacts ) somewhere." ,
566496 )
567497 parser .add_argument (
568498 "-o" ,
@@ -616,13 +546,6 @@ def get_args():
616546 )
617547 args = parser .parse_args ()
618548
619- if args .evaluate and (
620- (not args .quantize ) or args .intermediates is None or (not args .delegate )
621- ):
622- raise RuntimeError (
623- "--evaluate requires --quantize, --intermediates and --delegate to be enabled."
624- )
625-
626549 LOGGING_FORMAT = "[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s"
627550 logging_level = logging .DEBUG if args .debug else logging .WARNING
628551 logging .basicConfig (level = logging_level , format = LOGGING_FORMAT , force = True )
@@ -639,6 +562,11 @@ def get_args():
639562 ):
640563 raise RuntimeError (f"Model { args .model_name } cannot be delegated." )
641564
565+ if args .evaluate is not None or args .evaluate_config is not None :
566+ logging .error (
567+ "Model evaluation is no longer supported in this script. Ignore and continue."
568+ )
569+
642570 return args
643571
644572
@@ -733,8 +661,6 @@ def quantize_model(
733661 args .model_name ,
734662 compile_spec ,
735663 example_inputs ,
736- args .evaluate ,
737- args .evaluate_config ,
738664 is_int16x8 ,
739665 )
740666 # Wrap quantized model back into an exported_program
@@ -812,15 +738,9 @@ def _to_channels_last(x):
812738 quantizer = CortexMQuantizer ()
813739 prepared = prepare_pt2e (model , quantizer )
814740
815- dataset = get_calibration_data (
816- args .model_name , example_inputs , args .evaluate , args .evaluate_config
817- )
818-
819- if isinstance (dataset , DataLoader ):
820- for sample , _ in dataset :
821- prepared (_to_channels_last (sample ))
822- else :
823- prepared (* tuple (_to_channels_last (x ) for x in dataset ))
741+ # Calibrate model using example inputs
742+ # TODO: Add support for using a calibration dataset
743+ prepared (* example_inputs )
824744
825745 model_quant = convert_pt2e (prepared )
826746
@@ -919,12 +839,6 @@ def to_edge_no_delegate(
919839 model_quant = None
920840 if args .target == "cortex-m55+int8" :
921841 # Cortex-M path: CMSIS-NN portable kernels, no delegation
922- if getattr (args , "evaluate" , False ):
923- logging .error (
924- "--evaluate is not supported for target 'cortex-m55+int8' "
925- "because this path does not use a TOSA delegate."
926- )
927- sys .exit (1 )
928842 if args .delegate :
929843 logging .warning (
930844 "--delegate is ignored for target 'cortex-m55+int8' "
@@ -1001,16 +915,3 @@ def to_edge_no_delegate(
1001915 else :
1002916 save_pte_program (exec_prog , output_file_name )
1003917 print (f"PTE file saved as { output_file_name } " )
1004-
1005- if args .evaluate :
1006- arm_eval = _load_arm_model_evaluator ()
1007- arm_eval .evaluate_model (
1008- args .model_name ,
1009- args .intermediates ,
1010- args .target ,
1011- model_fp32 ,
1012- model_quant ,
1013- example_inputs ,
1014- args .evaluate ,
1015- args .evaluate_config ,
1016- )
0 commit comments