2020from __future__ import annotations
2121
2222import argparse
23+ import copy
2324import csv
2425import importlib
2526import json
@@ -136,6 +137,17 @@ def load_rife_model(model_root: Path, checkpoint: Path | None) -> torch.nn.Modul
136137 return RIFEWrapper (flownet ).eval ()
137138
138139
140+ def clone_model (
141+ model : torch .nn .Module ,
142+ * ,
143+ channels_last : bool = False ,
144+ ) -> torch .nn .Module :
145+ cloned_model = copy .deepcopy (model ).eval ()
146+ if channels_last :
147+ cast (Any , cloned_model ).to (memory_format = torch .channels_last )
148+ return cloned_model
149+
150+
139151def resolve_path (path_text : str , base_dir : Path ) -> Path :
140152 path = Path (path_text )
141153 if not path .is_absolute ():
@@ -309,9 +321,15 @@ def load_triple_tensors(
309321 if triple .synthetic :
310322 index_text = triple .name .rsplit ("_" , maxsplit = 1 )[- 1 ]
311323 generator = torch .Generator ().manual_seed (random_seed + int (index_text ))
312- input0 = torch .rand ((1 , 3 , height , width ), generator = generator )
313- input1 = torch .rand ((1 , 3 , height , width ), generator = generator )
314- target = torch .rand ((1 , 3 , height , width ), generator = generator )
324+ input0 = torch .rand ((1 , 3 , height , width ), generator = generator ).contiguous (
325+ memory_format = torch .channels_last
326+ )
327+ input1 = torch .rand ((1 , 3 , height , width ), generator = generator ).contiguous (
328+ memory_format = torch .channels_last
329+ )
330+ target = torch .rand ((1 , 3 , height , width ), generator = generator ).contiguous (
331+ memory_format = torch .channels_last
332+ )
315333 return (input0 , input1 ), target
316334
317335 if triple .input0 is None or triple .input1 is None or triple .target is None :
@@ -326,7 +344,10 @@ def load_triple_tensors(
326344 )
327345 for path in (triple .input0 , triple .input1 , triple .target )
328346 ]
329- input0 , input1 , target = [frame .unsqueeze (0 ) for frame in frames ]
347+ input0 , input1 , target = [
348+ frame .unsqueeze (0 ).contiguous (memory_format = torch .channels_last )
349+ for frame in frames
350+ ]
330351 return (input0 , input1 ), target
331352
332353
@@ -494,10 +515,13 @@ def run_model(
494515 inputs : tuple [torch .Tensor , torch .Tensor ],
495516) -> torch .Tensor :
496517 with torch .no_grad ():
497- output = model (* inputs )
518+ output = model (
519+ inputs [0 ].contiguous (memory_format = torch .channels_last ),
520+ inputs [1 ].contiguous (memory_format = torch .channels_last ),
521+ )
498522 if not isinstance (output , torch .Tensor ):
499523 raise TypeError (f"Expected tensor output, got { type (output )} " )
500- return output
524+ return output . contiguous ( memory_format = torch . channels_last )
501525
502526
503527def target_name (target : Any ) -> str :
@@ -654,18 +678,29 @@ def build_ptq_model(
654678 * ,
655679 io_quantization : str ,
656680) -> QuantizedModel :
681+ model = clone_model (model , channels_last = True )
682+ calibration_inputs = [
683+ (
684+ inputs [0 ].contiguous (memory_format = torch .channels_last ),
685+ inputs [1 ].contiguous (memory_format = torch .channels_last ),
686+ )
687+ for inputs in calibration_inputs
688+ ]
657689 exported_model = export (model , calibration_inputs [0 ], strict = True ).module (
658690 check_guards = False
659691 )
692+ cast (Any , exported_model ).to (memory_format = torch .channels_last )
660693 quantizer = make_quantizer (is_qat = False , io_quantization = io_quantization )
661694 prepared_model = prepare_pt2e (exported_model , quantizer )
695+ cast (Any , prepared_model ).to (memory_format = torch .channels_last )
662696 prepared_model = move_exported_model_to_eval (prepared_model )
663697
664698 with torch .no_grad ():
665699 for inputs in calibration_inputs :
666700 prepared_model (* inputs )
667701
668702 converted_model = convert_pt2e (prepared_model )
703+ cast (Any , converted_model ).to (memory_format = torch .channels_last )
669704 return QuantizedModel (
670705 model = converted_model ,
671706 coverage = quantization_coverage (converted_model ),
@@ -680,19 +715,37 @@ def build_qat_model(
680715 lr : float ,
681716 io_quantization : str ,
682717) -> QuantizedModel :
718+ model = clone_model (model , channels_last = True )
719+ training_samples = [
720+ (
721+ (
722+ inputs [0 ].contiguous (memory_format = torch .channels_last ),
723+ inputs [1 ].contiguous (memory_format = torch .channels_last ),
724+ ),
725+ target .contiguous (memory_format = torch .channels_last ),
726+ )
727+ for inputs , target in training_samples
728+ ]
683729 exported_model = export (model , training_samples [0 ][0 ], strict = True ).module (
684730 check_guards = False
685731 )
732+ cast (Any , exported_model ).to (memory_format = torch .channels_last )
686733 quantizer = make_quantizer (is_qat = True , io_quantization = io_quantization )
687734 prepared_model = prepare_qat_pt2e (exported_model , quantizer )
735+ cast (Any , prepared_model ).to (memory_format = torch .channels_last )
688736 prepared_model = move_exported_model_to_train (prepared_model )
689737 optimizer = torch .optim .SGD (prepared_model .parameters (), lr = lr , momentum = 0.9 )
690738
691739 for step in range (steps ):
692740 total_loss = 0.0
693741 for inputs , target in training_samples :
694- prediction = prepared_model (* inputs )
695- loss = F .mse_loss (prediction , target )
742+ prediction = prepared_model (* inputs ).contiguous (
743+ memory_format = torch .channels_last
744+ )
745+ loss = F .mse_loss (
746+ prediction ,
747+ target .contiguous (memory_format = torch .channels_last ),
748+ )
696749 optimizer .zero_grad ()
697750 loss .backward ()
698751 optimizer .step ()
@@ -704,6 +757,7 @@ def build_qat_model(
704757
705758 prepared_model = move_exported_model_to_eval (prepared_model )
706759 converted_model = convert_pt2e (prepared_model )
760+ cast (Any , converted_model ).to (memory_format = torch .channels_last )
707761 return QuantizedModel (
708762 model = converted_model ,
709763 coverage = quantization_coverage (converted_model ),
@@ -722,6 +776,11 @@ def export_vgf_artifact(
722776 artifact_dir .mkdir (parents = True , exist_ok = True )
723777 compile_spec = make_vgf_compile_spec (artifact_dir )
724778 partitioner = VgfPartitioner (compile_spec )
779+ cast (Any , model ).to (memory_format = torch .channels_last )
780+ inputs = (
781+ inputs [0 ].contiguous (memory_format = torch .channels_last ),
782+ inputs [1 ].contiguous (memory_format = torch .channels_last ),
783+ )
725784
726785 print (f"[vgf] exporting { name } model to { artifact_dir } ..." )
727786 aten_dialect = export (model , args = inputs , strict = True )
@@ -777,6 +836,10 @@ def write_markdown_report(
777836 for key , label in (
778837 ("eager_psnr" , "Eager PSNR" ),
779838 ("eager_ssim" , "Eager SSIM" ),
839+ ("eager_channels_last_psnr" , "Eager Channels-Last PSNR" ),
840+ ("eager_channels_last_ssim" , "Eager Channels-Last SSIM" ),
841+ ("eager_channels_last_vs_eager_mae" , "Eager Channels-Last vs eager MAE" ),
842+ ("eager_channels_last_vs_eager_sqnr" , "Eager Channels-Last vs eager SQNR" ),
780843 ("ptq_psnr" , "PTQ PSNR" ),
781844 ("ptq_ssim" , "PTQ SSIM" ),
782845 ("ptq_vs_eager_mae" , "PTQ vs eager MAE" ),
@@ -1171,6 +1234,8 @@ def print_progress(index: int, total: int, row: dict[str, Any]) -> None:
11711234 progress = f"{ index + 1 } /{ total } { row ['name' ]} "
11721235 if "eager_psnr" in row :
11731236 progress += f" eager_psnr={ row ['eager_psnr' ]:.3f} "
1237+ if "eager_channels_last_psnr" in row :
1238+ progress += f" eager_cl_psnr={ row ['eager_channels_last_psnr' ]:.3f} "
11741239 if "ptq_psnr" in row :
11751240 progress += f" ptq_psnr={ row ['ptq_psnr' ]:.3f} "
11761241 if "qat_psnr" in row :
@@ -1181,7 +1246,8 @@ def print_progress(index: int, total: int, row: dict[str, Any]) -> None:
11811246def evaluate_triples (
11821247 args : argparse .Namespace ,
11831248 triples : list [FrameTriple ],
1184- model : torch .nn .Module ,
1249+ eager_model : torch .nn .Module ,
1250+ eager_channels_last_model : torch .nn .Module | None ,
11851251 ptq_model : torch .nn .Module | None ,
11861252 qat_model : torch .nn .Module | None ,
11871253) -> list [dict [str , Any ]]:
@@ -1205,13 +1271,29 @@ def evaluate_triples(
12051271 }
12061272 eager_output = None
12071273 if run_eager :
1208- eager_output = run_model (model , inputs )
1274+ eager_output = run_model (eager_model , inputs )
12091275 update_row_with_prediction_metrics (
12101276 row ,
12111277 prefix = "eager" ,
12121278 prediction = eager_output ,
12131279 target = target ,
12141280 )
1281+ if eager_channels_last_model is not None :
1282+ eager_channels_last_output = run_model (
1283+ eager_channels_last_model , inputs
1284+ )
1285+ update_row_with_prediction_metrics (
1286+ row ,
1287+ prefix = "eager_channels_last" ,
1288+ prediction = eager_channels_last_output ,
1289+ target = target ,
1290+ )
1291+ update_row_with_delta_metrics (
1292+ row ,
1293+ prefix = "eager_channels_last_vs_eager" ,
1294+ prediction = eager_channels_last_output ,
1295+ reference = eager_output ,
1296+ )
12151297
12161298 if ptq_model is not None :
12171299 ptq_output = run_model (ptq_model , inputs )
@@ -1222,7 +1304,7 @@ def evaluate_triples(
12221304 target = target ,
12231305 )
12241306 if eager_output is None :
1225- eager_output = run_model (model , inputs )
1307+ eager_output = run_model (eager_model , inputs )
12261308 update_row_with_delta_metrics (
12271309 row ,
12281310 prefix = "ptq_vs_eager" ,
@@ -1239,7 +1321,7 @@ def evaluate_triples(
12391321 target = target ,
12401322 )
12411323 if eager_output is None :
1242- eager_output = run_model (model , inputs )
1324+ eager_output = run_model (eager_model , inputs )
12431325 update_row_with_delta_metrics (
12441326 row ,
12451327 prefix = "qat_vs_eager" ,
@@ -1254,7 +1336,8 @@ def evaluate_triples(
12541336
12551337
12561338def build_aggregate (
1257- args : argparse .Namespace , rows : list [dict [str , Any ]]
1339+ args : argparse .Namespace ,
1340+ rows : list [dict [str , Any ]],
12581341) -> dict [str , Any ]:
12591342 aggregate : dict [str , Any ] = {
12601343 "triple_count" : len (rows ),
@@ -1266,7 +1349,15 @@ def build_aggregate(
12661349 "data_source" : "random" if rows and rows [0 ].get ("synthetic" ) else "frames" ,
12671350 "random_seed" : args .random_seed ,
12681351 }
1269- for prefix in ("eager" , "ptq" , "ptq_vs_eager" , "qat" , "qat_vs_eager" ):
1352+ for prefix in (
1353+ "eager" ,
1354+ "eager_channels_last" ,
1355+ "eager_channels_last_vs_eager" ,
1356+ "ptq" ,
1357+ "ptq_vs_eager" ,
1358+ "qat" ,
1359+ "qat_vs_eager" ,
1360+ ):
12701361 aggregate .update (aggregate_metrics (rows , prefix ))
12711362 return aggregate
12721363
@@ -1303,6 +1394,32 @@ def write_outputs(
13031394 print (f"Wrote { args .output_dir / 'report.md' } " )
13041395
13051396
1397+ def print_summary (aggregate : dict [str , Any ]) -> None :
1398+ print ("" )
1399+ print ("== Summary ==" )
1400+ for prefix , label in (
1401+ ("eager" , "eager" ),
1402+ ("eager_channels_last" , "eager_channels_last" ),
1403+ ("ptq" , "ptq" ),
1404+ ("qat" , "qat" ),
1405+ ):
1406+ if f"{ prefix } _psnr_mean" not in aggregate :
1407+ continue
1408+ summary = (
1409+ f"{ label } : "
1410+ f"psnr={ format_metric (aggregate .get (f'{ prefix } _psnr_mean' ))} "
1411+ f"ssim={ format_metric (aggregate .get (f'{ prefix } _ssim_mean' ))} "
1412+ )
1413+ mae_key = f"{ prefix } _vs_eager_mae_mean"
1414+ sqnr_key = f"{ prefix } _vs_eager_sqnr_mean"
1415+ if mae_key in aggregate :
1416+ summary += (
1417+ f" vs_eager_mae={ format_metric (aggregate .get (mae_key ))} "
1418+ f" vs_eager_sqnr={ format_metric (aggregate .get (sqnr_key ))} "
1419+ )
1420+ print (summary )
1421+
1422+
13061423def main () -> int :
13071424 args = parse_args ()
13081425 validate_args (args )
@@ -1319,13 +1436,28 @@ def main() -> int:
13191436 print (f"Data source: { source } " )
13201437
13211438 model = load_rife_model (args .model_root , args .checkpoint )
1439+ eager_channels_last_model = clone_model (model , channels_last = True )
13221440 ptq_model , qat_model , quantization_reports = build_requested_quantized_models (
13231441 args ,
13241442 model ,
13251443 triples ,
13261444 )
1327- rows = evaluate_triples (args , triples , model , ptq_model , qat_model )
1328- write_outputs (args , build_aggregate (args , rows ), rows , quantization_reports )
1445+ rows = evaluate_triples (
1446+ args ,
1447+ triples ,
1448+ model ,
1449+ eager_channels_last_model ,
1450+ ptq_model ,
1451+ qat_model ,
1452+ )
1453+ aggregate = build_aggregate (args , rows )
1454+ write_outputs (
1455+ args ,
1456+ aggregate ,
1457+ rows ,
1458+ quantization_reports ,
1459+ )
1460+ print_summary (aggregate )
13291461 export_requested_vgf_artifacts (args , triples , ptq_model , qat_model )
13301462 return 0
13311463
0 commit comments