Skip to content

Commit 1f7c6d0

Browse files
authored
Arm backend: run RIFE QAT flow in channels-last (pytorch#21395)
Change-Id: If842e08ba261e791a823e4d4c228e71a1d926e46 Signed-off-by: Rob Elliott <Robert.Elliott@arm.com> small adjustment for the example to ensure the inputs are channels last to conform to heuristic expectations.
1 parent d0fe35d commit 1f7c6d0

1 file changed

Lines changed: 148 additions & 16 deletions

File tree

‎examples/arm/QAT_example/qat_loop.py‎

Lines changed: 148 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from __future__ import annotations
2121

2222
import argparse
23+
import copy
2324
import csv
2425
import importlib
2526
import json
@@ -136,6 +137,17 @@ def load_rife_model(model_root: Path, checkpoint: Path | None) -> torch.nn.Modul
136137
return RIFEWrapper(flownet).eval()
137138

138139

140+
def clone_model(
141+
model: torch.nn.Module,
142+
*,
143+
channels_last: bool = False,
144+
) -> torch.nn.Module:
145+
cloned_model = copy.deepcopy(model).eval()
146+
if channels_last:
147+
cast(Any, cloned_model).to(memory_format=torch.channels_last)
148+
return cloned_model
149+
150+
139151
def resolve_path(path_text: str, base_dir: Path) -> Path:
140152
path = Path(path_text)
141153
if not path.is_absolute():
@@ -309,9 +321,15 @@ def load_triple_tensors(
309321
if triple.synthetic:
310322
index_text = triple.name.rsplit("_", maxsplit=1)[-1]
311323
generator = torch.Generator().manual_seed(random_seed + int(index_text))
312-
input0 = torch.rand((1, 3, height, width), generator=generator)
313-
input1 = torch.rand((1, 3, height, width), generator=generator)
314-
target = torch.rand((1, 3, height, width), generator=generator)
324+
input0 = torch.rand((1, 3, height, width), generator=generator).contiguous(
325+
memory_format=torch.channels_last
326+
)
327+
input1 = torch.rand((1, 3, height, width), generator=generator).contiguous(
328+
memory_format=torch.channels_last
329+
)
330+
target = torch.rand((1, 3, height, width), generator=generator).contiguous(
331+
memory_format=torch.channels_last
332+
)
315333
return (input0, input1), target
316334

317335
if triple.input0 is None or triple.input1 is None or triple.target is None:
@@ -326,7 +344,10 @@ def load_triple_tensors(
326344
)
327345
for path in (triple.input0, triple.input1, triple.target)
328346
]
329-
input0, input1, target = [frame.unsqueeze(0) for frame in frames]
347+
input0, input1, target = [
348+
frame.unsqueeze(0).contiguous(memory_format=torch.channels_last)
349+
for frame in frames
350+
]
330351
return (input0, input1), target
331352

332353

@@ -494,10 +515,13 @@ def run_model(
494515
inputs: tuple[torch.Tensor, torch.Tensor],
495516
) -> torch.Tensor:
496517
with torch.no_grad():
497-
output = model(*inputs)
518+
output = model(
519+
inputs[0].contiguous(memory_format=torch.channels_last),
520+
inputs[1].contiguous(memory_format=torch.channels_last),
521+
)
498522
if not isinstance(output, torch.Tensor):
499523
raise TypeError(f"Expected tensor output, got {type(output)}")
500-
return output
524+
return output.contiguous(memory_format=torch.channels_last)
501525

502526

503527
def target_name(target: Any) -> str:
@@ -654,18 +678,29 @@ def build_ptq_model(
654678
*,
655679
io_quantization: str,
656680
) -> QuantizedModel:
681+
model = clone_model(model, channels_last=True)
682+
calibration_inputs = [
683+
(
684+
inputs[0].contiguous(memory_format=torch.channels_last),
685+
inputs[1].contiguous(memory_format=torch.channels_last),
686+
)
687+
for inputs in calibration_inputs
688+
]
657689
exported_model = export(model, calibration_inputs[0], strict=True).module(
658690
check_guards=False
659691
)
692+
cast(Any, exported_model).to(memory_format=torch.channels_last)
660693
quantizer = make_quantizer(is_qat=False, io_quantization=io_quantization)
661694
prepared_model = prepare_pt2e(exported_model, quantizer)
695+
cast(Any, prepared_model).to(memory_format=torch.channels_last)
662696
prepared_model = move_exported_model_to_eval(prepared_model)
663697

664698
with torch.no_grad():
665699
for inputs in calibration_inputs:
666700
prepared_model(*inputs)
667701

668702
converted_model = convert_pt2e(prepared_model)
703+
cast(Any, converted_model).to(memory_format=torch.channels_last)
669704
return QuantizedModel(
670705
model=converted_model,
671706
coverage=quantization_coverage(converted_model),
@@ -680,19 +715,37 @@ def build_qat_model(
680715
lr: float,
681716
io_quantization: str,
682717
) -> QuantizedModel:
718+
model = clone_model(model, channels_last=True)
719+
training_samples = [
720+
(
721+
(
722+
inputs[0].contiguous(memory_format=torch.channels_last),
723+
inputs[1].contiguous(memory_format=torch.channels_last),
724+
),
725+
target.contiguous(memory_format=torch.channels_last),
726+
)
727+
for inputs, target in training_samples
728+
]
683729
exported_model = export(model, training_samples[0][0], strict=True).module(
684730
check_guards=False
685731
)
732+
cast(Any, exported_model).to(memory_format=torch.channels_last)
686733
quantizer = make_quantizer(is_qat=True, io_quantization=io_quantization)
687734
prepared_model = prepare_qat_pt2e(exported_model, quantizer)
735+
cast(Any, prepared_model).to(memory_format=torch.channels_last)
688736
prepared_model = move_exported_model_to_train(prepared_model)
689737
optimizer = torch.optim.SGD(prepared_model.parameters(), lr=lr, momentum=0.9)
690738

691739
for step in range(steps):
692740
total_loss = 0.0
693741
for inputs, target in training_samples:
694-
prediction = prepared_model(*inputs)
695-
loss = F.mse_loss(prediction, target)
742+
prediction = prepared_model(*inputs).contiguous(
743+
memory_format=torch.channels_last
744+
)
745+
loss = F.mse_loss(
746+
prediction,
747+
target.contiguous(memory_format=torch.channels_last),
748+
)
696749
optimizer.zero_grad()
697750
loss.backward()
698751
optimizer.step()
@@ -704,6 +757,7 @@ def build_qat_model(
704757

705758
prepared_model = move_exported_model_to_eval(prepared_model)
706759
converted_model = convert_pt2e(prepared_model)
760+
cast(Any, converted_model).to(memory_format=torch.channels_last)
707761
return QuantizedModel(
708762
model=converted_model,
709763
coverage=quantization_coverage(converted_model),
@@ -722,6 +776,11 @@ def export_vgf_artifact(
722776
artifact_dir.mkdir(parents=True, exist_ok=True)
723777
compile_spec = make_vgf_compile_spec(artifact_dir)
724778
partitioner = VgfPartitioner(compile_spec)
779+
cast(Any, model).to(memory_format=torch.channels_last)
780+
inputs = (
781+
inputs[0].contiguous(memory_format=torch.channels_last),
782+
inputs[1].contiguous(memory_format=torch.channels_last),
783+
)
725784

726785
print(f"[vgf] exporting {name} model to {artifact_dir} ...")
727786
aten_dialect = export(model, args=inputs, strict=True)
@@ -777,6 +836,10 @@ def write_markdown_report(
777836
for key, label in (
778837
("eager_psnr", "Eager PSNR"),
779838
("eager_ssim", "Eager SSIM"),
839+
("eager_channels_last_psnr", "Eager Channels-Last PSNR"),
840+
("eager_channels_last_ssim", "Eager Channels-Last SSIM"),
841+
("eager_channels_last_vs_eager_mae", "Eager Channels-Last vs eager MAE"),
842+
("eager_channels_last_vs_eager_sqnr", "Eager Channels-Last vs eager SQNR"),
780843
("ptq_psnr", "PTQ PSNR"),
781844
("ptq_ssim", "PTQ SSIM"),
782845
("ptq_vs_eager_mae", "PTQ vs eager MAE"),
@@ -1171,6 +1234,8 @@ def print_progress(index: int, total: int, row: dict[str, Any]) -> None:
11711234
progress = f"{index + 1}/{total} {row['name']}"
11721235
if "eager_psnr" in row:
11731236
progress += f" eager_psnr={row['eager_psnr']:.3f}"
1237+
if "eager_channels_last_psnr" in row:
1238+
progress += f" eager_cl_psnr={row['eager_channels_last_psnr']:.3f}"
11741239
if "ptq_psnr" in row:
11751240
progress += f" ptq_psnr={row['ptq_psnr']:.3f}"
11761241
if "qat_psnr" in row:
@@ -1181,7 +1246,8 @@ def print_progress(index: int, total: int, row: dict[str, Any]) -> None:
11811246
def evaluate_triples(
11821247
args: argparse.Namespace,
11831248
triples: list[FrameTriple],
1184-
model: torch.nn.Module,
1249+
eager_model: torch.nn.Module,
1250+
eager_channels_last_model: torch.nn.Module | None,
11851251
ptq_model: torch.nn.Module | None,
11861252
qat_model: torch.nn.Module | None,
11871253
) -> list[dict[str, Any]]:
@@ -1205,13 +1271,29 @@ def evaluate_triples(
12051271
}
12061272
eager_output = None
12071273
if run_eager:
1208-
eager_output = run_model(model, inputs)
1274+
eager_output = run_model(eager_model, inputs)
12091275
update_row_with_prediction_metrics(
12101276
row,
12111277
prefix="eager",
12121278
prediction=eager_output,
12131279
target=target,
12141280
)
1281+
if eager_channels_last_model is not None:
1282+
eager_channels_last_output = run_model(
1283+
eager_channels_last_model, inputs
1284+
)
1285+
update_row_with_prediction_metrics(
1286+
row,
1287+
prefix="eager_channels_last",
1288+
prediction=eager_channels_last_output,
1289+
target=target,
1290+
)
1291+
update_row_with_delta_metrics(
1292+
row,
1293+
prefix="eager_channels_last_vs_eager",
1294+
prediction=eager_channels_last_output,
1295+
reference=eager_output,
1296+
)
12151297

12161298
if ptq_model is not None:
12171299
ptq_output = run_model(ptq_model, inputs)
@@ -1222,7 +1304,7 @@ def evaluate_triples(
12221304
target=target,
12231305
)
12241306
if eager_output is None:
1225-
eager_output = run_model(model, inputs)
1307+
eager_output = run_model(eager_model, inputs)
12261308
update_row_with_delta_metrics(
12271309
row,
12281310
prefix="ptq_vs_eager",
@@ -1239,7 +1321,7 @@ def evaluate_triples(
12391321
target=target,
12401322
)
12411323
if eager_output is None:
1242-
eager_output = run_model(model, inputs)
1324+
eager_output = run_model(eager_model, inputs)
12431325
update_row_with_delta_metrics(
12441326
row,
12451327
prefix="qat_vs_eager",
@@ -1254,7 +1336,8 @@ def evaluate_triples(
12541336

12551337

12561338
def build_aggregate(
1257-
args: argparse.Namespace, rows: list[dict[str, Any]]
1339+
args: argparse.Namespace,
1340+
rows: list[dict[str, Any]],
12581341
) -> dict[str, Any]:
12591342
aggregate: dict[str, Any] = {
12601343
"triple_count": len(rows),
@@ -1266,7 +1349,15 @@ def build_aggregate(
12661349
"data_source": "random" if rows and rows[0].get("synthetic") else "frames",
12671350
"random_seed": args.random_seed,
12681351
}
1269-
for prefix in ("eager", "ptq", "ptq_vs_eager", "qat", "qat_vs_eager"):
1352+
for prefix in (
1353+
"eager",
1354+
"eager_channels_last",
1355+
"eager_channels_last_vs_eager",
1356+
"ptq",
1357+
"ptq_vs_eager",
1358+
"qat",
1359+
"qat_vs_eager",
1360+
):
12701361
aggregate.update(aggregate_metrics(rows, prefix))
12711362
return aggregate
12721363

@@ -1303,6 +1394,32 @@ def write_outputs(
13031394
print(f"Wrote {args.output_dir / 'report.md'}")
13041395

13051396

1397+
def print_summary(aggregate: dict[str, Any]) -> None:
1398+
print("")
1399+
print("== Summary ==")
1400+
for prefix, label in (
1401+
("eager", "eager"),
1402+
("eager_channels_last", "eager_channels_last"),
1403+
("ptq", "ptq"),
1404+
("qat", "qat"),
1405+
):
1406+
if f"{prefix}_psnr_mean" not in aggregate:
1407+
continue
1408+
summary = (
1409+
f"{label}: "
1410+
f"psnr={format_metric(aggregate.get(f'{prefix}_psnr_mean'))} "
1411+
f"ssim={format_metric(aggregate.get(f'{prefix}_ssim_mean'))}"
1412+
)
1413+
mae_key = f"{prefix}_vs_eager_mae_mean"
1414+
sqnr_key = f"{prefix}_vs_eager_sqnr_mean"
1415+
if mae_key in aggregate:
1416+
summary += (
1417+
f" vs_eager_mae={format_metric(aggregate.get(mae_key))}"
1418+
f" vs_eager_sqnr={format_metric(aggregate.get(sqnr_key))}"
1419+
)
1420+
print(summary)
1421+
1422+
13061423
def main() -> int:
13071424
args = parse_args()
13081425
validate_args(args)
@@ -1319,13 +1436,28 @@ def main() -> int:
13191436
print(f"Data source: {source}")
13201437

13211438
model = load_rife_model(args.model_root, args.checkpoint)
1439+
eager_channels_last_model = clone_model(model, channels_last=True)
13221440
ptq_model, qat_model, quantization_reports = build_requested_quantized_models(
13231441
args,
13241442
model,
13251443
triples,
13261444
)
1327-
rows = evaluate_triples(args, triples, model, ptq_model, qat_model)
1328-
write_outputs(args, build_aggregate(args, rows), rows, quantization_reports)
1445+
rows = evaluate_triples(
1446+
args,
1447+
triples,
1448+
model,
1449+
eager_channels_last_model,
1450+
ptq_model,
1451+
qat_model,
1452+
)
1453+
aggregate = build_aggregate(args, rows)
1454+
write_outputs(
1455+
args,
1456+
aggregate,
1457+
rows,
1458+
quantization_reports,
1459+
)
1460+
print_summary(aggregate)
13291461
export_requested_vgf_artifacts(args, triples, ptq_model, qat_model)
13301462
return 0
13311463

0 commit comments

Comments
 (0)