Skip to content

Commit a1eee71

Browse files
authored
Support extra quantizable ops in advance quant above def chain (pytorch#21757)
Differential Revision: D115632906 Pull Request resolved: pytorch#21757
1 parent 8bd712d commit a1eee71

2 files changed

Lines changed: 139 additions & 1 deletion

File tree

‎backends/cadence/aot/reorder_ops.py‎

Lines changed: 72 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -258,13 +258,21 @@ class AdvanceQuantizeOpAboveDefChainPass(ExportPass):
258258
input individually. A later pass can clean up any redundant
259259
dequant-quant pairs on the inputs.
260260
261+
3. Caller-supplied ops: advance the quantize above a value-preserving op by
262+
quantizing selected direct floating-point tensor inputs. The caller is
263+
responsible for supplying only inputs that support the quantized dtype.
264+
261265
For the cat case, SplitDequantizedCatPass should run first to ensure
262266
each cat has at most one quantize consumer.
263267
"""
264268

265-
def __init__(self):
269+
def __init__(
270+
self,
271+
extra_quantizable_ops: dict[EdgeOpOverload, tuple[int, ...]] | None = None,
272+
) -> None:
266273
super().__init__()
267274
self.graph_module = None
275+
self._extra_quantizable_ops = extra_quantizable_ops or {}
268276

269277
# Return true if advancing the quantize node is feasible
270278
def advancing_feasible(self, quant_node: torch.fx.Node):
@@ -349,6 +357,56 @@ def _advance_above_cat(
349357
quant_node.replace_all_uses_with(new_cat)
350358
graph.erase_node(quant_node)
351359

360+
def _advance_above_extra_quantizable_op(
361+
self,
362+
quant_node: torch.fx.Node,
363+
op_node: torch.fx.Node,
364+
quantizable_input_indices: tuple[int, ...],
365+
) -> bool:
366+
graph = quant_node.graph
367+
new_op_args = list(op_node.args)
368+
quantized_input = False
369+
370+
for index in quantizable_input_indices:
371+
assert 0 <= index < len(op_node.args)
372+
arg = op_node.args[index]
373+
assert isinstance(arg, torch.fx.Node)
374+
value = arg.meta["val"]
375+
assert isinstance(value, torch.Tensor)
376+
if not value.dtype.is_floating_point:
377+
continue
378+
379+
quant_args = list(quant_node.args)
380+
quant_args[0] = arg
381+
with graph.inserting_before(op_node):
382+
new_quant = graph.call_function(
383+
# pyre-ignore[6]
384+
quant_node.target,
385+
args=tuple(quant_args),
386+
kwargs=quant_node.kwargs,
387+
)
388+
# We will correct the dtype when we run
389+
# ExportPass call at the end.
390+
new_quant.meta = arg.meta.copy()
391+
new_op_args[index] = new_quant
392+
quantized_input = True
393+
394+
if not quantized_input:
395+
return False
396+
397+
with graph.inserting_before(quant_node):
398+
new_op = graph.call_function(
399+
# pyre-ignore[6]
400+
op_node.target,
401+
args=tuple(new_op_args),
402+
kwargs=op_node.kwargs,
403+
)
404+
new_op.meta = quant_node.meta.copy()
405+
406+
quant_node.replace_all_uses_with(new_op)
407+
graph.erase_node(quant_node)
408+
return True
409+
352410
def advance_quantize_op(self, graph_module: torch.fx.GraphModule) -> bool:
353411
graph = graph_module.graph
354412
modified = False
@@ -372,6 +430,19 @@ def advance_quantize_op(self, graph_module: torch.fx.GraphModule) -> bool:
372430
modified = True
373431
continue
374432

433+
if (
434+
isinstance(inp, torch.fx.Node)
435+
and inp.target in self._extra_quantizable_ops
436+
and len(inp.users) == 1
437+
and self._advance_above_extra_quantizable_op(
438+
node,
439+
inp,
440+
self._extra_quantizable_ops[inp.target],
441+
)
442+
):
443+
modified = True
444+
continue
445+
375446
if not self.advancing_feasible(node):
376447
continue
377448

‎backends/cadence/aot/tests/test_reorder_ops_passes.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,73 @@ def test_advance_quantize(self) -> None:
359359
< get_node_pos(converted_graph, exir_ops.edge.aten.permute_copy.default)
360360
)
361361

362+
def test_advance_quantize_above_extra_quantizable_op(self) -> None:
363+
builder = GraphBuilder()
364+
cache_data = torch.randint(-128, 127, (4, 8), dtype=torch.int8)
365+
update_data = torch.randint(-128, 127, (2, 8), dtype=torch.int8)
366+
cache = builder.placeholder("cache", cache_data)
367+
update = builder.placeholder("update", update_data)
368+
qparams = (0.25, 3, -128, 127, torch.int8)
369+
cache_float = builder.call_operator(
370+
op=exir_ops.edge.quantized_decomposed.dequantize_per_tensor.default,
371+
args=(cache, *qparams),
372+
)
373+
update_float = builder.call_operator(
374+
op=exir_ops.edge.quantized_decomposed.dequantize_per_tensor.default,
375+
args=(update, *qparams),
376+
)
377+
slice_scatter = builder.call_operator(
378+
op=exir_ops.edge.aten.slice_scatter.default,
379+
args=(cache_float, update_float, 0, 1, 3, 1),
380+
)
381+
quantized = builder.call_operator(
382+
op=exir_ops.edge.quantized_decomposed.quantize_per_tensor.default,
383+
args=(slice_scatter, *qparams),
384+
)
385+
builder.output([quantized])
386+
387+
result = transform_and_check_numerics(
388+
builder.get_graph_module(),
389+
(cache_data, update_data),
390+
AdvanceQuantizeOpAboveDefChainPass(
391+
extra_quantizable_ops={exir_ops.edge.aten.slice_scatter.default: (0, 1)}
392+
),
393+
)
394+
395+
self.assertTrue(result.modified)
396+
self.assertEqual(
397+
2,
398+
count_node(
399+
result.graph_module,
400+
exir_ops.edge.quantized_decomposed.quantize_per_tensor.default,
401+
),
402+
)
403+
scatter_nodes = result.graph_module.graph.find_nodes(
404+
op="call_function", target=exir_ops.edge.aten.slice_scatter.default
405+
)
406+
self.assertEqual(1, len(scatter_nodes))
407+
scatter_node = scatter_nodes[0]
408+
self.assertEqual((0, 1, 3, 1), scatter_node.args[2:])
409+
self.assertEqual(torch.int8, scatter_node.meta["val"].dtype)
410+
411+
for scatter_input in scatter_node.args[:2]:
412+
self.assertIsInstance(scatter_input, torch.fx.Node)
413+
scatter_input = cast(torch.fx.Node, scatter_input)
414+
self.assertEqual(
415+
exir_ops.edge.quantized_decomposed.quantize_per_tensor.default,
416+
scatter_input.target,
417+
)
418+
self.assertEqual(torch.int8, scatter_input.meta["val"].dtype)
419+
420+
quant_input = scatter_input.args[0]
421+
self.assertIsInstance(quant_input, torch.fx.Node)
422+
quant_input = cast(torch.fx.Node, quant_input)
423+
self.assertEqual(torch.float32, quant_input.meta["val"].dtype)
424+
dequant_input = quant_input.args[0]
425+
self.assertIsInstance(dequant_input, torch.fx.Node)
426+
dequant_input = cast(torch.fx.Node, dequant_input)
427+
self.assertEqual(torch.int8, dequant_input.meta["val"].dtype)
428+
362429
def test_postpone_dequantize1(self) -> None:
363430
builder = GraphBuilder()
364431
x_data = torch.randn(1, 16, 32, 6, dtype=torch.float32)

0 commit comments

Comments
 (0)