3333 ODHWI_ORDER ,
3434 OHWI_ORDER ,
3535)
36- from executorch .backends .arm .tosa .mapping import TosaSpecialDtype
36+ from executorch .backends .arm .tosa .mapping import (
37+ TOSA_CONTROL_FLOW_SOURCE_NODE_META ,
38+ TosaSpecialDtype ,
39+ )
3740from executorch .backends .arm .tosa .specification import get_context_shape_env
3841from executorch .backends .transforms .utils import create_constant_placeholder
3942from executorch .exir .dialects ._ops import ops as exir_ops
@@ -208,6 +211,111 @@ def _rewrite_fp8_bias(
208211 persistent_buffer = persistent_buffer ,
209212 )
210213
214+ def _resolve_control_flow_constant (
215+ self , node : torch .fx .Node
216+ ) -> tuple [torch .fx .Node , torch .Tensor , list [torch .fx .Node ]]:
217+ """Resolve a child input back to a constant in an enclosing graph."""
218+ source = node
219+ visited = set ()
220+ rescales = []
221+ while source not in visited :
222+ visited .add (source )
223+ # A captured constant can be requantized at every control-flow
224+ # boundary. Save those operations so the rewritten constant can
225+ # later be put through the same chain in its new layout.
226+ if source .target == exir_ops .backend .tosa .RESCALE .default :
227+ if not isinstance (source .args [0 ], torch .fx .Node ):
228+ raise RuntimeError (
229+ f"Rescale node { source .name } has no tensor input"
230+ )
231+ rescales .append (source )
232+ source = source .args [0 ]
233+ continue
234+ try :
235+ tensor = get_param_tensor (self .exported_program , source )
236+ except RuntimeError :
237+ # Branch placeholders are not constants themselves. The TOSA
238+ # backend records the corresponding operand in the parent
239+ # graph, which may itself be a placeholder in another branch.
240+ parent_source = source .meta .get (TOSA_CONTROL_FLOW_SOURCE_NODE_META )
241+ if not isinstance (parent_source , torch .fx .Node ):
242+ raise
243+ source = parent_source
244+ continue
245+ # get_param_tensor returns None only for a None input. Keep this
246+ # guard for its Optional return type; source is always an FX node.
247+ if tensor is None :
248+ raise RuntimeError (
249+ f"Node { source .name } is not a parameter, buffer, or constant"
250+ )
251+ return source , tensor , rescales
252+ raise RuntimeError (f"Cycle while resolving control-flow input { node .name } " )
253+
254+ def _rewrite_control_flow_bias (
255+ self ,
256+ graph_module : torch .fx .GraphModule ,
257+ conv_node : torch .fx .Node ,
258+ bias_node : torch .fx .Node ,
259+ input_node : torch .fx .Node ,
260+ weight_node : torch .fx .Node ,
261+ ) -> torch .fx .Node :
262+ """Create an INT32 bias from a rescaled constant control-flow input."""
263+ if bias_node .target != exir_ops .backend .tosa .RESCALE .default or not isinstance (
264+ bias_node .args [0 ], torch .fx .Node
265+ ):
266+ return bias_node
267+
268+ constant_source , quantized_bias , parent_rescales = (
269+ self ._resolve_control_flow_constant (bias_node .args [0 ])
270+ )
271+ conv_qparams = get_input_qparams (conv_node )
272+ input_qparams = conv_qparams [0 ]
273+ weight_qparams = conv_qparams [1 ]
274+ bias_qparams = conv_qparams [2 ]
275+
276+ # Recover the scale at which the captured bias was originally stored.
277+ # The rescales are discovered from the innermost graph outwards, hence
278+ # the reversal before composing their scale ratios.
279+ rescale_chain = [* reversed (parent_rescales ), bias_node ]
280+ boundary_scale = 1.0
281+ for rescale in rescale_chain :
282+ boundary_scales = cast (list [float ], rescale .args [2 ])
283+ boundary_scale *= boundary_scales [0 ]
284+ boundary_zp = cast (int , rescale_chain [0 ].args [3 ])
285+ source_scale = boundary_scale * bias_qparams .get_scale_per_tensor ()
286+ float_bias = (quantized_bias .to (torch .float32 ) - boundary_zp ) * source_scale
287+
288+ # TOSA convolution accumulates into INT32. Its bias therefore uses the
289+ # product of the activation and weight scales, rather than the scale
290+ # assigned to a captured control-flow operand by the quantizer.
291+ activation_scale = input_qparams .get_scale_per_tensor ()
292+ weight_scales = (
293+ torch .tensor (weight_qparams .get_scale_per_channel ())
294+ if weight_qparams .per_channel
295+ else torch .tensor (weight_qparams .get_scale_per_tensor ())
296+ )
297+ bias_scales = activation_scale * weight_scales
298+ rewritten_bias = torch .round (float_bias / bias_scales ).to (torch .int64 )
299+ rewritten_bias = rewritten_bias .clamp (
300+ torch .iinfo (torch .int32 ).min , torch .iinfo (torch .int32 ).max
301+ ).to (torch .int32 )
302+
303+ kind = get_constant_placeholder_kind (self .exported_program , constant_source )
304+ persistent_buffer = is_persistent_buffer (self .exported_program , constant_source )
305+ with graph_module .graph .inserting_after (bias_node ):
306+ rewritten_bias_node = create_constant_placeholder (
307+ self .exported_program ,
308+ graph = graph_module .graph ,
309+ name = f"{ conv_node .name } _bias_int32" ,
310+ kind = kind ,
311+ data = rewritten_bias ,
312+ persistent_buffer = persistent_buffer ,
313+ )
314+ # A16W8 stores the bias in an INT32 tensor but TOSA interprets it as
315+ # the INT48 accumulator type.
316+ self ._mark_bias_as_int48_if_needed (conv_node , rewritten_bias_node )
317+ return rewritten_bias_node
318+
211319 def _rewrite_weight (
212320 self ,
213321 graph_module : torch .fx .GraphModule ,
@@ -217,32 +325,73 @@ def _rewrite_weight(
217325 name_suffix : str ,
218326 reshape_dims : tuple [int , ...] | None = None ,
219327 ) -> torch .fx .Node :
220- """Create a convolution-local rewritten weight placeholder."""
221- weight_tensor = get_param_tensor (self .exported_program , weight_node ) # type: ignore[arg-type]
222- if weight_tensor is None :
223- raise RuntimeError (
224- f"Weight node { weight_node .name } is not a parameter or buffer"
225- )
328+ """Create a convolution-local weight in the layout required by TOSA."""
329+ source_node = weight_node
330+ rescale_node = None
331+ if (
332+ weight_node .target == exir_ops .edge .aten .view_copy .default
333+ and isinstance (weight_node .args [0 ], torch .fx .Node )
334+ and weight_node .args [0 ].target == exir_ops .backend .tosa .RESCALE .default
335+ ):
336+ rescale_node = weight_node .args [0 ]
337+ if not isinstance (rescale_node .args [0 ], torch .fx .Node ):
338+ raise RuntimeError (
339+ f"Rescaled weight node { rescale_node .name } has no tensor input"
340+ )
341+ source_node = rescale_node .args [0 ]
342+
343+ constant_source , weight_tensor , parent_rescales = (
344+ self ._resolve_control_flow_constant (source_node )
345+ )
226346
227- rewritten_weight = weight_tensor .permute (permute_dims )
347+ if rescale_node is not None :
348+ # RESCALE is elementwise, so reshape the constant before it instead
349+ # of trying to evaluate the quantized rescale during compilation.
350+ weight_shape = cast (list [int ], weight_node .args [1 ])
351+ weight_tensor = weight_tensor .reshape (weight_shape )
352+ rewritten_weight_value = weight_tensor .permute (permute_dims )
228353 if reshape_dims is not None :
229- rewritten_weight = rewritten_weight .reshape (* reshape_dims )
230- rewritten_weight = rewritten_weight .contiguous ()
231- kind = get_constant_placeholder_kind (self .exported_program , weight_node )
232- persistent_buffer = is_persistent_buffer (self .exported_program , weight_node )
354+ rewritten_weight_value = rewritten_weight_value .reshape (* reshape_dims )
355+ rewritten_weight_value = rewritten_weight_value .contiguous ()
356+ kind = get_constant_placeholder_kind (self .exported_program , constant_source )
357+ persistent_buffer = is_persistent_buffer (self .exported_program , constant_source )
233358
234359 with graph_module .graph .inserting_after (weight_node ):
235360 rewritten_weight_node = create_constant_placeholder (
236361 self .exported_program ,
237362 graph = graph_module .graph ,
238363 name = f"{ conv_node .name } _weight_{ name_suffix } " ,
239364 kind = kind ,
240- data = rewritten_weight ,
365+ data = rewritten_weight_value ,
241366 persistent_buffer = persistent_buffer ,
242367 )
243- if special_dtype := weight_node .meta .get (TosaSpecialDtype .meta_key ()):
368+ if special_dtype := source_node .meta .get (TosaSpecialDtype .meta_key ()):
244369 rewritten_weight_node .meta [TosaSpecialDtype .meta_key ()] = special_dtype
245- return rewritten_weight_node
370+
371+ # Layout changes commute with elementwise RESCALE operations. Rebuild
372+ # the saved outer-to-inner chain after the permuted constant so nested
373+ # branches retain exactly the quantization boundaries they started with.
374+ rescale_chain = [* reversed (parent_rescales )]
375+ if rescale_node is not None :
376+ rescale_chain .append (rescale_node )
377+ rewritten_weight : torch .fx .Node = rewritten_weight_node
378+ for original_rescale in rescale_chain :
379+ with graph_module .graph .inserting_after (rewritten_weight ):
380+ rewritten_rescale = create_node (
381+ graph = graph_module .graph ,
382+ op_target = exir_ops .backend .tosa .RESCALE .default ,
383+ args = (rewritten_weight , * original_rescale .args [1 :]),
384+ kwargs = original_rescale .kwargs ,
385+ from_node = original_rescale ,
386+ )
387+ rewritten_rescale .meta ["val" ] = exir_ops .backend .tosa .RESCALE .default (
388+ get_first_fake_tensor (rewritten_weight ),
389+ * original_rescale .args [1 :],
390+ ** original_rescale .kwargs ,
391+ )
392+ rewritten_weight = rewritten_rescale
393+
394+ return rewritten_weight
246395
247396 def _is_quantized_conv (self , node : torch .fx .Node ) -> bool :
248397 return bool (node .meta .get ("input_qparams" , {}))
@@ -280,21 +429,37 @@ def _get_effective_output_qparams(self, node: torch.fx.Node):
280429 )
281430
282431 activation = users [0 ]
432+ if activation .target == exir_ops .edge .aten .view_copy .default :
433+ view_output_qparams = activation .meta .get ("output_qparams" , {})
434+ if view_output_qparams :
435+ return view_output_qparams
283436 if activation .target == exir_ops .edge .aten .clamp .default :
284437 activation_output_qparams = activation .meta .get ("output_qparams" , {})
285438 if activation_output_qparams :
286439 return activation_output_qparams
287440
288441 return get_output_qparams (node )
289442
443+ def _get_effective_conv_input_qparams (self , node : torch .fx .Node ):
444+ """Return conv qparams, including those attached to input views."""
445+ input_qparams = dict (node .meta .get ("input_qparams" , {}))
446+ for index in (0 , 1 ):
447+ arg = node .args [index ]
448+ if index in input_qparams or not isinstance (arg , torch .fx .Node ):
449+ continue
450+ arg_qparams = arg .meta .get ("input_qparams" , {})
451+ if 0 in arg_qparams :
452+ input_qparams [index ] = arg_qparams [0 ]
453+ return input_qparams
454+
290455 def insert_output_rescale (
291456 self ,
292457 graph_module ,
293458 source_node ,
294459 conv_node ,
295460 conv_fake_tensor : torch .Tensor ,
296461 ):
297- input_qparams = get_input_qparams (source_node )
462+ input_qparams = self . _get_effective_conv_input_qparams (source_node )
298463 output_qparams = self ._get_effective_output_qparams (source_node )[0 ]
299464 weight_qparams = input_qparams [1 ]
300465 input_qparams = input_qparams [0 ]
@@ -643,6 +808,11 @@ def call(self, graph_module: torch.fx.GraphModule) -> PassResult: # noqa: C901
643808 group ,
644809 ) = node .args
645810
811+ if self ._is_quantized_conv (node ):
812+ node .meta ["input_qparams" ] = self ._get_effective_conv_input_qparams (
813+ node
814+ )
815+
646816 input_fake_tensor = get_first_fake_tensor (x )
647817 weight_fake_tensor = get_first_fake_tensor (weight )
648818 input_shape = input_fake_tensor .shape
@@ -660,6 +830,13 @@ def call(self, graph_module: torch.fx.GraphModule) -> PassResult: # noqa: C901
660830 elif isinstance (bias , torch .fx .Node ):
661831 if input_fake_tensor .dtype in self ._FP8_DTYPES :
662832 bias = self ._rewrite_fp8_bias (graph_module , node , bias )
833+ elif (
834+ self ._is_quantized_conv (node )
835+ and get_first_fake_tensor (bias ).dtype != torch .int32
836+ ):
837+ bias = self ._rewrite_control_flow_bias (
838+ graph_module , node , bias , x , weight
839+ )
663840 else :
664841 self ._mark_bias_as_int48_if_needed (node , bias )
665842
0 commit comments