@@ -369,3 +369,216 @@ def call(self, graph_module: GraphModule) -> PassResult:
369369 graph_module .recompile ()
370370
371371 return PassResult (graph_module , modified )
372+
373+
374+ class InsertControlFlowRescalesPass (ArmPass ):
375+ """The quantization parameters for tensors going into and coming out of a submodule are not guaranteed to
376+ match the quantization parameters for the corresponding tensors inside the submodule. For example, cond has
377+ different annotation on input and output, while the entire graph inside the submodule could be using shared
378+ annotation. This pass solves this by inserting rescales in the beginning and end of the submodule
379+ that transform the tensor from one set of quantization parameters to another.
380+
381+ The pass is run by the graph_module containing the control flow operator, but requires that the affected nodes
382+ inside the submodule have been q-dq folded and have input/output_qparams meta.
383+ """
384+
385+ _passes_required_after : Set [Type [ExportPass ]] = set ()
386+
387+ def _get_input_nodes (self , graph_module : GraphModule ):
388+ return [node for node in graph_module .graph .nodes if node .op == "placeholder" ]
389+
390+ def _insert_rescale (
391+ self ,
392+ in_qparams : QuantArgs ,
393+ out_qparams : QuantArgs ,
394+ from_node : Node ,
395+ graph_module : GraphModule ,
396+ ):
397+ """Insert a rescale into the graph, inheriting meta from `from_node`.
398+ The node is not connected to anything, that is up to the user."""
399+
400+ new_scales = [
401+ in_qparams .get_scale_per_tensor () / out_qparams .get_scale_per_tensor ()
402+ ]
403+
404+ rescale_node = create_node (
405+ graph_module .graph ,
406+ exir_ops .backend .tosa .RESCALE .default ,
407+ (
408+ None ,
409+ out_qparams .dtype ,
410+ new_scales ,
411+ in_qparams .get_zp_per_tensor (), # Old zero point
412+ out_qparams .get_zp_per_tensor (), # New zero point
413+ ),
414+ from_node = from_node ,
415+ )
416+ return rescale_node
417+
418+ def _rescale_submodule_inputs (
419+ self , submodule : GraphModule , input_qparams_map : Dict [int , QuantArgs ]
420+ ) -> bool :
421+ """Insert rescales at the inputs of `submodule` to match the qparams outside the submodule.
422+ Matching the correct qparams gets a bit tricky:
423+ Containing module: | submodule:
424+ ops => cond | => placeholders => ...
425+
426+ The dq->q qparam pair we want to convert to a rescale is:
427+ (input qparams of op, output qparams of placeholder)
428+ And the rescale is inserted after the placeholder.
429+
430+ Args:
431+ submodule: GraphModule: the GraphModule in which to rescale the inputs.
432+ input_qparams_map: A map of input indexes mapping to QuantArgs. Not guaranteed to contain a mapping
433+ for every submodule input.
434+ Returns:
435+ True if at least one rescale was inserted, False otherwise.
436+ """
437+
438+ modified = False
439+ input_nodes = self ._get_input_nodes (submodule )
440+ for qargs_index in input_qparams_map :
441+ input_node = input_nodes [qargs_index ]
442+ if len (input_node .users ) == 0 :
443+ continue
444+ if len (out_qparams_map := input_node .meta .get ("output_qparams" , {})) != 1 :
445+ raise ValueError (
446+ f"Expected submodule input { input_node } to have exactly one output qparam, got { out_qparams_map } "
447+ )
448+ in_qparams = input_qparams_map [qargs_index ]
449+ out_qparams = cast (QuantArgs , out_qparams_map [0 ])
450+
451+ # Remove qparam meta to not confuse folding pass.
452+ del input_node .meta ["output_qparams" ]
453+ if in_qparams == out_qparams :
454+ continue
455+ with submodule .graph .inserting_after (input_node ):
456+ modified = True
457+ rescale_node = self ._insert_rescale (
458+ in_qparams , out_qparams , input_node , submodule
459+ )
460+ input_node .replace_all_uses_with (replace_with = rescale_node )
461+ rescale_node .update_arg (0 , input_node )
462+ return modified
463+
464+ def _rescale_submodule_outputs (
465+ self , submodule : GraphModule , output_qparams_map : Dict [int , QuantArgs ]
466+ ) -> bool :
467+ """Insert rescales at the outputs of `submodule` to match the qparams outside the submodule.
468+ Matching the correct qparams gets a bit tricky:
469+ Submodule: | Containing module:
470+ output_nodes => output |=> getitems => ...
471+
472+ The dq->q qparam pair we want to convert to a rescale is:
473+ (input qparam of output_node, output qparam of getitem)
474+ And the rescale is inserted between op and output. Note that the output qparam of op is called input_qargs,
475+ since the it is the input to the dq-q pair.
476+
477+ Args:
478+ submodule: GraphModule: the GraphModule in which to rescale the outputs.
479+ output_qparams_map: A map of output indexes mapping to QuantArgs. Not guaranteed to contain a mapping
480+ for every submodule output.
481+ Returns:
482+ True if at least one rescale was inserted, False otherwise.
483+ """
484+
485+ modified = False
486+ output_node = submodule .graph .output_node ()
487+ output_args = list (cast (tuple [Node ], output_node .args [0 ]))
488+ input_qparams_map = cast (
489+ dict [int , QuantArgs ], output_node .meta ["input_qparams" ]
490+ )
491+ for qargs_index in output_qparams_map :
492+ output_arg_node = output_args [qargs_index ]
493+ in_qparams = input_qparams_map [qargs_index ]
494+ out_qparams = output_qparams_map [qargs_index ]
495+ if in_qparams == out_qparams :
496+ continue
497+ with submodule .graph .inserting_before (output_node ):
498+ modified = True
499+ rescale_node = self ._insert_rescale (
500+ in_qparams , out_qparams , output_arg_node , submodule
501+ )
502+ output_args [qargs_index ] = rescale_node
503+ rescale_node .update_arg (0 , output_arg_node )
504+ output_node .update_arg (0 , tuple (output_args ))
505+ # Remove qparam meta to not confuse folding pass.
506+ del output_node .meta ["input_qparams" ]
507+ return modified
508+
509+ def _get_input_qparams_map (self , node : Node , idx : int ):
510+ input_qparams_meta = cast (
511+ dict [int , QuantArgs ], node .meta .get ("input_qparams" , None )
512+ )
513+ if input_qparams_meta :
514+ input_qparams = cast (QuantArgs , input_qparams_meta .get (idx , None ))
515+ if not input_qparams :
516+ raise ValueError (
517+ f"Expected entry with key { idx } in input_qparams meta, got { input_qparams_meta } "
518+ )
519+ num_inputs = len (cast (list , node .args [idx ]))
520+
521+ # Currently, infra only supports one set of qparams for a list of inputs
522+ # Map all inputs to the same qparams.
523+ input_qparams_map = {i : input_qparams for i in range (num_inputs )}
524+ return input_qparams_map
525+ return None
526+
527+ def _get_output_qparams_map (self , node : Node ):
528+ output_qparams_map : dict [int , QuantArgs ] = {}
529+ for getitem_node in node .users :
530+ idx = cast (int , getitem_node .args [1 ])
531+ qparam = getitem_node .meta .get ("output_qparams" , None )
532+ if qparam :
533+ output_qparams_map [idx ] = cast (QuantArgs , qparam [0 ])
534+ return output_qparams_map
535+
536+ def _rescale_cond_submodules (self , node : Node , graph_module : GraphModule ) -> bool :
537+ modified = False
538+ if_graph : GraphModule = cast (GraphModule , graph_module .get_submodule (node .args [1 ].target )) # type: ignore
539+ else_graph : GraphModule = cast (GraphModule , graph_module .get_submodule (node .args [2 ].target )) # type: ignore
540+ input_qparams_map = self ._get_input_qparams_map (node , 3 )
541+ if input_qparams_map :
542+ modified |= self ._rescale_submodule_inputs (if_graph , input_qparams_map )
543+ modified |= self ._rescale_submodule_inputs (else_graph , input_qparams_map )
544+
545+ output_qparams_map = self ._get_output_qparams_map (node )
546+ if output_qparams_map :
547+ modified |= self ._rescale_submodule_outputs (if_graph , output_qparams_map )
548+ modified |= self ._rescale_submodule_outputs (else_graph , output_qparams_map )
549+ return modified
550+
551+ def _rescale_while_submodules (self , node : Node , graph_module : GraphModule ):
552+ modified = False
553+ cond_graph : GraphModule = cast (GraphModule , graph_module .get_submodule (node .args [0 ].target )) # type: ignore
554+ body_graph : GraphModule = cast (GraphModule , graph_module .get_submodule (node .args [1 ].target )) # type: ignore
555+
556+ input_qparams_map = self ._get_input_qparams_map (node , 2 )
557+ if input_qparams_map :
558+ modified |= self ._rescale_submodule_inputs (cond_graph , input_qparams_map )
559+ modified |= self ._rescale_submodule_inputs (body_graph , input_qparams_map )
560+
561+ output_qparams_map = self ._get_output_qparams_map (node )
562+ if output_qparams_map :
563+ modified |= self ._rescale_submodule_outputs (body_graph , output_qparams_map )
564+ return modified
565+
566+ def call (self , graph_module : GraphModule ) -> PassResult :
567+ modified = False
568+
569+ for node in list (graph_module .graph .nodes ):
570+ node = cast (Node , node )
571+ if node .op != "call_function" :
572+ continue
573+
574+ if node .target == torch .ops .higher_order .cond :
575+ modified = self ._rescale_cond_submodules (node , graph_module )
576+ if node .target == torch .ops .higher_order .while_loop :
577+ modified = self ._rescale_while_submodules (node , graph_module )
578+
579+ if modified :
580+ # Retrace the graph to update the fake tensor types
581+ graph_module = super ().call (graph_module ).graph_module
582+ graph_module .recompile ()
583+
584+ return PassResult (graph_module , modified )
0 commit comments