@@ -287,54 +287,37 @@ def get_16a8w_qnn_ptq_config(
287287 return quantization_config
288288
289289
290- def get_16a8w_qnn_qat_config (
290+ def get_16a16w_qnn_ptq_config (
291291 act_observer = MovingAverageMinMaxObserver , eps : float = None
292292) -> QuantizationConfig :
293293 # the smallest defaults to DEFAULT_EPS_16BIT
294294 extra_args : Dict [str , Any ] = {"eps" : eps if eps else DEFAULT_EPS_16BIT }
295- act_fake_quant_ctr = FusedMovingAvgObsFakeQuantize .with_args (
296- dtype = torch .int32 ,
297- quant_min = torch .iinfo (torch .uint16 ).min ,
298- quant_max = torch .iinfo (torch .uint16 ).max ,
299- qscheme = torch .per_tensor_affine ,
300- observer = act_observer .with_args (** extra_args ),
301- )
302295 act_quantization_spec = QuantizationSpec (
303296 dtype = torch .int32 ,
304297 quant_min = torch .iinfo (torch .uint16 ).min ,
305298 quant_max = torch .iinfo (torch .uint16 ).max ,
306299 qscheme = torch .per_tensor_affine ,
307- observer_or_fake_quant_ctr = act_fake_quant_ctr ,
308- )
309- weight_fake_quant_ctr = FusedMovingAvgObsFakeQuantize .with_args (
310- dtype = torch .int8 ,
311- quant_min = torch .iinfo (torch .int8 ).min + 1 ,
312- quant_max = torch .iinfo (torch .int8 ).max ,
313- qscheme = torch .per_tensor_symmetric ,
314- observer = MovingAverageMinMaxObserver ,
300+ observer_or_fake_quant_ctr = act_observer .with_args (** extra_args ),
315301 )
302+
316303 weight_quantization_spec = QuantizationSpec (
317- dtype = torch .int8 ,
318- quant_min = torch .iinfo (torch .int8 ).min + 1 ,
319- quant_max = torch .iinfo (torch .int8 ).max ,
304+ dtype = torch .int16 ,
305+ quant_min = torch .iinfo (torch .int16 ).min + 1 ,
306+ quant_max = torch .iinfo (torch .int16 ).max ,
320307 qscheme = torch .per_tensor_symmetric ,
321308 ch_axis = 0 ,
322- observer_or_fake_quant_ctr = weight_fake_quant_ctr ,
323- )
324- bias_fake_quant_ctr = FakeQuantize .with_args (
325- dtype = torch .int32 ,
326- quant_min = torch .iinfo (torch .int32 ).min ,
327- quant_max = torch .iinfo (torch .int32 ).max ,
328- qscheme = torch .per_tensor_symmetric ,
329- observer = MovingAverageMinMaxObserver ,
309+ observer_or_fake_quant_ctr = MinMaxObserver .with_args (** extra_args ),
330310 )
311+
312+ # torch does not support uint16 quantization, use int32 to bypass
331313 bias_quantization_spec = QuantizationSpec (
332314 dtype = torch .int32 ,
333315 quant_min = torch .iinfo (torch .int32 ).min ,
334316 quant_max = torch .iinfo (torch .int32 ).max ,
335317 qscheme = torch .per_tensor_symmetric ,
336- observer_or_fake_quant_ctr = bias_fake_quant_ctr ,
318+ observer_or_fake_quant_ctr = MinMaxObserver . with_args ( ** extra_args ) ,
337319 )
320+
338321 quantization_config = QuantizationConfig (
339322 input_activation = act_quantization_spec ,
340323 output_activation = act_quantization_spec ,
@@ -345,35 +328,56 @@ def get_16a8w_qnn_qat_config(
345328 return quantization_config
346329
347330
348- def get_16a16w_qnn_ptq_config (
331+ # TODO merge qat and ptq to a function, and use a bool flag to control it
332+ def get_16a8w_qnn_qat_config (
349333 act_observer = MovingAverageMinMaxObserver , eps : float = None
350334) -> QuantizationConfig :
351335 # the smallest defaults to DEFAULT_EPS_16BIT
352336 extra_args : Dict [str , Any ] = {"eps" : eps if eps else DEFAULT_EPS_16BIT }
337+ act_fake_quant_ctr = FusedMovingAvgObsFakeQuantize .with_args (
338+ dtype = torch .int32 ,
339+ quant_min = torch .iinfo (torch .uint16 ).min ,
340+ quant_max = torch .iinfo (torch .uint16 ).max ,
341+ qscheme = torch .per_tensor_affine ,
342+ observer = act_observer .with_args (** extra_args ),
343+ )
353344 act_quantization_spec = QuantizationSpec (
354345 dtype = torch .int32 ,
355346 quant_min = torch .iinfo (torch .uint16 ).min ,
356347 quant_max = torch .iinfo (torch .uint16 ).max ,
357348 qscheme = torch .per_tensor_affine ,
358- observer_or_fake_quant_ctr = act_observer . with_args ( ** extra_args ) ,
349+ observer_or_fake_quant_ctr = act_fake_quant_ctr ,
359350 )
360351
352+ weight_fake_quant_ctr = FusedMovingAvgObsFakeQuantize .with_args (
353+ dtype = torch .int8 ,
354+ quant_min = torch .iinfo (torch .int8 ).min + 1 ,
355+ quant_max = torch .iinfo (torch .int8 ).max ,
356+ qscheme = torch .per_tensor_symmetric ,
357+ observer = MovingAverageMinMaxObserver .with_args (** extra_args ),
358+ )
361359 weight_quantization_spec = QuantizationSpec (
362- dtype = torch .int16 ,
363- quant_min = torch .iinfo (torch .int16 ).min + 1 ,
364- quant_max = torch .iinfo (torch .int16 ).max ,
360+ dtype = torch .int8 ,
361+ quant_min = torch .iinfo (torch .int8 ).min + 1 ,
362+ quant_max = torch .iinfo (torch .int8 ).max ,
365363 qscheme = torch .per_tensor_symmetric ,
366364 ch_axis = 0 ,
367- observer_or_fake_quant_ctr = MinMaxObserver . with_args ( ** extra_args ) ,
365+ observer_or_fake_quant_ctr = weight_fake_quant_ctr ,
368366 )
369367
370- # torch does not support uint16 quantization, use int32 to bypass
368+ bias_fake_quant_ctr = FakeQuantize .with_args (
369+ dtype = torch .int32 ,
370+ quant_min = torch .iinfo (torch .int32 ).min ,
371+ quant_max = torch .iinfo (torch .int32 ).max ,
372+ qscheme = torch .per_tensor_symmetric ,
373+ observer = MovingAverageMinMaxObserver .with_args (** extra_args ),
374+ )
371375 bias_quantization_spec = QuantizationSpec (
372376 dtype = torch .int32 ,
373377 quant_min = torch .iinfo (torch .int32 ).min ,
374378 quant_max = torch .iinfo (torch .int32 ).max ,
375379 qscheme = torch .per_tensor_symmetric ,
376- observer_or_fake_quant_ctr = MinMaxObserver . with_args ( ** extra_args ) ,
380+ observer_or_fake_quant_ctr = bias_fake_quant_ctr ,
377381 )
378382
379383 quantization_config = QuantizationConfig (
@@ -580,14 +584,18 @@ def get_qat_per_block_quant_config(
580584
581585# TODO merge qat and ptq to a function, and use a bool flag to control it
582586def get_8a8w_qnn_qat_config (
583- act_symmetric : bool = False , act_observer = MovingAverageMinMaxObserver
587+ act_symmetric : bool = False ,
588+ act_observer = MovingAverageMinMaxObserver ,
589+ eps : float = None ,
584590) -> QuantizationConfig :
591+ # the smallest scale defaults to DEFAULT_EPS_8BIT
592+ extra_args : Dict [str , Any ] = {"eps" : eps if eps else DEFAULT_EPS_8BIT }
585593 act_fake_quant_ctr = FusedMovingAvgObsFakeQuantize .with_args (
586594 dtype = torch .uint8 ,
587595 qscheme = (
588596 torch .per_tensor_symmetric if act_symmetric else torch .per_tensor_affine
589597 ),
590- observer = act_observer ,
598+ observer = act_observer . with_args ( ** extra_args ) ,
591599 )
592600 act_quantization_spec = QuantizationSpec (
593601 dtype = torch .uint8 ,
@@ -603,7 +611,7 @@ def get_8a8w_qnn_qat_config(
603611 quant_min = torch .iinfo (torch .int8 ).min + 1 ,
604612 quant_max = torch .iinfo (torch .int8 ).max ,
605613 qscheme = torch .per_tensor_symmetric ,
606- observer = MovingAverageMinMaxObserver ,
614+ observer = MovingAverageMinMaxObserver . with_args ( ** extra_args ) ,
607615 )
608616 weight_quantization_spec = QuantizationSpec (
609617 dtype = torch .int8 ,
@@ -619,7 +627,7 @@ def get_8a8w_qnn_qat_config(
619627 quant_min = torch .iinfo (torch .int32 ).min ,
620628 quant_max = torch .iinfo (torch .int32 ).max ,
621629 qscheme = torch .per_tensor_symmetric ,
622- observer = MovingAverageMinMaxObserver ,
630+ observer = MovingAverageMinMaxObserver . with_args ( ** extra_args ) ,
623631 )
624632 bias_quantization_spec = QuantizationSpec (
625633 dtype = torch .int32 ,
@@ -640,14 +648,16 @@ def get_8a8w_qnn_qat_config(
640648
641649
642650def get_16a4w_qnn_qat_config (
643- act_observer = MovingAverageMinMaxObserver ,
651+ act_observer = MovingAverageMinMaxObserver , eps : float = None
644652) -> QuantizationConfig :
653+ # the smallest defaults to DEFAULT_EPS_16BIT
654+ extra_args : Dict [str , Any ] = {"eps" : eps if eps else DEFAULT_EPS_16BIT }
645655 act_fake_quant_ctr = FusedMovingAvgObsFakeQuantize .with_args (
646656 dtype = torch .int32 ,
647657 quant_min = torch .iinfo (torch .uint16 ).min ,
648658 quant_max = torch .iinfo (torch .uint16 ).max ,
649659 qscheme = torch .per_tensor_affine ,
650- observer = act_observer ,
660+ observer = act_observer . with_args ( ** extra_args ) ,
651661 )
652662 act_quantization_spec = QuantizationSpec (
653663 dtype = torch .int32 ,
@@ -663,7 +673,7 @@ def get_16a4w_qnn_qat_config(
663673 quant_max = 7 ,
664674 qscheme = torch .per_tensor_symmetric ,
665675 ch_axis = 0 ,
666- observer = MovingAverageMinMaxObserver ,
676+ observer = MovingAverageMinMaxObserver . with_args ( ** extra_args ) ,
667677 )
668678 weight_quantization_spec = QuantizationSpec (
669679 dtype = torch .int8 ,
@@ -679,7 +689,7 @@ def get_16a4w_qnn_qat_config(
679689 quant_min = torch .iinfo (torch .int32 ).min ,
680690 quant_max = torch .iinfo (torch .int32 ).max ,
681691 qscheme = torch .per_tensor_symmetric ,
682- observer = MovingAverageMinMaxObserver ,
692+ observer = MovingAverageMinMaxObserver . with_args ( ** extra_args ) ,
683693 )
684694 bias_quantization_spec = QuantizationSpec (
685695 dtype = torch .int32 ,
@@ -705,7 +715,13 @@ def get_qat_per_channel_quant_config(
705715 act_observer = MovingAverageMinMaxObserver ,
706716 act_symmetric = False ,
707717 ch_axis : int = 0 ,
718+ eps : float = None ,
708719) -> QuantizationConfig :
720+ # Use appropriate eps based on activation dtype
721+ if act_dtype in {torch .uint8 , torch .int8 }:
722+ extra_args : Dict [str , Any ] = {"eps" : eps if eps else DEFAULT_EPS_8BIT }
723+ else :
724+ extra_args : Dict [str , Any ] = {"eps" : eps if eps else DEFAULT_EPS_16BIT }
709725 supported_act_types = {
710726 torch .uint8 ,
711727 torch .uint16 ,
@@ -729,7 +745,7 @@ def get_qat_per_channel_quant_config(
729745 act_fake_quant_ctr = FusedMovingAvgObsFakeQuantize .with_args (
730746 dtype = torch .int32 if act_dtype == torch .uint16 else act_dtype ,
731747 qscheme = torch .per_tensor_symmetric ,
732- observer = act_observer ,
748+ observer = act_observer . with_args ( ** extra_args ) ,
733749 )
734750 act_quantization_spec = QuantizationSpec (
735751 dtype = torch .int32 if act_dtype == torch .uint16 else act_dtype ,
@@ -743,7 +759,7 @@ def get_qat_per_channel_quant_config(
743759 quant_min = torch .iinfo (act_dtype ).min ,
744760 quant_max = torch .iinfo (act_dtype ).max ,
745761 qscheme = torch .per_tensor_affine ,
746- observer = act_observer ,
762+ observer = act_observer . with_args ( ** extra_args ) ,
747763 )
748764 act_quantization_spec = QuantizationSpec (
749765 dtype = torch .int32 if act_dtype == torch .uint16 else act_dtype ,
@@ -761,7 +777,7 @@ def get_qat_per_channel_quant_config(
761777 quant_max = 7 if weight_dtype == torch .int4 else torch .iinfo (weight_dtype ).max ,
762778 qscheme = torch .per_channel_symmetric ,
763779 ch_axis = ch_axis ,
764- observer = MovingAveragePerChannelMinMaxObserver ,
780+ observer = MovingAveragePerChannelMinMaxObserver . with_args ( ** extra_args ) ,
765781 )
766782 weight_quantization_spec = QuantizationSpec (
767783 dtype = torch .int8 if weight_dtype == torch .int4 else weight_dtype ,
0 commit comments