Skip to content

Commit 888d5b9

Browse files
authored
Add bf16 enable flag to LLM Config and thread through export (pytorch#21981)
Differential Revision: D116814175 Pull Request resolved: pytorch#21981
1 parent 7f4b8ad commit 888d5b9

4 files changed

Lines changed: 40 additions & 2 deletions

File tree

‎examples/models/llama/export_llama_lib.py‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -458,6 +458,11 @@ def build_args_parser() -> argparse.ArgumentParser:
458458
action="store_true",
459459
help="Delegate more operators beyond DQLinear to the xnnpack backend. Requires -X or --xnnpack to be set.",
460460
)
461+
parser.add_argument(
462+
"--xnnpack-enable-bf16",
463+
action="store_true",
464+
help="Delegate BF16 operators to XNNPACK. Requires runtime hardware support.",
465+
)
461466
parser.add_argument(
462467
"--use-torchao-kernels",
463468
action="store_true",
@@ -1569,11 +1574,17 @@ def _get_xnnpack_partitioners(llm_config: LlmConfig) -> Optional[List[Partitione
15691574
# both xnnpack and xnnpack_extended_ops are enabled.
15701575
if llm_config.backend.xnnpack.enabled:
15711576
partitioners.append(
1572-
get_xnnpack_partitioner(dynamic_quant_only_partitioner=True)
1577+
get_xnnpack_partitioner(
1578+
dynamic_quant_only_partitioner=True,
1579+
enable_bf16=llm_config.backend.xnnpack.enable_bf16,
1580+
)
15731581
)
15741582
if llm_config.backend.xnnpack.extended_ops:
15751583
partitioners.append(
1576-
get_xnnpack_partitioner(dynamic_quant_only_partitioner=False)
1584+
get_xnnpack_partitioner(
1585+
dynamic_quant_only_partitioner=False,
1586+
enable_bf16=llm_config.backend.xnnpack.enable_bf16,
1587+
)
15771588
)
15781589

15791590
return partitioners if partitioners else None
@@ -1759,6 +1770,7 @@ def _export_llama(llm_config: LlmConfig) -> LLMEdgeManager: # noqa: C901
17591770
generate_etrecord=llm_config.debug.generate_etrecord,
17601771
verbose=llm_config.debug.verbose,
17611772
gen_tag_fn=gen_tag_fn,
1773+
enable_bf16=llm_config.backend.xnnpack.enable_bf16,
17621774
)
17631775
elif llm_config.backend.openvino.enabled:
17641776
builder = _to_edge_and_lower_llama_openvino(

‎examples/models/llama/tests/test_export_llama_lib.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,24 @@ def test_has_expected_ops_and_op_counts(self):
7070
for op, _op_info in delegation_info.delegation_by_operator.items():
7171
self.assertTrue(op not in UNWANTED_OPS)
7272

73+
def test_bf16_xnnpack_delegates_linears_when_enabled(self):
74+
parser = build_args_parser()
75+
args = parser.parse_args([])
76+
args.use_kv_cache = True
77+
args.xnnpack = True
78+
args.xnnpack_extended_ops = True
79+
args.xnnpack_enable_bf16 = True
80+
args.dtype_override = "bf16"
81+
82+
llm_config = LlmConfig.from_args(args)
83+
builder = _export_llama(llm_config)
84+
graph_module = builder.edge_manager.exported_program().graph_module
85+
delegation_info = get_delegation_info(graph_module)
86+
87+
linear = delegation_info.delegation_by_operator["aten_linear_default"]
88+
self.assertGreater(linear.delegated, 0)
89+
self.assertEqual(linear.non_delegated, 0)
90+
7391
@unittest.skipUnless(HAS_ARM_BACKEND, "ARM backend not available")
7492
def test_get_quantizer_and_quant_params_returns_tosa_quantizer(self):
7593
llm_config = LlmConfig()

‎extension/llm/export/config/llm_config.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -528,10 +528,13 @@ class XNNPackConfig:
528528
Attributes:
529529
enabled: :)
530530
extended_ops: Whether to match more types of ops to delegates to XNNPack.
531+
enable_bf16: Whether to delegate BF16 ops to XNNPack. The target runtime
532+
must have hardware support for XNNPACK's BF16 kernels.
531533
"""
532534

533535
enabled: bool = False
534536
extended_ops: bool = False
537+
enable_bf16: bool = False
535538

536539

537540
class CoreMLQuantize(str, Enum):
@@ -812,6 +815,8 @@ def from_args(cls, args: argparse.Namespace) -> "LlmConfig": # noqa: C901
812815
llm_config.backend.xnnpack.enabled = args.xnnpack
813816
if hasattr(args, "xnnpack_extended_ops"):
814817
llm_config.backend.xnnpack.extended_ops = args.xnnpack_extended_ops
818+
if hasattr(args, "xnnpack_enable_bf16"):
819+
llm_config.backend.xnnpack.enable_bf16 = args.xnnpack_enable_bf16
815820

816821
# CoreML
817822
if hasattr(args, "coreml"):

‎extension/llm/export/test/test_export_llm.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ def test_with_config(self, mock_export_llama: MagicMock) -> None:
7272
pt2e_quantize: xnnpack_dynamic
7373
use_spin_quant: cuda
7474
backend:
75+
xnnpack:
76+
enable_bf16: true
7577
coreml:
7678
quantize: c4w
7779
compute_units: cpu_and_gpu
@@ -98,6 +100,7 @@ def test_with_config(self, mock_export_llama: MagicMock) -> None:
98100
called_config.quantization.pt2e_quantize.value, "xnnpack_dynamic"
99101
)
100102
self.assertEqual(called_config.quantization.use_spin_quant.value, "cuda")
103+
self.assertTrue(called_config.backend.xnnpack.enable_bf16)
101104
self.assertEqual(called_config.backend.coreml.quantize.value, "c4w")
102105
self.assertEqual(
103106
called_config.backend.coreml.compute_units.value, "cpu_and_gpu"

0 commit comments

Comments
 (0)