Skip to content

Commit a886f38

Browse files
committed
Revert "[xnnpack] Support bf16 delegation for fully-connected (pytorch#21409)"
This reverts commit 4019e4c.
1 parent c6f166d commit a886f38

5 files changed

Lines changed: 12 additions & 120 deletions

File tree

‎backends/xnnpack/operators/node_visitor.py‎

Lines changed: 7 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -265,15 +265,12 @@ def get_per_channel_dtype(
265265
)
266266
else:
267267
node_dtype = get_node_dtype(node)
268-
# fp16/bf16 tensors keep their datatype unless we've been asked to
269-
# force fp32 (e.g. biases for dynamic-quant or bf16 fully-connected),
270-
# in which case they fall back to the default fp32.
271-
float_dtype_map = {
272-
torch.float16: XNNDatatype.xnn_datatype_fp16,
273-
torch.bfloat16: XNNDatatype.xnn_datatype_bf16,
274-
}
275-
if not force_fp32:
276-
dtype = float_dtype_map.get(node_dtype, dtype)
268+
if node_dtype is not None and node_dtype == torch.float16:
269+
dtype = (
270+
XNNDatatype.xnn_datatype_fp32
271+
if force_fp32
272+
else XNNDatatype.xnn_datatype_fp16
273+
)
277274

278275
return dtype
279276

@@ -594,7 +591,7 @@ def get_serialized_buffer_index(
594591
# Quantize buffer if static data is indeed quantized
595592
if quant_params is not None and not quant_params.is_dynamic:
596593
const_val = quant_params.quantize_tensor(const_val).contiguous()
597-
elif const_val.dtype not in (torch.float16, torch.bfloat16) or force_fp32:
594+
elif const_val.dtype != torch.float16 or force_fp32:
598595
# ensure that the const is fp32
599596
const_val = const_val.to(dtype=torch.float32).contiguous()
600597

@@ -715,19 +712,12 @@ def define_nodes_tensor_inputs_outputs(
715712
bias_quant_params = QuantParams.from_bias(
716713
bias_node, weight_quant_params, input_quant_params
717714
)
718-
# XNNPACK's bf16 fully-connected (bf16_bf16_f32) takes a bf16
719-
# activation/weight but an fp32 bias, so force the bias to fp32.
720-
weight_val = weight_node.meta.get("val", None)
721-
bias_force_fp32 = (
722-
weight_val is not None and weight_val.dtype == torch.bfloat16
723-
)
724715
self.define_tensor(
725716
bias_node,
726717
xnn_graph,
727718
vals_to_ids,
728719
quant_params=bias_quant_params,
729720
convert_to_nhwc=False, # Bias is generally 1d and can not be in NHWC
730-
force_fp32=bias_force_fp32,
731721
)
732722

733723
def define_node(

‎backends/xnnpack/operators/op_linear.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,6 @@ def define_node(
7373
force_fp32 = False
7474
if input_quant_params is not None and input_quant_params.is_dynamic:
7575
force_fp32 = True
76-
# XNNPACK's bf16 fully-connected (bf16_bf16_f32) takes a bf16
77-
# activation/weight but an fp32 bias, so force the bias to fp32.
78-
weight_val = weight_node.meta.get("val", None)
79-
if weight_val is not None and weight_val.dtype == torch.bfloat16:
80-
force_fp32 = True
8176

8277
self.define_tensor(
8378
get_input_node(node, 2),

‎backends/xnnpack/partition/config/xnnpack_config.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,6 @@ def _check_node_has_valid_dtype(self, node):
228228
valid_dtypes = {
229229
torch.float32,
230230
torch.float16,
231-
torch.bfloat16,
232231
}
233232
# Only allow int8 and quant dtypes for quant operations
234233
if is_quant(node) or is_dequant(node) or is_qparam(node):

‎backends/xnnpack/runtime/XNNCompiler.cpp‎

Lines changed: 1 addition & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -791,32 +791,6 @@ Error defineConvertNode(
791791

792792
return Error::Ok;
793793
};
794-
/*
795-
Look up a serialized tensor value (plain or quantized wrapper) by its
796-
output id. Returns nullptr if not found.
797-
*/
798-
const fb_xnnpack::XNNTensorValue* getSerializedTensorValue(
799-
const fb_xnnpack::XNNGraph* graph,
800-
uint32_t id) noexcept {
801-
if (graph == nullptr || graph->xvalues() == nullptr) {
802-
return nullptr;
803-
}
804-
for (auto value : *graph->xvalues()) {
805-
const fb_xnnpack::XNNTensorValue* tv = nullptr;
806-
if (value->xvalue_union_type() == fb_xnnpack::XValueUnion::XNNTensorValue) {
807-
tv = value->xvalue_union_as_XNNTensorValue();
808-
} else if (
809-
value->xvalue_union_type() ==
810-
fb_xnnpack::XValueUnion::XNNQuantizedTensorValue) {
811-
tv = value->xvalue_union_as_XNNQuantizedTensorValue()->tensor_value();
812-
}
813-
if (tv != nullptr && tv->id_out() == id) {
814-
return tv;
815-
}
816-
}
817-
return nullptr;
818-
}
819-
820794
/*
821795
Define serialized linear(fully-connected) node into the subgraph using
822796
the remapped ids to map the serialized ids, to the new ids generated
@@ -836,52 +810,14 @@ Error defineFullyConnectedNode(
836810
REMAP_ID(remapped_ids, graph_node->bias_id(), fc_bias);
837811
REMAP_ID(remapped_ids, graph_node->output_id(), fc_output);
838812

839-
// XNNPACK only provides a bf16 fully-connected of type bf16_bf16_f32:
840-
// bf16 activation x bf16 weight -> fp32 output. When the serialized graph
841-
// asks for a bf16 output (e.g. a fully bf16 model), define the FC with an
842-
// fp32 intermediate output and append a convert (fp32 -> bf16) so the
843-
// delegate boundary stays bf16.
844-
const auto* in_tv = getSerializedTensorValue(graph, graph_node->input1_id());
845-
const auto* filt_tv =
846-
getSerializedTensorValue(graph, graph_node->filter_id());
847-
const auto* out_tv = getSerializedTensorValue(graph, graph_node->output_id());
848-
const bool needs_bf16_output_convert = in_tv != nullptr &&
849-
filt_tv != nullptr && out_tv != nullptr &&
850-
in_tv->datatype() == DataType::xnn_datatype_bf16 &&
851-
filt_tv->datatype() == DataType::xnn_datatype_bf16 &&
852-
out_tv->datatype() == DataType::xnn_datatype_bf16;
853-
854-
uint32_t fc_compute_output = fc_output;
855-
if (needs_bf16_output_convert) {
856-
std::vector<size_t> out_dims =
857-
flatbufferDimsToVector<size_t>(out_tv->dims());
858-
uint32_t intermediate_id = XNN_INVALID_VALUE_ID;
859-
xnn_status ts = xnn_define_tensor_value(
860-
subgraph_ptr,
861-
xnn_datatype_fp32,
862-
out_dims.size(),
863-
out_dims.data(),
864-
/*data=*/nullptr,
865-
/*external_id=*/XNN_INVALID_VALUE_ID,
866-
/*flags=*/0,
867-
&intermediate_id);
868-
ET_CHECK_OR_RETURN_ERROR(
869-
ts == xnn_status_success,
870-
Internal,
871-
"Failed to define fp32 intermediate for bf16 linear node %i: %s",
872-
node->debug_handle(),
873-
xnn_status_to_string(ts));
874-
fc_compute_output = intermediate_id;
875-
}
876-
877813
xnn_status status = xnn_define_fully_connected(
878814
subgraph_ptr,
879815
min_max.first,
880816
min_max.second,
881817
fc_input1,
882818
fc_filter,
883819
fc_bias,
884-
fc_compute_output,
820+
fc_output,
885821
graph_node->flags());
886822
ET_CHECK_OR_RETURN_ERROR(
887823
status == xnn_status_success,
@@ -890,17 +826,6 @@ Error defineFullyConnectedNode(
890826
node->debug_handle(),
891827
xnn_status_to_string(status));
892828

893-
if (needs_bf16_output_convert) {
894-
xnn_status cs = xnn_define_convert(
895-
subgraph_ptr, fc_compute_output, fc_output, /*flags=*/0);
896-
ET_CHECK_OR_RETURN_ERROR(
897-
cs == xnn_status_success,
898-
Internal,
899-
"Failed to define bf16 output convert for linear node %i: %s",
900-
node->debug_handle(),
901-
xnn_status_to_string(cs));
902-
}
903-
904829
return Error::Ok;
905830
};
906831

‎backends/xnnpack/test/ops/test_linear.py‎

Lines changed: 4 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,7 @@ def __init__(
6464
self.ic = input_channels
6565
self.oc = output_channels
6666

67-
assert dtype in [
68-
torch.float,
69-
torch.half,
70-
torch.bfloat16,
71-
], "Unsupported op dtype"
67+
assert dtype in [torch.float, torch.half], "Unsupported op dtype"
7268
self.op_dtype = dtype
7369
self.in_size = in_size
7470

@@ -710,15 +706,9 @@ def _test_qd8_per_token_weight_per_channel_group_int4(
710706
# Mean: 0.2373046875, 0.237060546875
711707
# Max: 1.0078125, 1.0078125
712708
# Min: -0.08465576171875, -0.08441162109375
713-
# bf16 has ~8x coarser mantissa than fp16, so it needs a
714-
# looser atol.
715-
# TODO(T212995726): Investigate right atol for rand[n] inputs
716-
if dtype == torch.bfloat16:
717-
atol = 8e-2
718-
elif dtype == torch.half:
719-
atol = 1e-2
720-
else:
721-
atol = 5e-3
709+
atol = (
710+
1e-2 if dtype == torch.half else 5e-3
711+
) # TODO(T212995726): Investigate right atol for rand[n] inputs
722712
self._test_groupwise_dq_linear(
723713
lin_mod, inputs, group_size=bl, use_bias=use_bias, atol=atol
724714
)
@@ -849,13 +839,6 @@ def test_linear_qd8_f16_per_token_weight_per_channel_group_int4(self):
849839
def test_linear_qd8_f32_per_token_weight_per_channel_group_int4(self):
850840
self._test_qd8_per_token_weight_per_channel_group_int4(dtype=torch.float)
851841

852-
# Tests for q[dp]8-bf16-qb4w
853-
@unittest.skipIf(
854-
not torchao_installed, "Per Channel Group Quantization Required TorchAO"
855-
)
856-
def test_linear_qd8_bf16_per_token_weight_per_channel_group_int4(self):
857-
self._test_qd8_per_token_weight_per_channel_group_int4(dtype=torch.bfloat16)
858-
859842
@unittest.skipIf(
860843
not torchao_installed, "Per Channel Group Quantization Required TorchAO"
861844
)

0 commit comments

Comments
 (0)