Skip to content

Commit b48ffef

Browse files
authored
Add SDPA bitwise mask generator DSP kernel
Differential Revision: D90910503 Pull Request resolved: pytorch#17018
1 parent 11dc963 commit b48ffef

5 files changed

Lines changed: 227 additions & 0 deletions

File tree

‎backends/cadence/aot/ops_registrations.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -459,6 +459,12 @@ def register_fake(
459459
"quantized_softmax.per_tensor_out(Tensor input, Tensor mask, int dim, float in_scale, int in_zero_point, float out_scale, int out_zero_point, *, Tensor(a!) out) -> Tensor (a!)"
460460
)
461461

462+
# pack float/bool mask tensor into a bitmask of type uint8 (each element holding 8 bool mask elements)
463+
lib.define("sdpa_bitwise_mask_gen(Tensor mask, float threshold) -> (Tensor out)")
464+
lib.define(
465+
"sdpa_bitwise_mask_gen.out(Tensor mask, float threshold, *, Tensor(a!) out) -> Tensor (a!)"
466+
)
467+
462468
# Load/store with iDMA. These only exist before memory planning.
463469
# Post memory planning, we check that outputs/inputs for the load/store are in
464470
# DTCM and replace idma_load/idma_store with idma_copy.
@@ -2766,6 +2772,21 @@ def quantized_softmax_per_tensor_meta(
27662772
return input.new_empty(input.size(), dtype=input.dtype)
27672773

27682774

2775+
@register_fake("cadence::sdpa_bitwise_mask_gen")
2776+
def sdpa_bitwise_mask_gen_meta(
2777+
mask: torch.Tensor,
2778+
threshold: float,
2779+
) -> torch.Tensor:
2780+
# Expect mask to be a float/bool tensor with last dimension representing sequence length
2781+
assert mask.dim() >= 1, "mask must have at least 1 dimension"
2782+
mask_shape = list(mask.shape)
2783+
2784+
last = mask_shape[-1]
2785+
assert last % 8 == 0, "last dimension must be a multiple of 8"
2786+
mask_shape[-1] = last // 8 # pack 8 elements into 1 byte
2787+
return mask.new_empty(mask_shape, dtype=torch.uint8)
2788+
2789+
27692790
@register_fake("cadence::quantized_w8a32_linear")
27702791
def quantized_w8a32_linear_meta(
27712792
src: torch.Tensor,

‎backends/cadence/aot/ref_implementations.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2208,3 +2208,85 @@ def quantized_softmax(
22082208
out_scale,
22092209
out_zero_point,
22102210
)
2211+
2212+
2213+
@impl_tracked(m, "sdpa_bitwise_mask_gen")
2214+
def sdpa_bitwise_mask_gen(mask: torch.Tensor, threshold: float) -> torch.Tensor:
2215+
"""
2216+
Generate a bit-packed mask tensor for SDPA.
2217+
2218+
Notes:
2219+
- The semantic of "mask" here is inverted relative to PyTorch's typical boolean masks.
2220+
In PyTorch, a boolean mask generally uses True to indicate positions to keep/attend,
2221+
and False to indicate positions to mask out. In this function, the convention is
2222+
reversed: a value of 1 in the packed byte indicates a masked (disallowed) position,
2223+
while 0 indicates an unmasked (allowed) position.
2224+
Concretely:
2225+
* For a bool mask input:
2226+
True -> unmasked/allowed -> stored as bit 0 (we invert with ~)
2227+
False -> masked/disallowed -> stored as bit 1
2228+
* For a float mask input with threshold:
2229+
value < threshold -> masked/disallowed (bit 1)
2230+
value >= threshold -> unmasked/allowed (bit 0)
2231+
2232+
Behavior:
2233+
- Input mask can be torch.bool or torch.float.
2234+
- The last dimension must be a multiple of 8 so that each group of 8 elements
2235+
packs into one byte (little-endian within the byte: the first element maps
2236+
to the least significant bit).
2237+
- For bool masks, bits are computed as the inverse of the boolean value to align
2238+
with the "1 means masked" convention.
2239+
- For float masks, a comparison against `threshold` is used to determine masked
2240+
positions (value < threshold -> masked -> bit = 1).
2241+
"""
2242+
2243+
assert len(mask.shape) >= 1, "Mask must be at least 1D"
2244+
assert mask.dtype in [torch.bool, torch.float], "Mask must be bool or float"
2245+
assert mask.shape[-1] % 8 == 0, "Mask last dim must be a multiple of 8"
2246+
if mask.dtype == torch.bool:
2247+
# Pack every 8 boolean elements into a single byte in the output tensor.
2248+
# Flatten to 1D for straightforward packing
2249+
original_shape = mask.shape
2250+
flat = mask.contiguous().view(-1)
2251+
# Convert boolean to uint8 and group into chunks of 8
2252+
bits = flat.to(torch.uint8).view(-1, 8)
2253+
2254+
# Pack 8 bits into one byte (little-endian within a byte: first element -> LSB)
2255+
packed = (
2256+
(~bits[:, 0] & 1) << 0
2257+
| (~bits[:, 1] & 1) << 1
2258+
| (~bits[:, 2] & 1) << 2
2259+
| (~bits[:, 3] & 1) << 3
2260+
| (~bits[:, 4] & 1) << 4
2261+
| (~bits[:, 5] & 1) << 5
2262+
| (~bits[:, 6] & 1) << 6
2263+
| (~bits[:, 7] & 1) << 7
2264+
).to(torch.uint8)
2265+
2266+
# Compute packed last dim size
2267+
last_dim = original_shape[-1]
2268+
packed_last = last_dim // 8
2269+
# Reshape packed to match mask shape, with last dim packed to bytes
2270+
return packed.view(*original_shape[:-1], packed_last)
2271+
else:
2272+
assert mask.dtype == torch.float, "Mask must be bool or float"
2273+
# Pack every 8 boolean elements into a single byte in the output tensor.
2274+
# Flatten to 1D for straightforward packing
2275+
original_shape = mask.shape
2276+
flat = mask.contiguous().view(-1, 8)
2277+
packed = (
2278+
(flat[:, 0] < threshold) << 0
2279+
| (flat[:, 1] < threshold) << 1
2280+
| (flat[:, 2] < threshold) << 2
2281+
| (flat[:, 3] < threshold) << 3
2282+
| (flat[:, 4] < threshold) << 4
2283+
| (flat[:, 5] < threshold) << 5
2284+
| (flat[:, 6] < threshold) << 6
2285+
| (flat[:, 7] < threshold) << 7
2286+
).to(torch.uint8)
2287+
2288+
# Compute packed last dim size
2289+
last_dim = original_shape[-1]
2290+
packed_last = last_dim // 8
2291+
# Reshape packed to match mask shape, with last dim packed to bytes
2292+
return packed.view(*original_shape[:-1], packed_last)
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#include <executorch/backends/cadence/generic/operators/op_sdpa_bitwise_mask_gen.h>
10+
#include <executorch/runtime/core/exec_aten/util/tensor_util.h>
11+
12+
namespace impl {
13+
namespace generic {
14+
namespace native {
15+
16+
::executorch::aten::Tensor& sdpa_bitwise_mask_gen_out(
17+
ET_UNUSED ::executorch::runtime::KernelRuntimeContext& ctx,
18+
const ::executorch::aten::Tensor& mask,
19+
double threshold,
20+
::executorch::aten::Tensor& out) {
21+
// NOTE: Mask semantics and bit polarity
22+
//
23+
// PyTorch SDPA (scaled dot-product attention) commonly uses a mask where:
24+
// - True (or large positive values after conversion) means "keep/allow"
25+
// - False (or large negative values) means "mask/block"
26+
//
27+
// In this backend we deliberately flip the sense of the mask bit when
28+
// packing:
29+
// - We generate a bit = 1 to indicate "masked/blocked"
30+
// - We generate a bit = 0 to indicate "valid/kept"
31+
//
32+
// Rationale:
33+
// - The downstream kernels expect a compact bitmask where 1 means "do not
34+
// attend".
35+
// - Therefore, for a boolean mask we invert the value (!value):
36+
// input True (keep) -> bit 0
37+
// input False (block) -> bit 1
38+
// - For a floating mask, values strictly less than the threshold are
39+
// considered "masked":
40+
// input < threshold -> bit 1 (masked)
41+
// else -> bit 0 (kept)
42+
//
43+
// Packing layout:
44+
// - Elements are consumed in groups of 8 along the last dimension.
45+
// - The j-th element in a group maps to bit j of the output byte
46+
// (LSB-first).
47+
// - This produces a compact per-8-elements representation matching the
48+
// kernel’s expectation.
49+
50+
// Ensure the last dimension of mask is divisible by 8
51+
auto sizes = mask.sizes();
52+
ET_KERNEL_CHECK(ctx, sizes.size() > 0, InvalidArgument, out);
53+
auto last_dim = sizes.back();
54+
ET_KERNEL_CHECK(ctx, last_dim % 8 == 0, InvalidArgument, out);
55+
const auto dtype = mask.dtype();
56+
const int64_t numel = mask.numel();
57+
if (dtype == ::executorch::aten::ScalarType::Bool) {
58+
// Generate bitwise mask by iterating boolean tensor elements and inverting
59+
// each
60+
for (int64_t i = 0, out_index = 0; i < numel; i += 8, out_index++) {
61+
uint8_t packed_mask = 0;
62+
for (int64_t j = 0; j < 8; j++) {
63+
packed_mask |= (!mask.mutable_data_ptr<bool>()[i + j]) << j;
64+
}
65+
out.mutable_data_ptr<uint8_t>()[out_index] = packed_mask;
66+
}
67+
} else if (dtype == ::executorch::aten::ScalarType::Float) {
68+
for (int64_t i = 0, out_index = 0; i < numel; i += 8, out_index++) {
69+
uint8_t packed_mask = 0;
70+
for (int64_t j = 0; j < 8; j++) {
71+
packed_mask |= (mask.mutable_data_ptr<float>()[i + j] < threshold) << j;
72+
}
73+
out.mutable_data_ptr<uint8_t>()[out_index] = packed_mask;
74+
}
75+
} else {
76+
ET_KERNEL_CHECK(ctx, false, InvalidArgument, out);
77+
}
78+
return out;
79+
}
80+
81+
} // namespace native
82+
} // namespace generic
83+
} // namespace impl
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#pragma once
10+
11+
#include <executorch/runtime/core/exec_aten/exec_aten.h>
12+
#include <executorch/runtime/kernel/kernel_runtime_context.h>
13+
14+
namespace impl {
15+
namespace generic {
16+
namespace native {
17+
18+
// pack fp/bool mask to bitwise uint8 mask for SDPA (Scaled Dot-Product
19+
// Attention) op
20+
::executorch::aten::Tensor& sdpa_bitwise_mask_gen_out(
21+
::executorch::runtime::KernelRuntimeContext& ctx,
22+
const ::executorch::aten::Tensor& mask,
23+
double threshold,
24+
::executorch::aten::Tensor& out);
25+
26+
} // namespace native
27+
} // namespace generic
28+
} // namespace impl

‎backends/cadence/generic/operators/targets.bzl‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -411,3 +411,16 @@ def define_common_targets():
411411
],
412412
visibility = ["PUBLIC"],
413413
)
414+
415+
runtime.cxx_library(
416+
name = "op_sdpa_bitwise_mask_gen",
417+
srcs = ["op_sdpa_bitwise_mask_gen.cpp"],
418+
exported_headers = ["op_sdpa_bitwise_mask_gen.h"],
419+
platforms = CXX,
420+
deps = [
421+
"//executorch/runtime/kernel:kernel_includes",
422+
"//executorch/runtime/core/exec_aten:lib",
423+
"//executorch/runtime/kernel:kernel_runtime_context",
424+
],
425+
visibility = ["PUBLIC"],
426+
)

0 commit comments

Comments
 (0)