Skip to content

Commit 7bc5e0e

Browse files
authored
Implement Portable bucketize (pytorch#20287)
Fixes pytorch#20270 ### Summary Add portable scalar and tensor bucketize operator, based on the PyTorch implementation. Key differences from the PyTorch implementation: - Implemented only bucketize, no searchsorted - Boundaries tensor size is not checked against output type, as `SizesType` is an `int32_t` and must be 1D - In the original implementation the input scalar is wrapped into a tensor, here two sets of functions are used instead - Supports both NHWC and NCHW as it is a pointwise kernel and there is no sorter argument. This avoids creating contiguous tensors - Stayed as close to the PyTorch version as possible, should a searchsorted implementation be added in the future Release notes: ops & kernels ### Test plan C++ tests are extensive and include failure and edge cases tests. Python tests compare the output of an exported module to the PyTorch implementation only for normal and edge cases, similar to pytorch#15893. cc @larryliu0820 @manuelcandales --------- Signed-off-by: Gallinator <lcgallinator4@gmail.com>
1 parent 2fb0334 commit 7bc5e0e

8 files changed

Lines changed: 759 additions & 0 deletions

File tree

Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#include <executorch/kernels/portable/cpu/util/dtype_util.h>
10+
#include <executorch/kernels/portable/cpu/util/elementwise_util.h>
11+
#include <executorch/runtime/core/exec_aten/util/scalar_type_util.h>
12+
#include <executorch/runtime/core/exec_aten/util/tensor_util.h>
13+
#include <executorch/runtime/kernel/kernel_includes.h>
14+
#include <executorch/runtime/kernel/thread_parallel_interface.h>
15+
16+
namespace torch {
17+
namespace executor {
18+
namespace native {
19+
namespace {
20+
21+
using executorch::runtime::isRealHBF16Type;
22+
using torch::executor::native::utils::SupportedTensorDtypes;
23+
using torch::executor::native::utils::internal::get_load_to_compute_fn;
24+
using torch::executor::native::utils::internal::load_to_compute_fn;
25+
26+
constexpr int64_t BUCKETIZE_GRAIN_SIZE = 200;
27+
28+
template <typename CTYPE>
29+
int64_t cus_lower_bound(
30+
int64_t end,
31+
const CTYPE val,
32+
const char* bd,
33+
load_to_compute_fn<CTYPE> bd_load_fn,
34+
ssize_t bd_elem_size) {
35+
int64_t start = 0;
36+
37+
while (start < end) {
38+
const int64_t mid = start + ((end - start) >> 1);
39+
CTYPE mid_bd = bd_load_fn(&bd[mid * bd_elem_size]);
40+
41+
if (mid_bd < val) {
42+
start = mid + 1;
43+
} else {
44+
end = mid;
45+
}
46+
}
47+
return start;
48+
}
49+
50+
template <typename CTYPE>
51+
int64_t cus_upper_bound(
52+
int64_t end,
53+
const CTYPE val,
54+
const char* bd,
55+
load_to_compute_fn<CTYPE> bd_load_fn,
56+
ssize_t bd_elem_size) {
57+
int64_t start = 0;
58+
59+
while (start < end) {
60+
const int64_t mid = start + ((end - start) >> 1);
61+
CTYPE mid_bd = bd_load_fn(&bd[mid * bd_elem_size]);
62+
63+
if (mid_bd <= val) {
64+
start = mid + 1;
65+
} else {
66+
end = mid;
67+
}
68+
}
69+
return start;
70+
}
71+
72+
template <
73+
typename CTYPE_COMPUTE,
74+
typename CTYPE_OUT,
75+
ET_OPERATOR_NAME_TYPE op_name>
76+
void bucketize_tensor_impl(
77+
KernelRuntimeContext& context,
78+
const Tensor& self,
79+
const Tensor& boundaries,
80+
bool right,
81+
Tensor& out) {
82+
auto in_load_fn = get_load_to_compute_fn<CTYPE_COMPUTE, op_name>(
83+
context, self, SupportedTensorDtypes::REALHBF16);
84+
const ssize_t in_size = self.element_size();
85+
auto in_data = reinterpret_cast<const char*>(self.const_data_ptr());
86+
87+
auto bd_load_fn = get_load_to_compute_fn<CTYPE_COMPUTE, op_name>(
88+
context, boundaries, SupportedTensorDtypes::REALHBF16);
89+
const ssize_t bd_elem_size = boundaries.element_size();
90+
auto bd_data = reinterpret_cast<const char*>(boundaries.const_data_ptr());
91+
int64_t bd_end = boundaries.numel();
92+
93+
auto out_data = out.mutable_data_ptr<CTYPE_OUT>();
94+
95+
const bool success = parallel_for(
96+
0,
97+
self.numel(),
98+
BUCKETIZE_GRAIN_SIZE,
99+
[&](const auto begin, const auto end) {
100+
for (const auto i : c10::irange(begin, end)) {
101+
auto compute_val = in_load_fn(&in_data[i * in_size]);
102+
int64_t pos = right
103+
? cus_upper_bound(
104+
bd_end, compute_val, bd_data, bd_load_fn, bd_elem_size)
105+
: cus_lower_bound(
106+
bd_end, compute_val, bd_data, bd_load_fn, bd_elem_size);
107+
out_data[i] = pos;
108+
}
109+
});
110+
111+
ET_KERNEL_CHECK_MSG(context, success, Internal, , "parallel_for failed");
112+
}
113+
114+
template <
115+
typename CTYPE_COMPUTE,
116+
typename CTYPE_OUT,
117+
ET_OPERATOR_NAME_TYPE op_name>
118+
void bucketize_scalar_impl(
119+
KernelRuntimeContext& context,
120+
const Scalar& self,
121+
const Tensor& boundaries,
122+
bool right,
123+
Tensor& out) {
124+
CTYPE_COMPUTE compute_val = utils::scalar_to<CTYPE_COMPUTE>(self);
125+
126+
auto bd_load_fn = get_load_to_compute_fn<CTYPE_COMPUTE, op_name>(
127+
context, boundaries, SupportedTensorDtypes::REALHBF16);
128+
const ssize_t bd_elem_size = boundaries.element_size();
129+
auto bd_data = reinterpret_cast<const char*>(boundaries.const_data_ptr());
130+
int64_t bd_end = boundaries.numel();
131+
132+
auto out_data = out.mutable_data_ptr<CTYPE_OUT>();
133+
134+
int64_t pos = right
135+
? cus_upper_bound(bd_end, compute_val, bd_data, bd_load_fn, bd_elem_size)
136+
: cus_lower_bound(bd_end, compute_val, bd_data, bd_load_fn, bd_elem_size);
137+
out_data[0] = pos;
138+
}
139+
140+
// Performs check which are common to both tensor and scalar implementations:
141+
// - Boundaries must be 1D
142+
// - Out type must be consistent with out_int32 parameter
143+
// - Boundaries type must be realhbf16
144+
// Boundaries size is not checked against out type as SizesType always fits into
145+
// an int32_t.
146+
Error bucketize_common_pre_checks(
147+
const Tensor& boundaries,
148+
bool out_int32,
149+
Tensor& out) {
150+
ET_CHECK_OR_RETURN_ERROR(
151+
boundaries.dim() == 1,
152+
InvalidArgument,
153+
"boundaries tensor must be 1 dimension, but got dim(%zu)",
154+
boundaries.dim());
155+
156+
ScalarType out_dtype = out.scalar_type();
157+
ET_CHECK_OR_RETURN_ERROR(
158+
(out_dtype == ScalarType::Long && !out_int32) ||
159+
(out_dtype == ScalarType::Int && out_int32),
160+
InvalidArgument,
161+
"torch.bucketize(): output tensor's dtype is wrong, it can only be Int(int32) or Long(int64) depending on whether out_int32 flag is True, but we got output tensor dtype %s and out_int32 flag is %s",
162+
toString(out_dtype),
163+
(out_int32 ? "True" : "False"));
164+
165+
ScalarType bound_dtype = boundaries.scalar_type();
166+
ET_CHECK_OR_RETURN_ERROR(
167+
isRealHBF16Type(bound_dtype),
168+
InvalidArgument,
169+
"boundaries tensor of type %s is not supported",
170+
toString(bound_dtype));
171+
172+
return Error::Ok;
173+
}
174+
175+
} // namespace
176+
177+
using executorch::runtime::tensor_is_realhbf16_type;
178+
179+
Tensor& bucketize_tensor_out(
180+
KernelRuntimeContext& context,
181+
const Tensor& self,
182+
const Tensor& boundaries,
183+
bool out_int32,
184+
bool right,
185+
Tensor& out) {
186+
ET_KERNEL_CHECK(
187+
context,
188+
bucketize_common_pre_checks(boundaries, out_int32, out) == Error::Ok,
189+
InvalidArgument,
190+
out);
191+
ET_KERNEL_CHECK(
192+
context,
193+
resize_tensor(out, self.sizes()) == Error::Ok,
194+
InvalidArgument,
195+
out);
196+
ET_KERNEL_CHECK(
197+
context, tensors_have_same_dim_order(self, out), InvalidArgument, out);
198+
ET_KERNEL_CHECK(
199+
context, tensor_is_realhbf16_type(self), InvalidArgument, out);
200+
201+
ScalarType common_type =
202+
promoteTypes(self.scalar_type(), boundaries.scalar_type());
203+
ScalarType compute_type = utils::get_compute_type(common_type);
204+
205+
ET_DEFINE_OPERATOR_NAME(op_name, "bucketize.Tensor_out");
206+
207+
ET_SWITCH_REALHBF16_TYPES(
208+
compute_type, context, op_name, CTYPE_COMPUTE, [&]() {
209+
if (out_int32) {
210+
bucketize_tensor_impl<CTYPE_COMPUTE, int32_t, op_name>(
211+
context, self, boundaries, right, out);
212+
} else {
213+
bucketize_tensor_impl<CTYPE_COMPUTE, int64_t, op_name>(
214+
context, self, boundaries, right, out);
215+
}
216+
});
217+
return out;
218+
}
219+
220+
Tensor& bucketize_scalar_out(
221+
KernelRuntimeContext& context,
222+
const Scalar& self,
223+
const Tensor& boundaries,
224+
bool out_int32,
225+
bool right,
226+
Tensor& out) {
227+
ET_KERNEL_CHECK(
228+
context,
229+
bucketize_common_pre_checks(boundaries, out_int32, out) == Error::Ok,
230+
InvalidArgument,
231+
out);
232+
ET_KERNEL_CHECK(context, out.dim() == 0, InvalidArgument, out);
233+
234+
ScalarType common_type =
235+
utils::promote_type_with_scalar(boundaries.scalar_type(), self);
236+
ScalarType compute_type = utils::get_compute_type(common_type);
237+
238+
ET_DEFINE_OPERATOR_NAME(op_name, "bucketize.Scalar_out");
239+
240+
ET_SWITCH_REALHBF16_TYPES(
241+
compute_type, context, op_name, CTYPE_COMPUTE, [&]() {
242+
if (out_int32) {
243+
bucketize_scalar_impl<CTYPE_COMPUTE, int32_t, op_name>(
244+
context, self, boundaries, right, out);
245+
} else {
246+
bucketize_scalar_impl<CTYPE_COMPUTE, int64_t, op_name>(
247+
context, self, boundaries, right, out);
248+
}
249+
});
250+
251+
return out;
252+
}
253+
254+
} // namespace native
255+
} // namespace executor
256+
} // namespace torch

‎kernels/portable/functions.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,16 @@
247247
- arg_meta: null
248248
kernel_name: torch::executor::bmm_out
249249

250+
- op: bucketize.Tensor_out
251+
kernels:
252+
- arg_meta: null
253+
kernel_name: torch::executor::bucketize_tensor_out
254+
255+
- op: bucketize.Scalar_out
256+
kernels:
257+
- arg_meta: null
258+
kernel_name: torch::executor::bucketize_scalar_out
259+
250260
- op: cat.out
251261
kernels:
252262
- arg_meta: null

‎kernels/portable/test/register_ops_aot_for_test.cpp‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,34 @@ Tensor& grid_sampler_2d_out_no_context(
130130

131131
return ret;
132132
}
133+
134+
Tensor& bucketize_tensor_out(
135+
KernelRuntimeContext& context,
136+
const Tensor& self,
137+
const Tensor& boundaries,
138+
bool out_int32,
139+
bool right,
140+
Tensor& out);
141+
142+
Tensor& bucketize_tensor_out_no_context(
143+
const Tensor& self,
144+
const Tensor& boundaries,
145+
bool out_int32,
146+
bool right,
147+
Tensor& out) {
148+
KernelRuntimeContext ctx;
149+
auto& ret =
150+
bucketize_tensor_out(ctx, self, boundaries, out_int32, right, out);
151+
152+
if (ctx.failure_state() != Error::Ok) {
153+
throw std::runtime_error(
154+
std::string("Kernel failed with error: ") +
155+
std::to_string((int)ctx.failure_state()));
156+
}
157+
158+
return ret;
159+
}
160+
133161
// NOLINTEND(facebook-hte-ConstantArgumentPassByValue,
134162
// facebook-hte-ParameterMightThrowOnCopy)
135163

@@ -146,6 +174,9 @@ TORCH_LIBRARY(et_test, m) {
146174
m.def(
147175
"grid_sampler_2d.out(Tensor input, Tensor grid, int interpolation_mode, int padding_mode, bool align_corners, *, Tensor(a!) out) -> Tensor(a!)",
148176
WRAP_TO_ATEN(grid_sampler_2d_out_no_context, 5));
177+
m.def(
178+
"bucketize.Tensor_out(Tensor self, Tensor boundaries, *, bool out_int32=False, bool right=False, Tensor(a!) out) -> Tensor(a!)",
179+
WRAP_TO_ATEN(bucketize_tensor_out_no_context, 4));
149180
}
150181

151182
} // namespace native

‎kernels/test/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ set(all_test_sources
184184
"op_bitwise_right_shift_test.cpp"
185185
"op_bitwise_xor_test.cpp"
186186
"op_bmm_test.cpp"
187+
"op_bucketize_test.cpp"
187188
"op_cat_test.cpp"
188189
"op_cdist_forward_test.cpp"
189190
"op_ceil_test.cpp"

0 commit comments

Comments
 (0)