Skip to content

Commit 0782745

Browse files
authored
Fused quant add Tensor kernel (pytorch#19454)
Differential Revision: D103314469 Pull Request resolved: pytorch#19454
1 parent d306410 commit 0782745

7 files changed

Lines changed: 936 additions & 0 deletions

File tree

‎backends/cadence/fused_quant/BUCK‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
load(":targets.bzl", "define_common_targets")
2+
3+
oncall("odai_jarvis")
4+
5+
define_common_targets()
Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#include <executorch/backends/cadence/fused_quant/op_add.h>
10+
#include <executorch/backends/cadence/fused_quant/quant_utils.h>
11+
#include <executorch/runtime/kernel/kernel_includes.h>
12+
13+
namespace cadence {
14+
namespace fused_quant {
15+
namespace native {
16+
17+
using executorch::aten::optional;
18+
using executorch::aten::ScalarType;
19+
using executorch::aten::Tensor;
20+
using executorch::runtime::KernelRuntimeContext;
21+
22+
namespace {
23+
24+
void add_kernel(
25+
const float* inp,
26+
const float* other,
27+
float* out,
28+
int64_t numel,
29+
float alpha) {
30+
for (int64_t i = 0; i < numel; ++i) {
31+
out[i] = inp[i] + alpha * other[i];
32+
}
33+
}
34+
35+
} // namespace
36+
37+
Tensor& add_out(
38+
KernelRuntimeContext& ctx,
39+
const Tensor& inp,
40+
const Tensor& other,
41+
const optional<Tensor>& inp_scale,
42+
const optional<Tensor>& inp_zero_point,
43+
ScalarType inp_dtype,
44+
int64_t inp_quant_min,
45+
int64_t inp_quant_max,
46+
optional<int64_t> inp_axis,
47+
const optional<Tensor>& other_scale,
48+
const optional<Tensor>& other_zero_point,
49+
ScalarType other_dtype,
50+
int64_t other_quant_min,
51+
int64_t other_quant_max,
52+
optional<int64_t> other_axis,
53+
const optional<Tensor>& out_scale,
54+
const optional<Tensor>& out_zero_point,
55+
ScalarType out_dtype,
56+
int64_t out_quant_min,
57+
int64_t out_quant_max,
58+
optional<int64_t> out_axis,
59+
double alpha,
60+
Tensor& out) {
61+
int64_t numel = inp.numel();
62+
float alpha_f = static_cast<float>(alpha);
63+
64+
bool inp_quantized = inp_scale.has_value();
65+
bool other_quantized = other_scale.has_value();
66+
bool out_quantized = out_scale.has_value();
67+
68+
std::vector<float> inp_buf;
69+
const float* const inp_float = [&]() -> const float* {
70+
if (!inp_quantized) {
71+
return inp.const_data_ptr<float>();
72+
}
73+
inp_buf.resize(numel);
74+
QParams qp = extract_qparams(
75+
inp_scale, inp_zero_point, inp_quant_min, inp_quant_max, inp_axis, inp);
76+
FUSED_QUANT_DTYPE_SWITCH(
77+
inp.scalar_type(),
78+
scalar_t,
79+
dequantize_buffer(
80+
inp.const_data_ptr<scalar_t>(), inp_buf.data(), numel, qp);)
81+
return inp_buf.data();
82+
}();
83+
84+
std::vector<float> other_buf;
85+
const float* const other_float = [&]() -> const float* {
86+
if (!other_quantized) {
87+
return other.const_data_ptr<float>();
88+
}
89+
other_buf.resize(numel);
90+
QParams qp = extract_qparams(
91+
other_scale,
92+
other_zero_point,
93+
other_quant_min,
94+
other_quant_max,
95+
other_axis,
96+
other);
97+
FUSED_QUANT_DTYPE_SWITCH(
98+
other.scalar_type(),
99+
scalar_t,
100+
dequantize_buffer(
101+
other.const_data_ptr<scalar_t>(), other_buf.data(), numel, qp);)
102+
return other_buf.data();
103+
}();
104+
105+
if (out_quantized) {
106+
std::vector<float> result_float(numel);
107+
add_kernel(inp_float, other_float, result_float.data(), numel, alpha_f);
108+
109+
QParams qp = extract_qparams(
110+
out_scale, out_zero_point, out_quant_min, out_quant_max, out_axis, out);
111+
FUSED_QUANT_DTYPE_SWITCH(
112+
out.scalar_type(),
113+
scalar_t,
114+
quantize_buffer(
115+
result_float.data(), out.mutable_data_ptr<scalar_t>(), numel, qp);)
116+
} else {
117+
add_kernel(
118+
inp_float, other_float, out.mutable_data_ptr<float>(), numel, alpha_f);
119+
}
120+
121+
return out;
122+
}
123+
124+
} // namespace native
125+
} // namespace fused_quant
126+
} // namespace cadence
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#pragma once
10+
11+
#include <executorch/runtime/core/exec_aten/exec_aten.h>
12+
#include <executorch/runtime/kernel/kernel_includes.h>
13+
14+
namespace cadence {
15+
namespace fused_quant {
16+
namespace native {
17+
18+
executorch::aten::Tensor& add_out(
19+
executorch::runtime::KernelRuntimeContext& ctx,
20+
const executorch::aten::Tensor& inp,
21+
const executorch::aten::Tensor& other,
22+
const executorch::aten::optional<executorch::aten::Tensor>& inp_scale,
23+
const executorch::aten::optional<executorch::aten::Tensor>& inp_zero_point,
24+
executorch::aten::ScalarType inp_dtype,
25+
int64_t inp_quant_min,
26+
int64_t inp_quant_max,
27+
executorch::aten::optional<int64_t> inp_axis,
28+
const executorch::aten::optional<executorch::aten::Tensor>& other_scale,
29+
const executorch::aten::optional<executorch::aten::Tensor>&
30+
other_zero_point,
31+
executorch::aten::ScalarType other_dtype,
32+
int64_t other_quant_min,
33+
int64_t other_quant_max,
34+
executorch::aten::optional<int64_t> other_axis,
35+
const executorch::aten::optional<executorch::aten::Tensor>& out_scale,
36+
const executorch::aten::optional<executorch::aten::Tensor>& out_zero_point,
37+
executorch::aten::ScalarType out_dtype,
38+
int64_t out_quant_min,
39+
int64_t out_quant_max,
40+
executorch::aten::optional<int64_t> out_axis,
41+
double alpha,
42+
executorch::aten::Tensor& out);
43+
44+
} // namespace native
45+
} // namespace fused_quant
46+
} // namespace cadence
Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
#pragma once
9+
10+
#include <algorithm>
11+
#include <cmath>
12+
#include <cstdint>
13+
14+
#include <executorch/runtime/core/exec_aten/exec_aten.h>
15+
#include <executorch/runtime/platform/assert.h>
16+
17+
namespace cadence {
18+
namespace fused_quant {
19+
20+
struct QParams {
21+
const float* scales;
22+
const int64_t* zero_points;
23+
int32_t quant_min;
24+
int32_t quant_max;
25+
int64_t num_channels;
26+
int64_t axis_stride;
27+
28+
float scale_at(int64_t i) const {
29+
return scales[channel_idx(i)];
30+
}
31+
32+
int32_t zero_point_at(int64_t i) const {
33+
return static_cast<int32_t>(zero_points[channel_idx(i)]);
34+
}
35+
36+
private:
37+
int64_t channel_idx(int64_t i) const {
38+
if (num_channels == 1) {
39+
return 0;
40+
}
41+
return (i / axis_stride) % num_channels;
42+
}
43+
};
44+
45+
inline QParams extract_qparams(
46+
const executorch::aten::optional<executorch::aten::Tensor>& scale_tensor,
47+
const executorch::aten::optional<executorch::aten::Tensor>& zp_tensor,
48+
int64_t quant_min,
49+
int64_t quant_max,
50+
executorch::aten::optional<int64_t> axis,
51+
const executorch::aten::Tensor& data_tensor) {
52+
const auto& scale = scale_tensor.value();
53+
const auto& zp = zp_tensor.value();
54+
55+
int64_t num_channels = scale.numel();
56+
int64_t axis_stride = 1;
57+
if (axis.has_value()) {
58+
for (int64_t d = axis.value() + 1; d < data_tensor.dim(); ++d) {
59+
axis_stride *= data_tensor.size(d);
60+
}
61+
}
62+
63+
return {
64+
scale.const_data_ptr<float>(),
65+
zp.const_data_ptr<int64_t>(),
66+
static_cast<int32_t>(quant_min),
67+
static_cast<int32_t>(quant_max),
68+
num_channels,
69+
axis_stride,
70+
};
71+
}
72+
73+
template <typename T>
74+
inline float dequantize(T val, float scale, int32_t zero_point) {
75+
return (static_cast<float>(val) - static_cast<float>(zero_point)) * scale;
76+
}
77+
78+
template <typename T>
79+
inline T quantize(
80+
float val,
81+
float scale,
82+
int32_t zero_point,
83+
int32_t qmin,
84+
int32_t qmax) {
85+
int32_t quantized =
86+
static_cast<int32_t>(std::round(val / scale)) + zero_point;
87+
quantized = std::max(qmin, std::min(qmax, quantized));
88+
return static_cast<T>(quantized);
89+
}
90+
91+
template <typename T>
92+
inline void
93+
dequantize_buffer(const T* src, float* dst, int64_t numel, const QParams& qp) {
94+
for (int64_t i = 0; i < numel; ++i) {
95+
dst[i] = dequantize(src[i], qp.scale_at(i), qp.zero_point_at(i));
96+
}
97+
}
98+
99+
template <typename T>
100+
inline void
101+
quantize_buffer(const float* src, T* dst, int64_t numel, const QParams& qp) {
102+
for (int64_t i = 0; i < numel; ++i) {
103+
dst[i] = quantize<T>(
104+
src[i],
105+
qp.scale_at(i),
106+
qp.zero_point_at(i),
107+
qp.quant_min,
108+
qp.quant_max);
109+
}
110+
}
111+
112+
// Dispatch on ScalarType, binding `scalar_t` for the body.
113+
#define FUSED_QUANT_DTYPE_SWITCH(scalar_type, scalar_t, ...) \
114+
switch (scalar_type) { \
115+
case executorch::aten::ScalarType::Byte: { \
116+
using scalar_t = uint8_t; \
117+
__VA_ARGS__; \
118+
break; \
119+
} \
120+
case executorch::aten::ScalarType::Char: { \
121+
using scalar_t = int8_t; \
122+
__VA_ARGS__; \
123+
break; \
124+
} \
125+
case executorch::aten::ScalarType::Short: { \
126+
using scalar_t = int16_t; \
127+
__VA_ARGS__; \
128+
break; \
129+
} \
130+
case executorch::aten::ScalarType::Int: { \
131+
using scalar_t = int32_t; \
132+
__VA_ARGS__; \
133+
break; \
134+
} \
135+
default: \
136+
ET_CHECK_MSG( \
137+
false, "Unsupported dtype: %hhd", static_cast<int8_t>(scalar_type)); \
138+
}
139+
140+
} // namespace fused_quant
141+
} // namespace cadence
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
load("@fbsource//tools/build_defs:platform_defs.bzl", "CXX")
2+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
3+
4+
def define_common_targets():
5+
runtime.cxx_library(
6+
name = "quant_utils",
7+
exported_headers = ["quant_utils.h"],
8+
exported_deps = [
9+
"//executorch/runtime/core/exec_aten:lib",
10+
],
11+
visibility = ["PUBLIC"],
12+
)
13+
14+
runtime.cxx_library(
15+
name = "op_add",
16+
srcs = ["op_add.cpp"],
17+
exported_headers = ["op_add.h"],
18+
platforms = CXX,
19+
deps = [
20+
":quant_utils",
21+
"//executorch/runtime/kernel:kernel_includes",
22+
],
23+
visibility = ["PUBLIC"],
24+
)
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
load("@fbsource//tools/build_defs:platform_defs.bzl", "CXX")
2+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
3+
4+
oncall("odai_jarvis")
5+
6+
runtime.cxx_test(
7+
name = "test_op_add",
8+
srcs = ["test_op_add.cpp"],
9+
platforms = CXX,
10+
deps = [
11+
"//executorch/backends/cadence/fused_quant:op_add",
12+
"//executorch/kernels/test:gtest_utils",
13+
"//executorch/runtime/core/exec_aten/testing_util:tensor_util",
14+
],
15+
)

0 commit comments

Comments
 (0)