Skip to content

Commit 41b7a76

Browse files
authored
Carry a tensor's device when wrapping it for PyTorch (pytorch#21753)
Re-lands the change from pytorch#21722, which was merged into its stacked base branch rather than into main, so the fix never reached main. Wrapping an ExecuTorch tensor for PyTorch dropped the device, so the resulting tensor carried no device label. A caller that reads the label then either fails with "tensor does not have a device" or, for accelerator memory, reads device memory from the host and faults. Measured on an NVIDIA H100 with a CUDA delegated model whose activations stay on the GPU: before this change the Python path faults, and the same program run from C++ succeeds, which places the defect in the bindings rather than in the runtime. A CPU model is affected the same way, because the wrapped tensor has no device of either kind. Tests: extension/aten_util/test/aten_bridge_test.cpp covers the device mapping, the abort on an unknown device type, and an empty CUDA labelled tensor, which is the case that distinguishes a correct implementation from one that only sets the label or only sets the target device.
1 parent e4344c9 commit 41b7a76

3 files changed

Lines changed: 86 additions & 5 deletions

File tree

‎extension/aten_util/aten_bridge.cpp‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,22 @@ c10::ScalarType executorch_to_torch_scalar_type(
134134
return static_cast<c10::ScalarType>(intermediate);
135135
}
136136

137+
c10::Device executorch_to_torch_device(
138+
executorch::runtime::etensor::Device device) {
139+
switch (device.type()) {
140+
case executorch::runtime::etensor::DeviceType::CPU:
141+
return c10::Device(c10::DeviceType::CPU);
142+
case executorch::runtime::etensor::DeviceType::CUDA:
143+
return c10::Device(c10::DeviceType::CUDA, device.index());
144+
}
145+
// Aborting on an unknown device rather than falling back to CPU: a wrong
146+
// label made the host read accelerator memory and segfault.
147+
ET_CHECK_MSG(
148+
false,
149+
"Tensor reports device type %d, which this build cannot map to a PyTorch device",
150+
static_cast<int>(device.type()));
151+
}
152+
137153
/*
138154
* Following makes two assumptions:
139155
* 1. aten_tensor's lifetime is longer than the liftime within which mutable_et
@@ -167,11 +183,14 @@ at::Tensor alias_attensor_to_etensor(const torch::executor::Tensor& etensor) {
167183
std::vector<int64_t> at_tensor_strides(
168184
etensor.strides().begin(), etensor.strides().end());
169185

170-
at::Tensor t = at::from_blob(
171-
etensor.mutable_data_ptr(),
172-
at_tensor_sizes,
173-
at_tensor_strides,
174-
at::TensorOptions(dtype));
186+
// Both are needed: the dispatch key comes from options, and target_device
187+
// skips the pointer inspection that rejects an empty tensor's null pointer.
188+
const c10::Device device = executorch_to_torch_device(etensor.device());
189+
at::Tensor t = at::for_blob(etensor.mutable_data_ptr(), at_tensor_sizes)
190+
.strides(at_tensor_strides)
191+
.options(at::TensorOptions(dtype).device(device))
192+
.target_device(device)
193+
.make_tensor();
175194

176195
check_tensor_meta(t, etensor);
177196
return t;

‎extension/aten_util/aten_bridge.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
#include <executorch/extension/tensor/tensor.h>
1212
#include <executorch/runtime/core/exec_aten/exec_aten.h>
13+
#include <executorch/runtime/core/portable_type/device.h>
1314

1415
#include <ATen/Functions.h> // @manual=//caffe2/aten:ATen-cpu
1516
#include <ATen/Tensor.h> // @manual=//caffe2/aten:ATen-core
@@ -28,6 +29,9 @@ torch::executor::ScalarType torch_to_executorch_scalar_type(
2829
c10::ScalarType executorch_to_torch_scalar_type(
2930
torch::executor::ScalarType type);
3031

32+
c10::Device executorch_to_torch_device(
33+
executorch::runtime::etensor::Device device);
34+
3135
/*
3236
* @param[in] aten_tensor Input at::Tensor
3337
* @param[in,out] mutable_et ETensor whose underlying memory now will alias to

‎extension/aten_util/test/aten_bridge_test.cpp‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,3 +296,61 @@ TEST(ATenBridgeTest, AliasETensorToATenTensorFailUnsupportedDimOrder) {
296296
torch::executor::Tensor etensor(&tensor_impl);
297297
ET_EXPECT_DEATH(alias_etensor_to_attensor(at_tensor, etensor), "");
298298
}
299+
300+
TEST(ATenBridgeTest, DeviceMapping) {
301+
// Needs no accelerator: only the mapping is under test.
302+
using executorch::runtime::etensor::Device;
303+
using executorch::runtime::etensor::DeviceType;
304+
305+
EXPECT_EQ(
306+
executorch_to_torch_device(Device(DeviceType::CPU)).type(),
307+
c10::DeviceType::CPU);
308+
EXPECT_EQ(
309+
executorch_to_torch_device(Device(DeviceType::CUDA, 1)).type(),
310+
c10::DeviceType::CUDA);
311+
EXPECT_EQ(executorch_to_torch_device(Device(DeviceType::CUDA, 1)).index(), 1);
312+
}
313+
314+
TEST(ATenBridgeTest, DeviceMappingAbortsOnUnknownType) {
315+
using executorch::runtime::etensor::Device;
316+
using executorch::runtime::etensor::DeviceType;
317+
ET_EXPECT_DEATH(
318+
(void)executorch_to_torch_device(Device(static_cast<DeviceType>(99))),
319+
"");
320+
}
321+
322+
TEST(ATenBridgeTest, AliasATTensorToETensorHandlesAnEmptyTensor) {
323+
// An empty tensor has a null data pointer, so this is the only case that
324+
// distinguishes dropping the device, passing it through options alone (which
325+
// raises), and passing target_device. Needs no accelerator: nothing
326+
// allocates.
327+
auto at_tensor = at::empty({0});
328+
std::vector<Tensor::SizesType> sizes(
329+
at_tensor.sizes().begin(), at_tensor.sizes().end());
330+
auto dim_order = get_default_dim_order(at_tensor);
331+
std::vector<Tensor::StridesType> strides(
332+
at_tensor.strides().begin(), at_tensor.strides().end());
333+
auto dtype = torchToExecuTorchScalarType(at_tensor.options().dtype());
334+
torch::executor::TensorImpl tensor_impl(
335+
dtype,
336+
at_tensor.dim(),
337+
sizes.data(),
338+
/*data=*/nullptr,
339+
dim_order.data(),
340+
strides.data(),
341+
executorch::runtime::TensorShapeDynamism::STATIC,
342+
executorch::runtime::etensor::DeviceType::CUDA,
343+
0);
344+
torch::executor::Tensor etensor(&tensor_impl);
345+
346+
auto aliased = alias_attensor_to_etensor(etensor);
347+
EXPECT_EQ(aliased.numel(), 0);
348+
EXPECT_EQ(aliased.device().type(), c10::DeviceType::CUDA);
349+
EXPECT_EQ(aliased.device().index(), 0);
350+
// The dispatch key too, not just the label. The key comes from the options
351+
// while the label can come from target_device, so a call site that passes the
352+
// device only as target_device still satisfies the checks above and produces
353+
// a tensor that dispatches to CPU kernels.
354+
EXPECT_TRUE(aliased.is_cuda());
355+
EXPECT_TRUE(aliased.key_set().has(c10::DispatchKey::CUDA));
356+
}

0 commit comments

Comments
 (0)