Skip to content

Commit 91b2e90

Browse files
authored
Let the CUDA allocator honor any power of two alignment (pytorch#23446)
## Problem `CudaAllocator::allocate` refused alignments above 256 bytes. TensorRT asks for 512 bytes when loading an engine, but CUDA guarantees only 256. Accepting larger alignments must not depend on how a GPU packs its allocations. ## Change The allocator now reserves extra memory and rounds the returned pointer up when the requested alignment exceeds 256 bytes. | Requested alignment | Before | After | | --- | --- | --- | | Power of two, up to 256 bytes | Allocates | Allocates without padding | | Power of two, above 256 bytes | `NotSupported` | Allocates with padding and rounding | | Zero or not a power of two | `InvalidArgument` | `InvalidArgument` | Requests above 256 reserve `alignment - 1` extra bytes. Rounding consumes at most that many bytes. Size overflow returns `InvalidArgument` before any CUDA call. Only pointers changed by rounding are recorded. `deallocate` retrieves and frees the original pointer. A mutex protects the map; an atomic count keeps deallocation free of that lock when no rounded pointers are live. For alignments up to 256, the pointer check remains as a safety net. A misaligned block is freed and rejected with `NotSupported`, instead of `MemoryAllocationFailed`. Async allocation and memory pools are unchanged. The large alignment tests no longer skip on ROCm. ## Validation 20 allocator tests passed on Linux x86_64 with an H100 and CUDA 12.8. They cover 512 and 4096 byte round trips with mixed sizes live over 64 cycles, concurrent allocations, overflow boundaries, and original pointer deallocation through weak CUDA stand ins. Compute Sanitizer reported zero memory errors and zero leaked allocations. Intentional invalid device API errors were excluded from its API error reporting. The previous allocator fails four regression tests. Restoring this implementation passes the full suite again. ROCm and Windows were not built or run.
1 parent 535dd92 commit 91b2e90

3 files changed

Lines changed: 240 additions & 18 deletions

File tree

‎backends/cuda/runtime/cuda_allocator.cpp‎

Lines changed: 33 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@
1212
#include <executorch/extension/cuda/runtime_api.h>
1313
#include <executorch/runtime/platform/log.h>
1414

15+
#include <limits>
16+
1517
#if !defined(EXECUTORCH_USE_HIP)
16-
#include <mutex>
17-
#include <unordered_map>
1818
#include <vector>
1919
#endif
2020

@@ -240,19 +240,15 @@ CudaAllocator::allocate(size_t nbytes, DeviceIndex index, size_t alignment) {
240240
"CudaAllocator::allocate: alignment must be a power of 2, got %zu",
241241
alignment);
242242

243-
// cudaMalloc is documented to return memory aligned to at least 256 bytes,
244-
// which trivially satisfies kDefaultAlignment (alignof(void*)). For any
245-
// requested alignment <= 256 bytes, the returned pointer is already aligned.
246-
// Stricter alignment would require over-allocation plus bookkeeping that
247-
// deallocate() does not currently support, so reject that case.
243+
// cudaMalloc guarantees only 256-byte alignment.
248244
constexpr size_t kCudaMallocAlignment = 256;
245+
const size_t padding = alignment > kCudaMallocAlignment ? alignment - 1 : 0;
249246
ET_CHECK_OR_RETURN_ERROR(
250-
alignment <= kCudaMallocAlignment,
251-
NotSupported,
252-
"CudaAllocator::allocate: requested alignment %zu exceeds cudaMalloc's "
253-
"guaranteed alignment of %zu bytes; stricter alignment is not supported",
254-
alignment,
255-
kCudaMallocAlignment);
247+
nbytes <= std::numeric_limits<size_t>::max() - padding,
248+
InvalidArgument,
249+
"CudaAllocator::allocate: size %zu with alignment %zu overflows",
250+
nbytes,
251+
alignment);
256252

257253
void* ptr = nullptr;
258254
int prev_device = 0;
@@ -293,7 +289,7 @@ CudaAllocator::allocate(size_t nbytes, DeviceIndex index, size_t alignment) {
293289
}
294290
}
295291

296-
cudaError_t err = cudaMalloc(&ptr, nbytes);
292+
cudaError_t err = cudaMalloc(&ptr, nbytes + padding);
297293

298294
if (switch_device) {
299295
(void)cudaSetDevice(prev_device);
@@ -309,17 +305,25 @@ CudaAllocator::allocate(size_t nbytes, DeviceIndex index, size_t alignment) {
309305
return Error::MemoryAllocationFailed;
310306
}
311307

312-
// Sanity check: the pointer returned by cudaMalloc should already meet the
313-
// requested alignment. If a future CUDA runtime weakens this guarantee, we
314-
// want to fail loudly rather than silently return a misaligned pointer.
308+
if (padding != 0) {
309+
void* aligned_ptr = reinterpret_cast<void*>(
310+
(reinterpret_cast<uintptr_t>(ptr) + padding) & ~uintptr_t(padding));
311+
if (aligned_ptr != ptr) {
312+
const std::lock_guard<std::mutex> lock(padded_allocations_mutex_);
313+
padded_allocations_.emplace(aligned_ptr, ptr);
314+
padded_allocation_count_.fetch_add(1, std::memory_order_relaxed);
315+
}
316+
return aligned_ptr;
317+
}
318+
315319
if ((reinterpret_cast<uintptr_t>(ptr) & (alignment - 1)) != 0) {
316320
ET_LOG(
317321
Error,
318322
"cudaMalloc returned pointer %p not aligned to %zu bytes",
319323
ptr,
320324
alignment);
321325
(void)cudaFree(ptr);
322-
return Error::MemoryAllocationFailed;
326+
return Error::NotSupported;
323327
}
324328

325329
return ptr;
@@ -330,6 +334,17 @@ void CudaAllocator::deallocate(void* ptr, DeviceIndex index) {
330334
return;
331335
}
332336

337+
// Avoid locking when no rounded pointers are live.
338+
if (padded_allocation_count_.load(std::memory_order_relaxed) != 0) {
339+
const std::lock_guard<std::mutex> lock(padded_allocations_mutex_);
340+
const auto it = padded_allocations_.find(ptr);
341+
if (it != padded_allocations_.end()) {
342+
ptr = it->second;
343+
padded_allocations_.erase(it);
344+
padded_allocation_count_.fetch_sub(1, std::memory_order_relaxed);
345+
}
346+
}
347+
333348
int prev_device = 0;
334349
bool switched_device = false;
335350

‎backends/cuda/runtime/cuda_allocator.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@
1111
#include <executorch/extension/cuda/runtime_api.h>
1212
#include <executorch/runtime/core/device_allocator.h>
1313

14+
#include <atomic>
15+
#include <mutex>
16+
#include <unordered_map>
17+
1418
namespace executorch::backends::cuda {
1519

1620
/**
@@ -25,6 +29,7 @@ namespace executorch::backends::cuda {
2529
*/
2630
class CudaAllocator final : public executorch::runtime::DeviceAllocator {
2731
public:
32+
/// Alignments above 256 bytes add up to alignment minus one bytes of padding.
2833
executorch::runtime::Result<void*> allocate(
2934
size_t nbytes,
3035
executorch::runtime::etensor::DeviceIndex index,
@@ -125,6 +130,11 @@ class CudaAllocator final : public executorch::runtime::DeviceAllocator {
125130
size_t nbytes,
126131
cudaMemcpyKind direction,
127132
cudaStream_t stream);
133+
134+
private:
135+
std::mutex padded_allocations_mutex_;
136+
std::unordered_map<void*, void*> padded_allocations_;
137+
std::atomic<size_t> padded_allocation_count_{0};
128138
};
129139

130140
} // namespace executorch::backends::cuda

‎backends/cuda/runtime/test/test_cuda_allocator.cpp‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,13 @@
1212

1313
#include <cstdint>
1414
#include <limits>
15+
#include <thread>
1516
#include <vector>
1617

18+
#if defined(__linux__) && !defined(EXECUTORCH_USE_HIP)
19+
#include <dlfcn.h>
20+
#endif
21+
1722
#include <executorch/backends/cuda/runtime/cuda_allocator.h>
1823
#include <executorch/extension/cuda/caller_stream.h>
1924
#include <executorch/runtime/core/error.h>
@@ -23,6 +28,50 @@ using executorch::backends::cuda::CudaAllocator;
2328
using executorch::runtime::Error;
2429
using executorch::runtime::etensor::DeviceIndex;
2530

31+
#if defined(__linux__) && !defined(EXECUTORCH_USE_HIP)
32+
// Force rounding independently of how the GPU packs its allocations.
33+
namespace {
34+
constexpr uintptr_t kFakeBlock = 256;
35+
thread_local uintptr_t fake_block = kFakeBlock;
36+
thread_local bool hand_out_fake_block = false;
37+
thread_local void* freed_fake_block = nullptr;
38+
thread_local size_t fake_malloc_size = 0;
39+
} // namespace
40+
41+
// Weak, so the real ones win if a static CUDA runtime is linked in. The linker
42+
// can still leave them out, and then a stand-in has nothing to call.
43+
extern "C" __attribute__((weak)) cudaError_t
44+
cudaMalloc(void** ptr, size_t size) {
45+
if (hand_out_fake_block) {
46+
hand_out_fake_block = false;
47+
*ptr = reinterpret_cast<void*>(fake_block);
48+
fake_malloc_size = size;
49+
return cudaSuccess;
50+
}
51+
static const auto real = reinterpret_cast<cudaError_t (*)(void**, size_t)>(
52+
dlsym(RTLD_NEXT, "cudaMalloc"));
53+
if (real == nullptr) {
54+
ADD_FAILURE() << "the test replaced cudaMalloc but found no real one";
55+
return cudaErrorUnknown;
56+
}
57+
return real(ptr, size);
58+
}
59+
60+
extern "C" __attribute__((weak)) cudaError_t cudaFree(void* ptr) {
61+
if (ptr != nullptr && reinterpret_cast<uintptr_t>(ptr) <= 4096) {
62+
freed_fake_block = ptr;
63+
return cudaSuccess;
64+
}
65+
static const auto real =
66+
reinterpret_cast<cudaError_t (*)(void*)>(dlsym(RTLD_NEXT, "cudaFree"));
67+
if (real == nullptr) {
68+
ADD_FAILURE() << "the test replaced cudaFree but found no real one";
69+
return cudaErrorUnknown;
70+
}
71+
return real(ptr);
72+
}
73+
#endif
74+
2675
class CudaAllocatorTest : public testing::Test {
2776
protected:
2877
void SetUp() override {
@@ -153,6 +202,154 @@ TEST_F(CudaAllocatorTest, AllocateOnMissingDeviceFails) {
153202
EXPECT_EQ(res.error(), Error::Internal);
154203
}
155204

205+
TEST_F(CudaAllocatorTest, LargeAlignmentsWithLiveBlocksRoundtrip) {
206+
CudaAllocator& a = CudaAllocator::instance();
207+
for (int cycle = 0; cycle < 64; ++cycle) {
208+
std::vector<void*> live;
209+
for (size_t alignment : {512, 4096}) {
210+
for (size_t nbytes : {1, 256, 512, 1000, 2048, 8192, 65536}) {
211+
SCOPED_TRACE(
212+
testing::Message() << cycle << ": " << alignment << ", " << nbytes);
213+
auto res = a.allocate(nbytes, 0, alignment);
214+
EXPECT_TRUE(res.ok());
215+
if (!res.ok()) {
216+
for (void* ptr : live) {
217+
a.deallocate(ptr, 0);
218+
}
219+
return;
220+
}
221+
void* ptr = res.get();
222+
live.push_back(ptr);
223+
EXPECT_EQ(reinterpret_cast<uintptr_t>(ptr) % alignment, 0u);
224+
std::vector<uint8_t> src(nbytes, cycle + live.size()), dst(nbytes, 0);
225+
EXPECT_EQ(a.copy_host_to_device(ptr, src.data(), nbytes, 0), Error::Ok);
226+
EXPECT_EQ(a.copy_device_to_host(dst.data(), ptr, nbytes, 0), Error::Ok);
227+
EXPECT_EQ(src, dst);
228+
}
229+
}
230+
for (void* ptr : live) {
231+
a.deallocate(ptr, 0);
232+
}
233+
}
234+
}
235+
236+
TEST_F(CudaAllocatorTest, ConcurrentAlignedAllocations) {
237+
std::vector<std::thread> threads;
238+
for (int i = 0; i < 4; ++i) {
239+
threads.emplace_back([] {
240+
CudaAllocator& a = CudaAllocator::instance();
241+
for (int cycle = 0; cycle < 32; ++cycle) {
242+
for (size_t alignment : {256, 4096}) {
243+
auto res = a.allocate(1000, 0, alignment);
244+
ASSERT_TRUE(res.ok());
245+
EXPECT_EQ(reinterpret_cast<uintptr_t>(res.get()) % alignment, 0u);
246+
a.deallocate(res.get(), 0);
247+
}
248+
}
249+
});
250+
}
251+
for (auto& thread : threads) {
252+
thread.join();
253+
}
254+
}
255+
256+
// A device call would fail with Internal instead of the size validation error.
257+
TEST_F(CudaAllocatorTest, PaddingOverflowFailsBeforeAnyCudaCall) {
258+
if (!missing_device_fits()) {
259+
GTEST_SKIP() << "device count " << device_count_
260+
<< " leaves no absent ordinal in DeviceIndex";
261+
}
262+
CudaAllocator& a = CudaAllocator::instance();
263+
for (size_t alignment : {512, 4096}) {
264+
const size_t largest = std::numeric_limits<size_t>::max() - (alignment - 1);
265+
auto overflow = a.allocate(largest + 1, missing_device(), alignment);
266+
ASSERT_FALSE(overflow.ok());
267+
EXPECT_EQ(overflow.error(), Error::InvalidArgument);
268+
269+
auto fits = a.allocate(largest, missing_device(), alignment);
270+
ASSERT_FALSE(fits.ok());
271+
EXPECT_EQ(fits.error(), Error::Internal);
272+
}
273+
}
274+
275+
#if defined(__linux__) && !defined(EXECUTORCH_USE_HIP)
276+
TEST_F(CudaAllocatorTest, RoundedAllocationFreesOriginalPointer) {
277+
CudaAllocator& a = CudaAllocator::instance();
278+
// The default alignment is always met, so this round trip shows whether the
279+
// allocator reaches the stand-ins at all.
280+
hand_out_fake_block = true;
281+
freed_fake_block = nullptr;
282+
auto probe = a.allocate(1024, 0);
283+
const bool malloc_replaced = !hand_out_fake_block;
284+
hand_out_fake_block = false;
285+
if (probe.ok()) {
286+
a.deallocate(probe.get(), 0);
287+
}
288+
if (!malloc_replaced ||
289+
freed_fake_block != reinterpret_cast<void*>(kFakeBlock)) {
290+
GTEST_SKIP() << "cudaMalloc and cudaFree cannot be replaced in this build";
291+
}
292+
293+
for (size_t alignment : {512, 4096}) {
294+
for (int cycle = 0; cycle < 64; ++cycle) {
295+
freed_fake_block = nullptr;
296+
hand_out_fake_block = true;
297+
auto res = a.allocate(1024, 0, alignment);
298+
hand_out_fake_block = false;
299+
ASSERT_TRUE(res.ok());
300+
EXPECT_EQ(fake_malloc_size, 1024 + alignment - 1);
301+
EXPECT_EQ(res.get(), reinterpret_cast<void*>(alignment));
302+
EXPECT_EQ(freed_fake_block, nullptr);
303+
a.deallocate(res.get(), 0);
304+
EXPECT_EQ(freed_fake_block, reinterpret_cast<void*>(kFakeBlock));
305+
}
306+
307+
// Reusing the rounded address must not find a stale map entry.
308+
fake_block = alignment;
309+
freed_fake_block = nullptr;
310+
hand_out_fake_block = true;
311+
auto aligned = a.allocate(1024, 0, alignment);
312+
hand_out_fake_block = false;
313+
fake_block = kFakeBlock;
314+
ASSERT_TRUE(aligned.ok());
315+
EXPECT_EQ(aligned.get(), reinterpret_cast<void*>(alignment));
316+
EXPECT_EQ(freed_fake_block, nullptr);
317+
a.deallocate(aligned.get(), 0);
318+
EXPECT_EQ(freed_fake_block, reinterpret_cast<void*>(alignment));
319+
}
320+
}
321+
322+
TEST_F(CudaAllocatorTest, AllocateFreesAndRefusesAMisalignedPointer) {
323+
CudaAllocator& a = CudaAllocator::instance();
324+
hand_out_fake_block = true;
325+
freed_fake_block = nullptr;
326+
auto probe = a.allocate(1024, 0);
327+
const bool malloc_replaced = !hand_out_fake_block;
328+
hand_out_fake_block = false;
329+
if (probe.ok()) {
330+
a.deallocate(probe.get(), 0);
331+
}
332+
if (!malloc_replaced ||
333+
freed_fake_block != reinterpret_cast<void*>(kFakeBlock)) {
334+
GTEST_SKIP() << "cudaMalloc and cudaFree cannot be replaced in this build";
335+
}
336+
337+
freed_fake_block = nullptr;
338+
fake_block = 128;
339+
hand_out_fake_block = true;
340+
auto res = a.allocate(1024, 0, 256);
341+
hand_out_fake_block = false;
342+
fake_block = kFakeBlock;
343+
if (res.ok()) {
344+
a.deallocate(res.get(), 0);
345+
}
346+
ASSERT_FALSE(res.ok());
347+
EXPECT_EQ(res.error(), Error::NotSupported);
348+
EXPECT_EQ(fake_malloc_size, 1024u);
349+
EXPECT_EQ(freed_fake_block, reinterpret_cast<void*>(128));
350+
}
351+
#endif
352+
156353
TEST_F(CudaAllocatorTest, CopyHostToDeviceOnMissingDeviceFails) {
157354
if (!missing_device_fits()) {
158355
GTEST_SKIP() << "device count " << device_count_

0 commit comments

Comments
 (0)