Skip to content

Commit 27d124f

Browse files
authored
Fix exit safety, package linking and header hygiene in the shared CUDA allocator (pytorch#23499)
Freeing CUDA memory during program exit can access allocator state after it has been destroyed. Wheel consumers can gain an unnecessary dependency on the development toolkit's CUDA runtime. The allocator's public header also carries private bookkeeping that callers never use. This change keeps the allocator, its rounded pointer map and its pool table alive until the process exits, so a late free or pool trim from another library's static destructor still works. Construction is restricted to `instance()`, and a compile time check confirms that the backend name is the same class. The wheel package exports the toolkit include directories without linking another CUDA runtime, adding them once even when `find_package` runs again. A program that calls the CUDA runtime itself now links `CUDA::cudart`, which the C++ guide states. The rounded pointer map moves into the source file, and deallocation skips its lock while no rounded pointer is live. Alignments above 256 bytes still use one padded `cudaMalloc`, because retrying a misaligned block would free it first, and `cudaFree` waits for all work queued on the device. Compatibility: `CudaAllocator` can no longer be constructed directly; use `CudaAllocator::instance()`. Code that used `std::atomic`, `std::mutex` or `std::unordered_map` through `cuda_allocator.h` must include those headers itself. Validation on Linux x86_64 with CUDA 12.8: - The allocator suite and four other CUDA suites pass. Stand-in tests check that `allocate` makes one `cudaMalloc` call, never calls `cudaFree` on the padded path, frees a rounded pointer through its original block, and refuses a block that misses an alignment of 256 or less. - The exit test frees a rounded block and trims the pool from an exit handler. It passes 20 runs in Release and Debug builds. Reverting the lifetime of the allocator, its pool table or its pointer map each makes it fail in an optimized build. - Compute Sanitizer reports no errors or leaks in the allocator suite. - Wheel consumers build and run with early and late toolkit discovery, and an empty consumer records no direct `libcudart` dependency. Windows and ROCm were not run locally; this pull request's CI covers both.
1 parent 4567831 commit 27d124f

10 files changed

Lines changed: 390 additions & 134 deletions

File tree

‎.ci/scripts/wheel/test_cpp_sdk.py‎

Lines changed: 47 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1349,27 +1349,52 @@ def test_every_shipped_header_compiles(work_dir: Path) -> None:
13491349
)
13501350

13511351

1352-
def test_cuda_allocator_component(work_dir: Path) -> None:
1353-
"""The CUDA target must supply everything needed to use its allocator."""
1352+
def test_cuda_allocator_component(
1353+
work_dir: Path, *, late_toolkit: bool = False
1354+
) -> None:
1355+
"""Supply allocator headers without making consumers link another CUDA runtime."""
13541356
package = _installed_package_dir()
13551357
if not any(
13561358
(package / "lib").glob(_library_file_name("libexecutorch_extension_cuda") + "*")
13571359
):
13581360
print("- no CUDA extension is shipped; skipping its allocator consumer")
13591361
return
13601362

1361-
source_dir = work_dir / "cuda-allocator-consumer"
1363+
name = "cuda-allocator-late-toolkit" if late_toolkit else "cuda-allocator-consumer"
1364+
source_dir = work_dir / name
13621365
source_dir.mkdir(parents=True, exist_ok=True)
1363-
(source_dir / "consumer.cpp").write_text(
1366+
allocator_dir = source_dir / "child" if late_toolkit else source_dir
1367+
allocator_dir.mkdir(exist_ok=True)
1368+
(allocator_dir / "consumer.cpp").write_text(
13641369
"#include <executorch/extension/cuda/cuda_allocator.h>\n"
13651370
"using namespace executorch::extension::cuda;\n"
13661371
"int main() {\n"
13671372
" return CudaAllocator::instance().device_type() ==\n"
13681373
" executorch::runtime::etensor::DeviceType::CUDA ? 0 : 1;\n"
13691374
"}\n"
13701375
)
1371-
(source_dir / "CMakeLists.txt").write_text(_consumer_cmake(["extension_cuda"]))
1372-
build_dir = work_dir / "cuda-allocator-consumer-build"
1376+
consumer_cmake = _consumer_cmake(["extension_cuda"])
1377+
if late_toolkit:
1378+
(allocator_dir / "CMakeLists.txt").write_text(
1379+
"set(CMAKE_DISABLE_FIND_PACKAGE_CUDAToolkit FALSE)\n" + consumer_cmake
1380+
)
1381+
consumer_cmake = (
1382+
"cmake_minimum_required(VERSION 3.28)\n"
1383+
"project(consumer CXX)\n"
1384+
"set(CMAKE_DISABLE_FIND_PACKAGE_CUDAToolkit TRUE)\n"
1385+
"find_package(executorch REQUIRED COMPONENTS extension_cuda)\n"
1386+
"add_subdirectory(child)\n"
1387+
)
1388+
(source_dir / "empty.cpp").write_text("int main() { return 0; }\n")
1389+
consumer_cmake += (
1390+
"add_executable(empty empty.cpp)\n"
1391+
"target_link_libraries(empty PRIVATE executorch::extension_cuda)\n"
1392+
'if(CMAKE_SYSTEM_NAME STREQUAL "Linux")\n'
1393+
' target_link_options(empty PRIVATE "-Wl,--no-as-needed")\n'
1394+
"endif()\n"
1395+
)
1396+
(source_dir / "CMakeLists.txt").write_text(consumer_cmake)
1397+
build_dir = work_dir / f"{name}-build"
13731398
configure = [
13741399
_tool("cmake"),
13751400
"-S",
@@ -1386,13 +1411,26 @@ def test_cuda_allocator_component(work_dir: Path) -> None:
13861411
"a consumer linking only executorch::extension_cuda could not build:\n"
13871412
f"{result.stdout[-2000:]}\n{result.stderr[-2000:]}"
13881413
)
1414+
allocator_build = build_dir / "child" if late_toolkit else build_dir
13891415
subprocess.run(
1390-
[str(_executable(build_dir, "consumer"))],
1416+
[str(_executable(allocator_build, "consumer"))],
13911417
env=_loader_clean_environment(),
13921418
check=True,
13931419
timeout=_RUN_TIMEOUT,
13941420
)
1395-
print("CUDA allocator compiles, links and runs through executorch::extension_cuda")
1421+
if sys.platform.startswith("linux"):
1422+
empty = _executable(build_dir, "empty")
1423+
dynamic = _recorded_dependencies(empty)
1424+
assert not any(
1425+
"NEEDED" in line and "libcudart" in line for line in dynamic.splitlines()
1426+
), f"an empty CUDA component consumer directly links the toolkit runtime:\n{dynamic}"
1427+
subprocess.run(
1428+
[str(empty)],
1429+
env=_loader_clean_environment(),
1430+
check=True,
1431+
timeout=_RUN_TIMEOUT,
1432+
)
1433+
print(f"✓ CUDA allocator consumer passes (late toolkit: {late_toolkit})")
13961434

13971435

13981436
def test_shipped_headers_have_implementations(work_dir: Path) -> None:
@@ -2005,6 +2043,7 @@ def run_tests(work_dir: Path) -> None:
20052043
test_profiler_component_is_usable(work_dir)
20062044
test_every_shipped_header_compiles(work_dir)
20072045
test_cuda_allocator_component(work_dir)
2046+
test_cuda_allocator_component(work_dir, late_toolkit=True)
20082047
test_shipped_headers_have_implementations(work_dir)
20092048
test_documented_example_compiles(work_dir)
20102049
test_runtime_alone_links_but_cannot_compute(work_dir)

‎.ci/scripts/wheel/test_shared_libraries.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,10 @@
7979
# strong one elsewhere, so naming one of those would count a definition that may not be the one
8080
# the process uses.
8181
_CUDA_BACKEND_SYMBOLS = ("executorch::backends::cuda::load_library",)
82-
_CUDA_STREAM_SYMBOLS = ("executorch::extension::cuda::getCallerStream",)
82+
_CUDA_EXTENSION_SYMBOLS = (
83+
"executorch::extension::cuda::getCallerStream",
84+
"executorch::extension::cuda::CudaAllocator::instance",
85+
)
8386

8487
# The AOTI shim layer and the stream-guard state that lives with it. This is the state that was
8588
# genuinely duplicated: extracting the shims with a PUBLIC whole-archive replayed the extraction at
@@ -1236,8 +1239,8 @@ def _resolve_required(required):
12361239
_REQUIRED_ON_A_CUDA_WHEEL,
12371240
),
12381241
(
1239-
"CUDA stream helper",
1240-
_CUDA_STREAM_SYMBOLS,
1242+
"CUDA allocator and stream helpers",
1243+
_CUDA_EXTENSION_SYMBOLS,
12411244
_library_file_name("libexecutorch_extension_cuda"),
12421245
_REQUIRED_ON_A_CUDA_WHEEL,
12431246
),

‎.github/workflows/cuda.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -577,7 +577,7 @@ jobs:
577577
-v -o "addopts="
578578
579579
cmake --preset llm-release-cuda -DEXECUTORCH_BUILD_TESTS=ON
580-
cmake --build cmake-out --target test_cuda_allocator test_cuda_mutable_state test_cuda_weight_cache test_cuda_guard test_cuda_stream_guard -j$(nproc)
580+
cmake --build cmake-out --target test_cuda_allocator test_cuda_allocator_exit test_cuda_mutable_state test_cuda_weight_cache test_cuda_guard test_cuda_stream_guard -j$(nproc)
581581
ctest --test-dir cmake-out -R test_cuda_allocator --output-on-failure -V
582582
ctest --test-dir cmake-out -R test_cuda_mutable_state --output-on-failure -V
583583
ctest --test-dir cmake-out -R test_cuda_weight_cache --output-on-failure -V

‎backends/cuda/runtime/test/test_cuda_allocator.cpp‎

Lines changed: 114 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include <cstdint>
1414
#include <limits>
1515
#include <thread>
16+
#include <type_traits>
1617
#include <vector>
1718

1819
#if defined(__linux__) && !defined(EXECUTORCH_USE_HIP)
@@ -31,20 +32,21 @@ using executorch::runtime::etensor::DeviceIndex;
3132
#if defined(__linux__) && !defined(EXECUTORCH_USE_HIP)
3233
// Force rounding independently of how the GPU packs its allocations.
3334
namespace {
34-
constexpr uintptr_t kFakeBlock = 256;
35-
thread_local uintptr_t fake_block = kFakeBlock;
36-
thread_local bool hand_out_fake_block = false;
35+
alignas(4096) thread_local char fake_storage[16384];
36+
thread_local void* fake_block = fake_storage + 256;
37+
thread_local int fake_malloc_remaining = 0;
3738
thread_local void* freed_fake_block = nullptr;
3839
thread_local size_t fake_malloc_size = 0;
40+
thread_local int fake_free_count = 0;
3941
} // namespace
4042

4143
// Weak, so the real ones win if a static CUDA runtime is linked in. The linker
4244
// can still leave them out, and then a stand-in has nothing to call.
4345
extern "C" __attribute__((weak)) cudaError_t
4446
cudaMalloc(void** ptr, size_t size) {
45-
if (hand_out_fake_block) {
46-
hand_out_fake_block = false;
47-
*ptr = reinterpret_cast<void*>(fake_block);
47+
if (fake_malloc_remaining > 0) {
48+
--fake_malloc_remaining;
49+
*ptr = fake_block;
4850
fake_malloc_size = size;
4951
return cudaSuccess;
5052
}
@@ -58,8 +60,11 @@ cudaMalloc(void** ptr, size_t size) {
5860
}
5961

6062
extern "C" __attribute__((weak)) cudaError_t cudaFree(void* ptr) {
61-
if (ptr != nullptr && reinterpret_cast<uintptr_t>(ptr) <= 4096) {
63+
const auto address = reinterpret_cast<uintptr_t>(ptr);
64+
const auto begin = reinterpret_cast<uintptr_t>(fake_storage);
65+
if (address >= begin && address < begin + sizeof(fake_storage)) {
6266
freed_fake_block = ptr;
67+
++fake_free_count;
6368
return cudaSuccess;
6469
}
6570
static const auto real =
@@ -110,10 +115,16 @@ class CudaAllocatorTest : public testing::Test {
110115
int device_count_ = 0;
111116
};
112117

113-
TEST(CudaAllocatorCompatibilityTest, BackendAliasSharesTheExtensionSingleton) {
114-
EXPECT_EQ(
115-
&CudaAllocator::instance(),
116-
&executorch::extension::cuda::CudaAllocator::instance());
118+
// A distinct backend class would bring back a second allocator singleton.
119+
static_assert(
120+
std::is_same_v<CudaAllocator, executorch::extension::cuda::CudaAllocator>);
121+
122+
TEST(CudaAllocatorCompatibilityTest, OnlyTheSingletonCanBeConstructed) {
123+
EXPECT_FALSE(std::is_default_constructible_v<CudaAllocator>);
124+
EXPECT_FALSE(std::is_copy_constructible_v<CudaAllocator>);
125+
EXPECT_FALSE(std::is_move_constructible_v<CudaAllocator>);
126+
EXPECT_FALSE(std::is_copy_assignable_v<CudaAllocator>);
127+
EXPECT_FALSE(std::is_move_assignable_v<CudaAllocator>);
117128
}
118129

119130
TEST_F(CudaAllocatorTest, CopyRoundtrip) {
@@ -209,6 +220,7 @@ TEST_F(CudaAllocatorTest, AllocateOnMissingDeviceFails) {
209220
}
210221

211222
TEST_F(CudaAllocatorTest, LargeAlignmentsWithLiveBlocksRoundtrip) {
223+
(void)cudaGetLastError();
212224
CudaAllocator& a = CudaAllocator::instance();
213225
for (int cycle = 0; cycle < 64; ++cycle) {
214226
std::vector<void*> live;
@@ -235,12 +247,14 @@ TEST_F(CudaAllocatorTest, LargeAlignmentsWithLiveBlocksRoundtrip) {
235247
}
236248
for (void* ptr : live) {
237249
a.deallocate(ptr, 0);
250+
EXPECT_EQ(cudaGetLastError(), cudaSuccess);
238251
}
239252
}
240253
}
241254

242255
TEST_F(CudaAllocatorTest, ConcurrentAlignedAllocations) {
243256
std::vector<std::thread> threads;
257+
threads.reserve(4);
244258
for (int i = 0; i < 4; ++i) {
245259
threads.emplace_back([] {
246260
CudaAllocator& a = CudaAllocator::instance();
@@ -250,6 +264,7 @@ TEST_F(CudaAllocatorTest, ConcurrentAlignedAllocations) {
250264
ASSERT_TRUE(res.ok());
251265
EXPECT_EQ(reinterpret_cast<uintptr_t>(res.get()) % alignment, 0u);
252266
a.deallocate(res.get(), 0);
267+
EXPECT_EQ(cudaGetLastError(), cudaSuccess);
253268
}
254269
}
255270
});
@@ -279,80 +294,108 @@ TEST_F(CudaAllocatorTest, PaddingOverflowFailsBeforeAnyCudaCall) {
279294
}
280295

281296
#if defined(__linux__) && !defined(EXECUTORCH_USE_HIP)
282-
TEST_F(CudaAllocatorTest, RoundedAllocationFreesOriginalPointer) {
283-
CudaAllocator& a = CudaAllocator::instance();
284-
// The default alignment is always met, so this round trip shows whether the
285-
// allocator reaches the stand-ins at all.
286-
hand_out_fake_block = true;
287-
freed_fake_block = nullptr;
288-
auto probe = a.allocate(1024, 0);
289-
const bool malloc_replaced = !hand_out_fake_block;
290-
hand_out_fake_block = false;
291-
if (probe.ok()) {
292-
a.deallocate(probe.get(), 0);
297+
class CudaAllocatorStandInTest : public CudaAllocatorTest {
298+
protected:
299+
void SetUp() override {
300+
CudaAllocatorTest::SetUp();
301+
if (IsSkipped()) {
302+
return;
303+
}
304+
auto& a = CudaAllocator::instance();
305+
fake_block = fake_storage + 256;
306+
fake_malloc_remaining = 1;
307+
freed_fake_block = nullptr;
308+
auto probe = a.allocate(1024, 0);
309+
const bool malloc_replaced = fake_malloc_remaining == 0;
310+
fake_malloc_remaining = 0;
311+
if (probe.ok()) {
312+
a.deallocate(probe.get(), 0);
313+
}
314+
if (!malloc_replaced || freed_fake_block != fake_block) {
315+
GTEST_SKIP()
316+
<< "cudaMalloc and cudaFree cannot be replaced in this build";
317+
}
318+
fake_malloc_size = 0;
319+
fake_free_count = 0;
293320
}
294-
if (!malloc_replaced ||
295-
freed_fake_block != reinterpret_cast<void*>(kFakeBlock)) {
296-
GTEST_SKIP() << "cudaMalloc and cudaFree cannot be replaced in this build";
321+
322+
void TearDown() override {
323+
fake_malloc_remaining = 0;
324+
CudaAllocatorTest::TearDown();
297325
}
326+
};
298327

299-
for (size_t alignment : {512, 4096}) {
300-
for (int cycle = 0; cycle < 64; ++cycle) {
301-
freed_fake_block = nullptr;
302-
hand_out_fake_block = true;
328+
// A free inside allocate() would wait for all work queued on the device.
329+
TEST_F(CudaAllocatorStandInTest, AllocateMakesOneCallAndNeverFrees) {
330+
auto& a = CudaAllocator::instance();
331+
for (size_t alignment : {64, 256, 512, 4096}) {
332+
for (size_t offset : {0, 256}) {
333+
SCOPED_TRACE(testing::Message() << alignment << ", " << offset);
334+
fake_block = fake_storage + offset;
335+
fake_malloc_remaining = 1;
336+
fake_malloc_size = 0;
337+
fake_free_count = 0;
303338
auto res = a.allocate(1024, 0, alignment);
304-
hand_out_fake_block = false;
305339
ASSERT_TRUE(res.ok());
306-
EXPECT_EQ(fake_malloc_size, 1024 + alignment - 1);
307-
EXPECT_EQ(res.get(), reinterpret_cast<void*>(alignment));
308-
EXPECT_EQ(freed_fake_block, nullptr);
340+
EXPECT_EQ(fake_malloc_remaining, 0);
341+
EXPECT_EQ(fake_free_count, 0);
342+
EXPECT_EQ(
343+
fake_malloc_size, alignment > 256 ? 1024 + alignment - 1 : 1024);
344+
EXPECT_EQ(reinterpret_cast<uintptr_t>(res.get()) % alignment, 0u);
309345
a.deallocate(res.get(), 0);
310-
EXPECT_EQ(freed_fake_block, reinterpret_cast<void*>(kFakeBlock));
346+
EXPECT_EQ(freed_fake_block, fake_block);
347+
EXPECT_EQ(fake_free_count, 1);
311348
}
312-
313-
// Reusing the rounded address must not find a stale map entry.
314-
fake_block = alignment;
315-
freed_fake_block = nullptr;
316-
hand_out_fake_block = true;
317-
auto aligned = a.allocate(1024, 0, alignment);
318-
hand_out_fake_block = false;
319-
fake_block = kFakeBlock;
320-
ASSERT_TRUE(aligned.ok());
321-
EXPECT_EQ(aligned.get(), reinterpret_cast<void*>(alignment));
322-
EXPECT_EQ(freed_fake_block, nullptr);
323-
a.deallocate(aligned.get(), 0);
324-
EXPECT_EQ(freed_fake_block, reinterpret_cast<void*>(alignment));
325349
}
326350
}
327351

328-
TEST_F(CudaAllocatorTest, AllocateFreesAndRefusesAMisalignedPointer) {
329-
CudaAllocator& a = CudaAllocator::instance();
330-
hand_out_fake_block = true;
331-
freed_fake_block = nullptr;
332-
auto probe = a.allocate(1024, 0);
333-
const bool malloc_replaced = !hand_out_fake_block;
334-
hand_out_fake_block = false;
335-
if (probe.ok()) {
336-
a.deallocate(probe.get(), 0);
337-
}
338-
if (!malloc_replaced ||
339-
freed_fake_block != reinterpret_cast<void*>(kFakeBlock)) {
340-
GTEST_SKIP() << "cudaMalloc and cudaFree cannot be replaced in this build";
341-
}
342-
343-
freed_fake_block = nullptr;
344-
fake_block = 128;
345-
hand_out_fake_block = true;
352+
// cudaMalloc promises 256 bytes, so a block that misses a smaller alignment
353+
// is a broken runtime, not something padding should hide.
354+
TEST_F(CudaAllocatorStandInTest, MisalignedBlockAtSmallAlignmentIsRefused) {
355+
auto& a = CudaAllocator::instance();
356+
fake_block = fake_storage + 128;
357+
fake_malloc_remaining = 1;
358+
fake_malloc_size = 0;
359+
fake_free_count = 0;
346360
auto res = a.allocate(1024, 0, 256);
347-
hand_out_fake_block = false;
348-
fake_block = kFakeBlock;
349-
if (res.ok()) {
350-
a.deallocate(res.get(), 0);
351-
}
352361
ASSERT_FALSE(res.ok());
353362
EXPECT_EQ(res.error(), Error::NotSupported);
354363
EXPECT_EQ(fake_malloc_size, 1024u);
355-
EXPECT_EQ(freed_fake_block, reinterpret_cast<void*>(128));
364+
EXPECT_EQ(fake_free_count, 1);
365+
EXPECT_EQ(freed_fake_block, fake_block);
366+
}
367+
368+
TEST_F(CudaAllocatorStandInTest, RoundedAllocationFreesOriginalPointer) {
369+
auto& a = CudaAllocator::instance();
370+
for (size_t alignment : {512, 4096}) {
371+
for (int cycle = 0; cycle < 64; ++cycle) {
372+
fake_block = fake_storage + 256;
373+
fake_malloc_remaining = 1;
374+
fake_free_count = 0;
375+
auto res = a.allocate(1024, 0, alignment);
376+
ASSERT_TRUE(res.ok());
377+
EXPECT_EQ(res.get(), fake_storage + alignment);
378+
a.deallocate(res.get(), 0);
379+
EXPECT_EQ(freed_fake_block, fake_block);
380+
EXPECT_EQ(fake_free_count, 1);
381+
}
382+
383+
// A second live rounded block keeps the lock-free path from skipping the
384+
// lookup, so a stale entry for the reused address would be found.
385+
fake_block = fake_storage + alignment + 256;
386+
fake_malloc_remaining = 1;
387+
auto live = a.allocate(1024, 0, alignment);
388+
ASSERT_TRUE(live.ok());
389+
fake_block = fake_storage + alignment;
390+
fake_malloc_remaining = 1;
391+
auto reused = a.allocate(1024, 0, alignment);
392+
ASSERT_TRUE(reused.ok());
393+
EXPECT_EQ(reused.get(), fake_block);
394+
a.deallocate(reused.get(), 0);
395+
EXPECT_EQ(freed_fake_block, fake_block);
396+
a.deallocate(live.get(), 0);
397+
EXPECT_EQ(freed_fake_block, fake_storage + alignment + 256);
398+
}
356399
}
357400
#endif
358401

0 commit comments

Comments
 (0)