Skip to content

Commit 255e9e0

Browse files
Remove unnecessary cuda sync for better perf (pytorch#17594)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: pytorch#17315 by @Gasoonjia ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/gasoonjia/116/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/gasoonjia/116/head Merge bot PR base: https://github.com/pytorch/executorch/tree/main Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/gasoonjia/116/orig Differential Revision: [D92193164](https://our.internmc.facebook.com/intern/diff/D92193164/) @diff-train-skip-merge Co-authored-by: gasoonjia <gasoonjia@icloud.com>
1 parent d6e8ad1 commit 255e9e0

7 files changed

Lines changed: 270 additions & 53 deletions

File tree

‎backends/aoti/aoti_delegate_handle.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,6 @@ struct AOTIDelegateHandle {
8484
void* so_handle;
8585
std::string so_path;
8686
AOTInductorModelContainerHandle container_handle;
87-
void* cuda_stream; // cudaStream_t stored as void* to avoid CUDA header
88-
// dependency
8987
std::string method_name;
9088

9189
// Function pointers specific to this handle's shared library

‎backends/aoti/slim/core/storage.h‎

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -127,16 +127,47 @@ struct DeviceTraits<c10::DeviceType::CUDA> {
127127
/// @param ptr Pointer to device memory to free.
128128
static void free(void* ptr) {
129129
// Get the current stream for the current device
130+
// Currently all cuda slimtensors should be on the same device same stream,
131+
// so we can just use the stream on current device.
132+
// TODO(gasoonjia): add cuda stream as a member of MaybeOwningStorage to
133+
// support multiple devices.
130134
auto stream_result = executorch::backends::cuda::getCurrentCUDAStream(-1);
131-
if (stream_result.ok()) {
132-
ET_CUDA_LOG_WARN(cudaFreeAsync(ptr, stream_result.get()));
135+
ET_CHECK_MSG(stream_result.ok(), "Failed to get current CUDA stream");
136+
ET_CUDA_LOG_WARN(cudaFreeAsync(ptr, stream_result.get()));
137+
}
138+
139+
/// Copies memory between CPU and CUDA or CUDA and CUDA asynchronously.
140+
/// @param dst Destination pointer.
141+
/// @param src Source pointer.
142+
/// @param nbytes Number of bytes to copy.
143+
/// @param dst_device Destination device.
144+
/// @param src_device Source device.
145+
/// @param stream CUDA stream for async copy.
146+
static void memcpy_async(
147+
void* dst,
148+
const void* src,
149+
size_t nbytes,
150+
const c10::Device& dst_device,
151+
const c10::Device& src_device,
152+
cudaStream_t stream) {
153+
cudaMemcpyKind direction = cudaMemcpyDeviceToDevice;
154+
155+
if (src_device.is_cpu()) {
156+
direction = cudaMemcpyHostToDevice;
157+
} else if (dst_device.is_cpu()) {
158+
direction = cudaMemcpyDeviceToHost;
133159
} else {
134-
// Fallback to synchronous free if we can't get the stream
135-
ET_CUDA_LOG_WARN(cudaFree(ptr));
160+
ET_CHECK_MSG(
161+
src_device.index() == dst_device.index(),
162+
"CUDA memcpy across different device indices not supported: %d != %d",
163+
static_cast<int>(src_device.index()),
164+
static_cast<int>(dst_device.index()));
136165
}
166+
167+
ET_CUDA_CHECK(cudaMemcpyAsync(dst, src, nbytes, direction, stream));
137168
}
138169

139-
/// Copies memory between CPU and CUDA or CUDA and CUDA.
170+
/// Copies memory between CPU and CUDA or CUDA and CUDA synchronously.
140171
/// @param dst Destination pointer.
141172
/// @param src Source pointer.
142173
/// @param nbytes Number of bytes to copy.

‎backends/cuda/runtime/TARGETS‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ runtime.cxx_library(
7575
srcs = [
7676
"cuda_backend.cpp",
7777
],
78+
headers = [
79+
"cuda_delegate_handle.h",
80+
],
7881
# @lint-ignore BUCKLINT: Avoid `link_whole=True` (https://fburl.com/avoid-link-whole)
7982
link_whole = True,
8083
supports_python_dlopen = True,

‎backends/cuda/runtime/cuda_backend.cpp‎

Lines changed: 92 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737

3838
// Include our shim layer headers
3939
#include <executorch/backends/aoti/aoti_delegate_handle.h>
40+
#include <executorch/backends/cuda/runtime/cuda_delegate_handle.h>
4041
#include <executorch/backends/cuda/runtime/platform/platform.h>
4142
#include <executorch/backends/cuda/runtime/shims/memory.h>
4243
#include <executorch/backends/cuda/runtime/utils.h>
@@ -77,6 +78,7 @@ using slim::c10::DeviceType;
7778
namespace {
7879
constexpr char kSkipCopyOutputToCpuForMethod[] =
7980
"skip_copy_output_to_cpu_for_method";
81+
constexpr char kUseSharedCudaStream[] = "use_shared_cuda_stream";
8082
} // anonymous namespace
8183

8284
class ET_EXPERIMENTAL CudaBackend final
@@ -143,6 +145,33 @@ class ET_EXPERIMENTAL CudaBackend final
143145
return method_in_csv(method_name, skip_copy_method_);
144146
}
145147

148+
// Create the shared CUDA stream. Called when use_shared_cuda_stream option
149+
// is set to true. The presence of shared_cuda_stream_ indicates shared mode.
150+
void create_shared_cuda_stream() {
151+
std::lock_guard<std::mutex> guard(cuda_stream_mutex_);
152+
if (shared_cuda_stream_ != nullptr) {
153+
return; // Already created
154+
}
155+
shared_cuda_stream_ = cuda::create_cuda_stream();
156+
if (shared_cuda_stream_ == nullptr) {
157+
ET_LOG(Error, "Failed to create shared CUDA stream");
158+
return;
159+
}
160+
ET_LOG(Info, "Created shared CUDA stream: %p", *shared_cuda_stream_);
161+
}
162+
163+
// Get the shared CUDA stream. Returns nullptr if not in shared mode.
164+
std::shared_ptr<cudaStream_t> get_shared_cuda_stream() const {
165+
std::lock_guard<std::mutex> guard(cuda_stream_mutex_);
166+
return shared_cuda_stream_;
167+
}
168+
169+
// Check if we're using shared CUDA stream mode.
170+
bool is_using_shared_cuda_stream() const {
171+
std::lock_guard<std::mutex> guard(cuda_stream_mutex_);
172+
return shared_cuda_stream_ != nullptr;
173+
}
174+
146175
Error load_function_pointers_into_handle(
147176
void* so_handle,
148177
AOTIDelegateHandle* handle) const {
@@ -201,6 +230,15 @@ class ET_EXPERIMENTAL CudaBackend final
201230
kSkipCopyOutputToCpuForMethod);
202231
return Error::InvalidArgument;
203232
}
233+
} else if (std::strcmp(option.key, kUseSharedCudaStream) == 0) {
234+
if (auto* val = std::get_if<bool>(&option.value)) {
235+
if (*val) {
236+
create_shared_cuda_stream();
237+
}
238+
} else {
239+
ET_LOG(Error, "Option %s must be a boolean.", kUseSharedCudaStream);
240+
return Error::InvalidArgument;
241+
}
204242
}
205243
}
206244
return Error::Ok;
@@ -282,7 +320,7 @@ class ET_EXPERIMENTAL CudaBackend final
282320
processed->Free();
283321

284322
// Create handle and load function pointers into it
285-
AOTIDelegateHandle* handle = new AOTIDelegateHandle();
323+
cuda::CudaDelegateHandle* handle = new cuda::CudaDelegateHandle();
286324
handle->so_handle = lib_handle;
287325
handle->so_path = so_path.string();
288326
handle->method_name = method_name;
@@ -313,10 +351,31 @@ class ET_EXPERIMENTAL CudaBackend final
313351
handle->container_handle, static_cast<const uint8_t*>(weights_blob)));
314352
buffer_res->Free();
315353
}
316-
// Create a CUDA stream for asynchronous execution
317-
cudaStream_t cuda_stream;
318-
ET_CUDA_CHECK_OR_RETURN_ERROR(cudaStreamCreate(&cuda_stream));
319-
handle->cuda_stream = static_cast<void*>(cuda_stream);
354+
355+
// Use shared CUDA stream if enabled via options, otherwise create one.
356+
// A shared stream ensures proper ordering across multiple methods
357+
// (e.g., encoder, decoder, sampler) when using skip-copy optimization.
358+
if (is_using_shared_cuda_stream()) {
359+
// Shared stream mode: all handles share the same stream.
360+
handle->cuda_stream = get_shared_cuda_stream();
361+
ET_LOG(
362+
Info,
363+
"Using shared CUDA stream %p for method %s",
364+
handle->get_cuda_stream(),
365+
method_name.c_str());
366+
} else {
367+
// Per-handle stream mode: each handle owns its own stream.
368+
handle->cuda_stream = cuda::create_cuda_stream();
369+
if (handle->cuda_stream == nullptr) {
370+
delete handle;
371+
return Error::Internal;
372+
}
373+
ET_LOG(
374+
Info,
375+
"Created new CUDA stream %p for method %s",
376+
handle->get_cuda_stream(),
377+
method_name.c_str());
378+
}
320379

321380
return (DelegateHandle*)handle; // Return the handle post-processing
322381
}
@@ -326,14 +385,16 @@ class ET_EXPERIMENTAL CudaBackend final
326385
BackendExecutionContext& context,
327386
DelegateHandle* handle_,
328387
Span<EValue*> args) const override {
329-
AOTIDelegateHandle* handle = (AOTIDelegateHandle*)handle_;
388+
cuda::CudaDelegateHandle* handle = (cuda::CudaDelegateHandle*)handle_;
330389

331390
size_t n_inputs;
332391
handle->get_num_inputs(handle->container_handle, &n_inputs);
333392

334393
size_t n_outputs;
335394
handle->get_num_outputs(handle->container_handle, &n_outputs);
336395

396+
setCurrentCUDAStream(handle->get_cuda_stream(), 0);
397+
337398
ET_CHECK_OR_RETURN_ERROR(
338399
n_inputs + n_outputs == args.size(),
339400
InvalidArgument,
@@ -406,13 +467,18 @@ class ET_EXPERIMENTAL CudaBackend final
406467
// expects ETensor* as input/output. We avoid changing its signature since
407468
// it's shared with the Metal backend. Instead, we reinterpret_cast
408469
// SlimTensor* to Tensor*
470+
//
471+
// Get current CUDA stream and pass it to AOTInductorModelContainerRun
472+
Result<cudaStream_t> cuda_stream_ret = getCurrentCUDAStream(0);
473+
cudaStream_t cuda_stream = cuda_stream_ret.get();
474+
ET_CHECK_OK_OR_RETURN_ERROR(cuda_stream_ret.error());
409475
AOTIRuntimeError error = handle->run(
410476
handle->container_handle,
411477
reinterpret_cast<Tensor**>(gpu_inputs.data()),
412478
n_inputs,
413479
reinterpret_cast<Tensor**>(gpu_outputs.data()),
414480
n_outputs,
415-
handle->cuda_stream,
481+
static_cast<void*>(cuda_stream),
416482
nullptr);
417483

418484
ET_CHECK_OR_RETURN_ERROR(
@@ -423,22 +489,12 @@ class ET_EXPERIMENTAL CudaBackend final
423489

424490
const bool copy_outputs = !should_skip_copy_for_method(handle->method_name);
425491

426-
// Synchronize CUDA stream to ensure kernel execution is complete
427-
// before accessing output data (either for copy or skip-copy path)
428-
cudaStream_t cuda_stream = static_cast<cudaStream_t>(handle->cuda_stream);
429-
cudaError_t sync_err = cudaStreamSynchronize(cuda_stream);
430-
ET_CHECK_OR_RETURN_ERROR(
431-
sync_err == cudaSuccess,
432-
Internal,
433-
"cudaStreamSynchronize failed: %s",
434-
cudaGetErrorString(sync_err));
435-
436492
if (copy_outputs) {
437-
// Deep copy GPU SlimTensor results back to CPU ETensors
438493
for (size_t i = 0; i < n_outputs; i++) {
439494
auto* cpu_output_tensor = &(args[i + n_inputs]->toTensor());
440495
ET_CHECK_OK_OR_RETURN_ERROR(
441-
copy_slimtensor_to_etensor(gpu_outputs[i], cpu_output_tensor),
496+
copy_slimtensor_to_etensor_async(
497+
gpu_outputs[i], cpu_output_tensor, cuda_stream),
442498
"Failed to copy GPU output %zu back to CPU ETensor",
443499
i);
444500
}
@@ -483,7 +539,7 @@ class ET_EXPERIMENTAL CudaBackend final
483539
if (handle_ == nullptr) {
484540
return;
485541
}
486-
AOTIDelegateHandle* handle = (AOTIDelegateHandle*)handle_;
542+
cuda::CudaDelegateHandle* handle = (cuda::CudaDelegateHandle*)handle_;
487543

488544
// Clean up cached output tensors for this handle
489545
{
@@ -495,16 +551,10 @@ class ET_EXPERIMENTAL CudaBackend final
495551
}
496552
}
497553

498-
// Destroy the CUDA stream if it exists
499-
if (handle->cuda_stream != nullptr) {
500-
cudaStream_t cuda_stream = static_cast<cudaStream_t>(handle->cuda_stream);
501-
cudaError_t stream_err = cudaStreamDestroy(cuda_stream);
502-
ET_CHECK_OR_LOG_ERROR(
503-
stream_err == cudaSuccess,
504-
"Failed to destroy CUDA stream: %s",
505-
cudaGetErrorString(stream_err));
506-
handle->cuda_stream = nullptr;
507-
}
554+
// The CUDA stream is managed by shared_ptr in the handle.
555+
// It will be automatically destroyed when the last handle using it
556+
// is destroyed. Just reset our reference.
557+
handle->cuda_stream.reset();
508558

509559
// NOTE: AOTInductorModelContainerDelete does not work correctly with
510560
// multiple .so files. Deleting one container frees shared resources,
@@ -541,13 +591,22 @@ class ET_EXPERIMENTAL CudaBackend final
541591
mutable std::mutex skip_copy_method_mutex_;
542592
std::string skip_copy_method_;
543593

594+
// Shared CUDA stream for all methods. When set (non-null), all methods use
595+
// the same stream to ensure proper ordering (critical for skip-copy
596+
// optimization). Created when use_shared_cuda_stream option is set to true.
597+
// Managed via shared_ptr so it's automatically cleaned up when last handle
598+
// is destroyed.
599+
mutable std::mutex cuda_stream_mutex_;
600+
std::shared_ptr<cudaStream_t> shared_cuda_stream_ = nullptr;
601+
544602
// Cached output tensors for skip-copy optimization.
545603
// When skip-copy is enabled, output SlimTensors are cached here to keep
546604
// the underlying GPU memory alive while the caller processes the results.
547-
// Maps each AOTIDelegateHandle* to its vector of cached output tensors.
605+
// Maps each CudaDelegateHandle* to its vector of cached output tensors.
548606
mutable std::mutex cached_outputs_mutex_;
549-
mutable std::unordered_map<AOTIDelegateHandle*, std::vector<SlimTensor*>>
550-
cached_outputs_;
607+
mutable std::
608+
unordered_map<cuda::CudaDelegateHandle*, std::vector<SlimTensor*>>
609+
cached_outputs_;
551610
};
552611

553612
} // namespace executorch::backends::cuda
Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#pragma once
10+
11+
#include <cuda_runtime.h>
12+
#include <executorch/backends/aoti/aoti_delegate_handle.h>
13+
#include <memory>
14+
15+
namespace executorch {
16+
namespace backends {
17+
namespace cuda {
18+
19+
// Shared CUDA stream wrapper with proper RAII cleanup.
20+
// This ensures the stream is destroyed when all handles using it are destroyed.
21+
struct CudaStreamDeleter {
22+
void operator()(cudaStream_t* stream) const {
23+
if (stream != nullptr && *stream != nullptr) {
24+
cudaStreamDestroy(*stream);
25+
}
26+
delete stream;
27+
}
28+
};
29+
30+
// Creates a new shared CUDA stream.
31+
// Returns nullptr on failure.
32+
inline std::shared_ptr<cudaStream_t> create_cuda_stream() {
33+
cudaStream_t stream;
34+
cudaError_t err = cudaStreamCreate(&stream);
35+
if (err != cudaSuccess) {
36+
return nullptr;
37+
}
38+
return std::shared_ptr<cudaStream_t>(
39+
new cudaStream_t(stream), CudaStreamDeleter());
40+
}
41+
// CUDA-specific delegate handle that extends AOTIDelegateHandle.
42+
// This consolidates CUDA stream management into a single location.
43+
struct CudaDelegateHandle : public aoti::AOTIDelegateHandle {
44+
// CUDA stream for this handle, support both shared mode and single mode.
45+
// In shared mode, all cuda delegate handles share the same stream (e.g., for
46+
// skip-copy optimization), they will all hold a reference to the same
47+
// shared_ptr. The stream is automatically destroyed when the last handle is
48+
// destroyed. In single mode, every cuda delegate handle has its own stream.
49+
std::shared_ptr<cudaStream_t> cuda_stream;
50+
51+
// Get the raw CUDA stream pointer for use in CUDA API calls.
52+
// Returns nullptr if no stream is set.
53+
cudaStream_t get_cuda_stream() const {
54+
return cuda_stream ? *cuda_stream : nullptr;
55+
}
56+
57+
// Check if this handle has a valid CUDA stream.
58+
bool has_cuda_stream() const {
59+
return cuda_stream != nullptr && *cuda_stream != nullptr;
60+
}
61+
};
62+
63+
} // namespace cuda
64+
} // namespace backends
65+
} // namespace executorch

0 commit comments

Comments
 (0)