3737
3838// Include our shim layer headers
3939#include < executorch/backends/aoti/aoti_delegate_handle.h>
40+ #include < executorch/backends/cuda/runtime/cuda_delegate_handle.h>
4041#include < executorch/backends/cuda/runtime/platform/platform.h>
4142#include < executorch/backends/cuda/runtime/shims/memory.h>
4243#include < executorch/backends/cuda/runtime/utils.h>
@@ -77,6 +78,7 @@ using slim::c10::DeviceType;
7778namespace {
7879constexpr char kSkipCopyOutputToCpuForMethod [] =
7980 " skip_copy_output_to_cpu_for_method" ;
81+ constexpr char kUseSharedCudaStream [] = " use_shared_cuda_stream" ;
8082} // anonymous namespace
8183
8284class ET_EXPERIMENTAL CudaBackend final
@@ -143,6 +145,33 @@ class ET_EXPERIMENTAL CudaBackend final
143145 return method_in_csv (method_name, skip_copy_method_);
144146 }
145147
148+ // Create the shared CUDA stream. Called when use_shared_cuda_stream option
149+ // is set to true. The presence of shared_cuda_stream_ indicates shared mode.
150+ void create_shared_cuda_stream () {
151+ std::lock_guard<std::mutex> guard (cuda_stream_mutex_);
152+ if (shared_cuda_stream_ != nullptr ) {
153+ return ; // Already created
154+ }
155+ shared_cuda_stream_ = cuda::create_cuda_stream ();
156+ if (shared_cuda_stream_ == nullptr ) {
157+ ET_LOG (Error, " Failed to create shared CUDA stream" );
158+ return ;
159+ }
160+ ET_LOG (Info, " Created shared CUDA stream: %p" , *shared_cuda_stream_);
161+ }
162+
163+ // Get the shared CUDA stream. Returns nullptr if not in shared mode.
164+ std::shared_ptr<cudaStream_t> get_shared_cuda_stream () const {
165+ std::lock_guard<std::mutex> guard (cuda_stream_mutex_);
166+ return shared_cuda_stream_;
167+ }
168+
169+ // Check if we're using shared CUDA stream mode.
170+ bool is_using_shared_cuda_stream () const {
171+ std::lock_guard<std::mutex> guard (cuda_stream_mutex_);
172+ return shared_cuda_stream_ != nullptr ;
173+ }
174+
146175 Error load_function_pointers_into_handle (
147176 void * so_handle,
148177 AOTIDelegateHandle* handle) const {
@@ -201,6 +230,15 @@ class ET_EXPERIMENTAL CudaBackend final
201230 kSkipCopyOutputToCpuForMethod );
202231 return Error::InvalidArgument;
203232 }
233+ } else if (std::strcmp (option.key , kUseSharedCudaStream ) == 0 ) {
234+ if (auto * val = std::get_if<bool >(&option.value )) {
235+ if (*val) {
236+ create_shared_cuda_stream ();
237+ }
238+ } else {
239+ ET_LOG (Error, " Option %s must be a boolean." , kUseSharedCudaStream );
240+ return Error::InvalidArgument;
241+ }
204242 }
205243 }
206244 return Error::Ok;
@@ -282,7 +320,7 @@ class ET_EXPERIMENTAL CudaBackend final
282320 processed->Free ();
283321
284322 // Create handle and load function pointers into it
285- AOTIDelegateHandle * handle = new AOTIDelegateHandle ();
323+ cuda::CudaDelegateHandle * handle = new cuda::CudaDelegateHandle ();
286324 handle->so_handle = lib_handle;
287325 handle->so_path = so_path.string ();
288326 handle->method_name = method_name;
@@ -313,10 +351,31 @@ class ET_EXPERIMENTAL CudaBackend final
313351 handle->container_handle , static_cast <const uint8_t *>(weights_blob)));
314352 buffer_res->Free ();
315353 }
316- // Create a CUDA stream for asynchronous execution
317- cudaStream_t cuda_stream;
318- ET_CUDA_CHECK_OR_RETURN_ERROR (cudaStreamCreate (&cuda_stream));
319- handle->cuda_stream = static_cast <void *>(cuda_stream);
354+
355+ // Use shared CUDA stream if enabled via options, otherwise create one.
356+ // A shared stream ensures proper ordering across multiple methods
357+ // (e.g., encoder, decoder, sampler) when using skip-copy optimization.
358+ if (is_using_shared_cuda_stream ()) {
359+ // Shared stream mode: all handles share the same stream.
360+ handle->cuda_stream = get_shared_cuda_stream ();
361+ ET_LOG (
362+ Info,
363+ " Using shared CUDA stream %p for method %s" ,
364+ handle->get_cuda_stream (),
365+ method_name.c_str ());
366+ } else {
367+ // Per-handle stream mode: each handle owns its own stream.
368+ handle->cuda_stream = cuda::create_cuda_stream ();
369+ if (handle->cuda_stream == nullptr ) {
370+ delete handle;
371+ return Error::Internal;
372+ }
373+ ET_LOG (
374+ Info,
375+ " Created new CUDA stream %p for method %s" ,
376+ handle->get_cuda_stream (),
377+ method_name.c_str ());
378+ }
320379
321380 return (DelegateHandle*)handle; // Return the handle post-processing
322381 }
@@ -326,14 +385,16 @@ class ET_EXPERIMENTAL CudaBackend final
326385 BackendExecutionContext& context,
327386 DelegateHandle* handle_,
328387 Span<EValue*> args) const override {
329- AOTIDelegateHandle * handle = (AOTIDelegateHandle *)handle_;
388+ cuda::CudaDelegateHandle * handle = (cuda::CudaDelegateHandle *)handle_;
330389
331390 size_t n_inputs;
332391 handle->get_num_inputs (handle->container_handle , &n_inputs);
333392
334393 size_t n_outputs;
335394 handle->get_num_outputs (handle->container_handle , &n_outputs);
336395
396+ setCurrentCUDAStream (handle->get_cuda_stream (), 0 );
397+
337398 ET_CHECK_OR_RETURN_ERROR (
338399 n_inputs + n_outputs == args.size (),
339400 InvalidArgument,
@@ -406,13 +467,18 @@ class ET_EXPERIMENTAL CudaBackend final
406467 // expects ETensor* as input/output. We avoid changing its signature since
407468 // it's shared with the Metal backend. Instead, we reinterpret_cast
408469 // SlimTensor* to Tensor*
470+ //
471+ // Get current CUDA stream and pass it to AOTInductorModelContainerRun
472+ Result<cudaStream_t> cuda_stream_ret = getCurrentCUDAStream (0 );
473+ cudaStream_t cuda_stream = cuda_stream_ret.get ();
474+ ET_CHECK_OK_OR_RETURN_ERROR (cuda_stream_ret.error ());
409475 AOTIRuntimeError error = handle->run (
410476 handle->container_handle ,
411477 reinterpret_cast <Tensor**>(gpu_inputs.data ()),
412478 n_inputs,
413479 reinterpret_cast <Tensor**>(gpu_outputs.data ()),
414480 n_outputs,
415- handle-> cuda_stream ,
481+ static_cast < void *>( cuda_stream) ,
416482 nullptr );
417483
418484 ET_CHECK_OR_RETURN_ERROR (
@@ -423,22 +489,12 @@ class ET_EXPERIMENTAL CudaBackend final
423489
424490 const bool copy_outputs = !should_skip_copy_for_method (handle->method_name );
425491
426- // Synchronize CUDA stream to ensure kernel execution is complete
427- // before accessing output data (either for copy or skip-copy path)
428- cudaStream_t cuda_stream = static_cast <cudaStream_t>(handle->cuda_stream );
429- cudaError_t sync_err = cudaStreamSynchronize (cuda_stream);
430- ET_CHECK_OR_RETURN_ERROR (
431- sync_err == cudaSuccess,
432- Internal,
433- " cudaStreamSynchronize failed: %s" ,
434- cudaGetErrorString (sync_err));
435-
436492 if (copy_outputs) {
437- // Deep copy GPU SlimTensor results back to CPU ETensors
438493 for (size_t i = 0 ; i < n_outputs; i++) {
439494 auto * cpu_output_tensor = &(args[i + n_inputs]->toTensor ());
440495 ET_CHECK_OK_OR_RETURN_ERROR (
441- copy_slimtensor_to_etensor (gpu_outputs[i], cpu_output_tensor),
496+ copy_slimtensor_to_etensor_async (
497+ gpu_outputs[i], cpu_output_tensor, cuda_stream),
442498 " Failed to copy GPU output %zu back to CPU ETensor" ,
443499 i);
444500 }
@@ -483,7 +539,7 @@ class ET_EXPERIMENTAL CudaBackend final
483539 if (handle_ == nullptr ) {
484540 return ;
485541 }
486- AOTIDelegateHandle * handle = (AOTIDelegateHandle *)handle_;
542+ cuda::CudaDelegateHandle * handle = (cuda::CudaDelegateHandle *)handle_;
487543
488544 // Clean up cached output tensors for this handle
489545 {
@@ -495,16 +551,10 @@ class ET_EXPERIMENTAL CudaBackend final
495551 }
496552 }
497553
498- // Destroy the CUDA stream if it exists
499- if (handle->cuda_stream != nullptr ) {
500- cudaStream_t cuda_stream = static_cast <cudaStream_t>(handle->cuda_stream );
501- cudaError_t stream_err = cudaStreamDestroy (cuda_stream);
502- ET_CHECK_OR_LOG_ERROR (
503- stream_err == cudaSuccess,
504- " Failed to destroy CUDA stream: %s" ,
505- cudaGetErrorString (stream_err));
506- handle->cuda_stream = nullptr ;
507- }
554+ // The CUDA stream is managed by shared_ptr in the handle.
555+ // It will be automatically destroyed when the last handle using it
556+ // is destroyed. Just reset our reference.
557+ handle->cuda_stream .reset ();
508558
509559 // NOTE: AOTInductorModelContainerDelete does not work correctly with
510560 // multiple .so files. Deleting one container frees shared resources,
@@ -541,13 +591,22 @@ class ET_EXPERIMENTAL CudaBackend final
541591 mutable std::mutex skip_copy_method_mutex_;
542592 std::string skip_copy_method_;
543593
594+ // Shared CUDA stream for all methods. When set (non-null), all methods use
595+ // the same stream to ensure proper ordering (critical for skip-copy
596+ // optimization). Created when use_shared_cuda_stream option is set to true.
597+ // Managed via shared_ptr so it's automatically cleaned up when last handle
598+ // is destroyed.
599+ mutable std::mutex cuda_stream_mutex_;
600+ std::shared_ptr<cudaStream_t> shared_cuda_stream_ = nullptr ;
601+
544602 // Cached output tensors for skip-copy optimization.
545603 // When skip-copy is enabled, output SlimTensors are cached here to keep
546604 // the underlying GPU memory alive while the caller processes the results.
547- // Maps each AOTIDelegateHandle * to its vector of cached output tensors.
605+ // Maps each CudaDelegateHandle * to its vector of cached output tensors.
548606 mutable std::mutex cached_outputs_mutex_;
549- mutable std::unordered_map<AOTIDelegateHandle*, std::vector<SlimTensor*>>
550- cached_outputs_;
607+ mutable std::
608+ unordered_map<cuda::CudaDelegateHandle*, std::vector<SlimTensor*>>
609+ cached_outputs_;
551610};
552611
553612} // namespace executorch::backends::cuda
0 commit comments