@@ -810,21 +810,40 @@ bool CudaExecutor::execute(const BatchInput& batch, BatchOutput& out) {
810810 });
811811}
812812
813+ std::optional<std::vector<Token>> CudaExecutor::sample_rows (
814+ const ::executorch::aten::Tensor& logits,
815+ std::size_t rows,
816+ const std::vector<float >& params) {
817+ if (params.size () != rows * 4 ) {
818+ return std::nullopt ;
819+ }
820+ bool all_greedy = true ;
821+ for (std::size_t row = 0 ; row < rows; ++row) {
822+ all_greedy = all_greedy && params[row * 4 ] == 0 .0f ;
823+ }
824+ if (all_greedy) {
825+ return argmax_rows (logits, rows);
826+ }
827+ if (logits.device_type () != ::executorch::aten::DeviceType::CUDA ) {
828+ ET_LOG (
829+ Error,
830+ " CudaExecutor: sampled rows need device logits and %s" ,
831+ kSampleMethod );
832+ return std::nullopt ;
833+ }
834+ if (!initialized_ || !has_sampler_ || logits.dim () < 2 || !is_dense (logits) ||
835+ logits.size (logits.dim () - 1 ) != vocab_size_ ||
836+ logits.scalar_type () != ::executorch::aten::ScalarType::Float ||
837+ rows == 0 ||
838+ rows > static_cast <std::size_t >(logits.numel () / vocab_size_)) {
839+ return std::nullopt ;
840+ }
841+ return run_device_sampler (logits, rows, params, false );
842+ }
843+
813844std::optional<std::vector<Token>> CudaExecutor::sample_on_device (
814845 const ::executorch::aten::Tensor& logits,
815846 const std::vector<SessionId>& sessions) {
816- using ::executorch::aten::ScalarType;
817- using ::executorch::aten::SizesType;
818- const ::executorch::aten::Device cuda (
819- ::executorch::aten::DeviceType::CUDA , logits.device_index ());
820- const auto rows = static_cast <SizesType>(sessions.size ());
821- // The rows that sample lead the logits; the rest are padding.
822- auto row_logits = make_tensor_ptr (
823- {rows, static_cast <SizesType>(vocab_size_)},
824- logits.mutable_data_ptr (),
825- logits.scalar_type (),
826- cuda);
827-
828847 const bool all_greedy =
829848 std::all_of (sessions.begin (), sessions.end (), [this ](SessionId session) {
830849 return sessions_.greedy (session);
@@ -840,6 +859,25 @@ std::optional<std::vector<Token>> CudaExecutor::sample_on_device(
840859 params.insert (
841860 params.end (), {row->temperature , row->top_p , row->top_k , row->coin });
842861 }
862+ return run_device_sampler (logits, sessions.size (), params, all_greedy);
863+ }
864+
865+ std::optional<std::vector<Token>> CudaExecutor::run_device_sampler (
866+ const ::executorch::aten::Tensor& logits,
867+ std::size_t row_count,
868+ const std::vector<float >& params,
869+ bool all_greedy) {
870+ using ::executorch::aten::ScalarType;
871+ using ::executorch::aten::SizesType;
872+ const ::executorch::aten::Device cuda (
873+ ::executorch::aten::DeviceType::CUDA , logits.device_index ());
874+ const auto rows = static_cast <SizesType>(row_count);
875+ // The rows that sample lead the logits; the rest are padding.
876+ auto row_logits = make_tensor_ptr (
877+ {rows, static_cast <SizesType>(vocab_size_)},
878+ logits.mutable_data_ptr (),
879+ logits.scalar_type (),
880+ cuda);
843881
844882 const cudaStream_t stream =
845883 ::executorch::extension::cuda::getCallerStream ().value_or(
@@ -851,18 +889,18 @@ std::optional<std::vector<Token>> CudaExecutor::sample_on_device(
851889 kArgmaxMethod ,
852890 std::vector<::executorch::runtime::EValue>{row_logits});
853891 }
854- if (sessions. size () > device_params_rows_) {
892+ if (row_count > device_params_rows_) {
855893 if (device_params_ != nullptr ) {
856894 (void )cudaFree (device_params_);
857895 device_params_ = nullptr ;
858896 device_params_rows_ = 0 ;
859897 }
860898 ET_CHECK_OR_RETURN_ERROR (
861- cudaMalloc (&device_params_, sessions. size () * 4 * sizeof (float )) ==
899+ cudaMalloc (&device_params_, row_count * 4 * sizeof (float )) ==
862900 cudaSuccess,
863901 MemoryAllocationFailed,
864902 " CudaExecutor: could not stage the sampler's params" );
865- device_params_rows_ = sessions. size () ;
903+ device_params_rows_ = row_count ;
866904 }
867905 // The backend runs every method on the caller's stream if one is set,
868906 // else the per-thread stream; ordering the upload there puts it ahead of
@@ -890,7 +928,7 @@ std::optional<std::vector<Token>> CudaExecutor::sample_on_device(
890928 ET_LOG (Error, " CudaExecutor: the device sampler returned a bad shape" );
891929 return std::nullopt ;
892930 }
893- std::vector<std::int64_t > host (sessions. size () );
931+ std::vector<std::int64_t > host (row_count );
894932 const auto kind = tokens.device_type () == ::executorch::aten::DeviceType::CUDA
895933 ? cudaMemcpyDeviceToHost
896934 : cudaMemcpyHostToHost;
0 commit comments