|
| 1 | +//! Execution-provider selection shared by every ONNX session in this crate. |
| 2 | +//! |
| 3 | +//! Which providers are compiled in is a build-time decision made by cargo |
| 4 | +//! features (`cuda`, `rocm`, `coreml`, `directml`, `openvino`). Each one is |
| 5 | +//! registered with ORT's default non-fatal behaviour, so a provider whose |
| 6 | +//! hardware, driver or dynamic library is absent at runtime is skipped with a |
| 7 | +//! warning instead of failing the session. CPU is always appended last, so |
| 8 | +//! every build -- including one with no GPU feature at all -- still runs. |
| 9 | +
|
| 10 | +use std::path::Path; |
| 11 | + |
| 12 | +use ort::{ep::ExecutionProviderDispatch, session::Session}; |
| 13 | + |
| 14 | +use crate::InferError; |
| 15 | + |
| 16 | +/// Build an ORT session that tries every compiled-in GPU provider in order and |
| 17 | +/// falls back to CPU. |
| 18 | +pub(crate) fn build_session(model_path: &Path) -> Result<Session, InferError> { |
| 19 | + let builder = Session::builder()?; |
| 20 | + let providers = gpu_providers(); |
| 21 | + let mut builder = if providers.is_empty() { |
| 22 | + // No GPU feature compiled in: ORT's own default list is CPU-only, and |
| 23 | + // leaving it alone keeps this identical to the pre-feature behaviour. |
| 24 | + builder |
| 25 | + } else { |
| 26 | + let mut providers = providers; |
| 27 | + // `with_execution_providers` replaces the default list rather than |
| 28 | + // prepending to it, so CPU has to be named explicitly -- otherwise a |
| 29 | + // build whose GPU provider fails to register has nothing left to run. |
| 30 | + providers.push(ort::ep::CPU::default().build()); |
| 31 | + // `with_execution_providers` reports a *recoverable* error, which keeps |
| 32 | + // the builder inside the error value and so has a different type from |
| 33 | + // the rest of ort's API. Drop the recovery payload to get back to the |
| 34 | + // plain `ort::Error` that InferError knows how to absorb. |
| 35 | + builder |
| 36 | + .with_execution_providers(providers) |
| 37 | + .map_err(ort::Error::from)? |
| 38 | + }; |
| 39 | + Ok(builder.commit_from_file(model_path)?) |
| 40 | +} |
| 41 | + |
| 42 | +/// The GPU providers this binary was compiled with, in registration order. |
| 43 | +/// |
| 44 | +/// Order is priority: ORT tries each in turn and uses the first that accepts a |
| 45 | +/// given node. The features are mutually exclusive in practice (each build |
| 46 | +/// targets one accelerator) but nothing here depends on that. |
| 47 | +fn gpu_providers() -> Vec<ExecutionProviderDispatch> { |
| 48 | + #[allow(unused_mut)] |
| 49 | + let mut providers: Vec<ExecutionProviderDispatch> = Vec::new(); |
| 50 | + |
| 51 | + #[cfg(feature = "cuda")] |
| 52 | + { |
| 53 | + tracing::info!("Requesting CUDA execution provider"); |
| 54 | + providers.push(ort::ep::CUDA::default().build()); |
| 55 | + } |
| 56 | + #[cfg(feature = "rocm")] |
| 57 | + { |
| 58 | + tracing::info!("Requesting ROCm execution provider"); |
| 59 | + providers.push(ort::ep::ROCm::default().build()); |
| 60 | + } |
| 61 | + #[cfg(feature = "coreml")] |
| 62 | + { |
| 63 | + tracing::info!("Requesting CoreML execution provider"); |
| 64 | + providers.push(ort::ep::CoreML::default().build()); |
| 65 | + } |
| 66 | + #[cfg(feature = "directml")] |
| 67 | + { |
| 68 | + tracing::info!("Requesting DirectML execution provider"); |
| 69 | + providers.push(ort::ep::DirectML::default().build()); |
| 70 | + } |
| 71 | + #[cfg(feature = "openvino")] |
| 72 | + { |
| 73 | + tracing::info!("Requesting OpenVINO execution provider"); |
| 74 | + providers.push(ort::ep::OpenVINO::default().build()); |
| 75 | + } |
| 76 | + |
| 77 | + providers |
| 78 | +} |
| 79 | + |
| 80 | +#[cfg(test)] |
| 81 | +mod tests { |
| 82 | + use super::*; |
| 83 | + |
| 84 | + /// The provider list must match the features this build enabled: a `#[cfg]` |
| 85 | + /// that stops matching its feature (a rename, a typo, a feature dropped |
| 86 | + /// from Cargo.toml) silently returns the binary to CPU-only inference, |
| 87 | + /// which is exactly the failure this module exists to end. Counting the |
| 88 | + /// features here fails the build instead of degrading it quietly. |
| 89 | + #[test] |
| 90 | + fn provider_count_matches_enabled_features() { |
| 91 | + let expected = usize::from(cfg!(feature = "cuda")) |
| 92 | + + usize::from(cfg!(feature = "rocm")) |
| 93 | + + usize::from(cfg!(feature = "coreml")) |
| 94 | + + usize::from(cfg!(feature = "directml")) |
| 95 | + + usize::from(cfg!(feature = "openvino")); |
| 96 | + assert_eq!(gpu_providers().len(), expected); |
| 97 | + } |
| 98 | +} |
0 commit comments