diff --git a/.ci/scripts/wheel/test_base.py b/.ci/scripts/wheel/test_base.py index dcdbacdf92b..9e2eb9d6149 100644 --- a/.ci/scripts/wheel/test_base.py +++ b/.ci/scripts/wheel/test_base.py @@ -8,10 +8,8 @@ import os import subprocess import sys -import tempfile from dataclasses import dataclass from functools import cache -from pathlib import Path from typing import List @@ -43,7 +41,7 @@ class ModelTest: backend: Backend -def test_a_model_runs_through_the_openvino_delegate() -> None: +def test_a_model_runs_through_the_openvino_delegate(required: bool = False) -> None: """Export a model to the OpenVINO delegate and run it, comparing against eager. The checks elsewhere prove the delegate library ships, owns its symbols and registers @@ -53,7 +51,8 @@ def test_a_model_runs_through_the_openvino_delegate() -> None: The OpenVINO runtime is not part of the wheel. It comes from the openvino extra, so a default install cannot run this and says so rather than passing quietly. Where the runtime is present the export and the comparison are the point of the check, and a failure there - is a real one. + is a real one. A caller that installed the export stack passes required=True, so a missing + piece fails instead of skipping. """ import torch from executorch.exir import to_edge_transform_and_lower @@ -67,6 +66,8 @@ def test_a_model_runs_through_the_openvino_delegate() -> None: try: from executorch.backends.openvino.partitioner import OpenvinoPartitioner except ImportError as error: + if required: + raise print( f"SKIP: the OpenVINO export stack is not installed, so the delegate cannot " f"execute here ({error}). The wheel ships the adapter only; the runtime and the " @@ -88,13 +89,16 @@ def forward(self, x, y): partitioner=[OpenvinoPartitioner([CompileSpec("device", b"CPU")])], ) program = lowered.to_executorch() + delegates = [d.id for d in program.executorch_program.execution_plan[0].delegates] + assert "OpenvinoBackend" in delegates, ( + f"the partitioner did not hand the model to OpenVINO (delegates: {delegates}), so " + f"running it would test the portable kernels instead of the delegate" + ) - with tempfile.TemporaryDirectory() as work_dir: - pte = Path(work_dir) / "add_openvino.pte" - pte.write_bytes(program.buffer) - - method = Runtime.get().load_program(pte).load_method("forward") - actual = method.execute(list(example))[0] + # From memory rather than a temporary file, which the loaded program keeps open and + # Windows then refuses to delete. + method = Runtime.get().load_program(program.buffer).load_method("forward") + actual = method.execute(list(example))[0] difference = (actual - eager).abs().max().item() assert difference == 0.0, ( diff --git a/.ci/scripts/wheel/test_cuda_windows.py b/.ci/scripts/wheel/test_cuda_windows.py index 3c5fbb60efb..3bc7ba82028 100644 --- a/.ci/scripts/wheel/test_cuda_windows.py +++ b/.ci/scripts/wheel/test_cuda_windows.py @@ -640,6 +640,15 @@ def export(destination: str) -> None: with tempfile.TemporaryDirectory() as work_dir: test_cpp_sdk.run_tests(Path(work_dir)) import test_base + from executorch.extension.pybindings.portable_lib import ( + _get_registered_backend_names, + ) + + registered = _get_registered_backend_names() + assert ( + "OpenvinoBackend" in registered + ), f"OpenvinoBackend not found in registered backends: {registered}" + print("✓ OpenvinoBackend is registered") test_base.test_cmsis_nn_install() # The same model run the CPU row ends with, through the Python bindings. @@ -650,3 +659,6 @@ def export(destination: str) -> None: test_windows.run_tests( model_tests=[ModelTest(model=Model.Mv3, backend=Backend.Xnnpack)] ) + + # Last, because it installs packages the wheel does not declare. + test_windows.test_the_openvino_delegate_runs() diff --git a/.ci/scripts/wheel/test_shared_libraries.py b/.ci/scripts/wheel/test_shared_libraries.py index 50e1f156833..a1d8c791b83 100644 --- a/.ci/scripts/wheel/test_shared_libraries.py +++ b/.ci/scripts/wheel/test_shared_libraries.py @@ -149,7 +149,11 @@ # The delegate's own entry points. A second definer means the delegate is compiled # into the Python extension as well, which would register it twice in one process. _QNN_BACKEND_SYMBOLS = ("QnnExecuTorchBackendRegister",) -_OPENVINO_BACKEND_SYMBOLS = ("executorch::backends::openvino::OpenvinoBackend",) +# A member rather than the bare class name, because the PE reader matches MSVC decorated +# names, which spell a member as ?is_available@OpenvinoBackend@... and never the class alone. +_OPENVINO_BACKEND_SYMBOLS = ( + "executorch::backends::openvino::OpenvinoBackend::is_available", +) _BUNDLED_XNNPACK_SYMBOLS = ("xnn_create_runtime_v4",) @@ -1097,22 +1101,23 @@ def _wheel_cuda_train() -> str: # wheel is inspected, and a row cannot call that at import time. _REQUIRED_ON_A_CUDA_WHEEL = "cuda-wheel-only" -# Marker for a row whose owner every Linux wheel carries and no macOS wheel does. -_REQUIRED_ON_LINUX = "linux-only" +# Marker for a row whose owner every Linux and Windows wheel carries and no macOS wheel +# does. +_REQUIRED_ON_LINUX_AND_WINDOWS = "linux-and-windows-only" # Marker for a row whose owner every macOS wheel carries and no other wheel does. # The Core ML delegate is built on every macOS wheel, so its presence is decidable # from the installed wheel and a macOS wheel that dropped it should fail rather than # quietly regress to folding it into the Python extension. _REQUIRED_ON_MACOS = "macos-only" -# Narrower than the marker above on purpose. The Qualcomm SDK the delegate builds -# against is published for Linux x86_64 only, so the aarch64 Linux wheels ship no such -# library and demanding it there would fail a wheel that is correct. Reusing the -# Linux-wide marker would do exactly that. +# Narrower than the Linux and Windows marker on purpose. The Qualcomm SDK the delegate +# builds against is published for Linux x86_64 only, so the aarch64 Linux wheels ship no +# such library and demanding it there would fail a wheel that is correct. Reusing that +# marker would do exactly that. _REQUIRED_ON_LINUX_X86 = "linux-x86-only" # Marker for a row whose owner every aarch64 wheel carries, macOS and Linux alike. The # TorchAO kernels are selected by the processor rather than the operating system, so -# neither the macOS nor the Linux marker describes them and a fixed False would let a -# wheel that silently dropped the library pass. +# neither the macOS nor the Linux and Windows marker describes them and a fixed False +# would let a wheel that silently dropped the library pass. _REQUIRED_ON_AARCH64 = "aarch64-only" @@ -1120,8 +1125,8 @@ def _resolve_required(required): """Turn a row's requirement marker into the answer for the installed wheel.""" if required == _REQUIRED_ON_A_CUDA_WHEEL: return bool(_wheel_cuda_train()) - if required == _REQUIRED_ON_LINUX: - return sys.platform == "linux" + if required == _REQUIRED_ON_LINUX_AND_WINDOWS: + return sys.platform in ("linux", "win32") if required == _REQUIRED_ON_MACOS: return sys.platform == "darwin" if required == _REQUIRED_ON_LINUX_X86: @@ -1279,15 +1284,15 @@ def _resolve_required(required): _library_file_name("libexecutorch_backend_mlx"), False, ), - # Required on Linux, where packaging turns the backend on for every non-minimal - # build. A fixed False passed a wheel that had compiled the delegate back into the - # extension: no library ships, so the row skips, and one definer inside the + # Required on Linux and Windows, where packaging turns the backend on for every + # non-minimal build. A fixed False passed a wheel that had compiled the delegate back + # into the extension: no library ships, so the row skips, and one definer inside the # extension is exactly the monolithic layout a definer count cannot distinguish. ( "OpenVINO delegate", _OPENVINO_BACKEND_SYMBOLS, _library_file_name("libexecutorch_backend_openvino"), - _REQUIRED_ON_LINUX, + _REQUIRED_ON_LINUX_AND_WINDOWS, ), # Required on Linux x86_64 only, where the Qualcomm SDK the delegate builds against # is published. A fixed False would pass a wheel that compiled the delegate back diff --git a/.ci/scripts/wheel/test_windows.py b/.ci/scripts/wheel/test_windows.py index 8600b676f1f..f0917c4671d 100644 --- a/.ci/scripts/wheel/test_windows.py +++ b/.ci/scripts/wheel/test_windows.py @@ -6,7 +6,13 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +import importlib.metadata +import importlib.util +import os import platform +import shutil +import subprocess +import sys import tempfile from pathlib import Path from typing import List @@ -24,6 +30,7 @@ from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower from executorch.extension.pybindings.portable_lib import ( _get_registered_backend_names, + _is_available, _load_for_executorch_from_buffer, ) from test_base import ModelTest @@ -71,7 +78,104 @@ def run_tests(model_tests: List[ModelTest]) -> None: raise RuntimeError(f"Unsupported backend {model_test.backend}.") +def test_the_openvino_delegate_runs() -> None: + """Install the OpenVINO runtime the way a user does, then run a model through it. + + The wheel ships the delegate but not the runtime, and registering the delegate does not + load the runtime, so nothing else here proves the delegate can open openvino_c.dll and + compute. The check runs in a new process because the package looks for the runtime when + it is imported, which in this process happened before the install. + """ + # Pinned, so pip cannot swap the wheel under test for another release. + install = [ + sys.executable, + "-m", + "pip", + "install", + f"executorch[openvino]=={importlib.metadata.version('executorch')}", + ] + # The same bound as the python_version marker on the openvino extra. + if sys.version_info >= (3, 15): + subprocess.run(install, check=True) + importlib.invalidate_caches() + assert ( + importlib.util.find_spec("openvino") is None + ), "the openvino extra installed a runtime on a Python its marker leaves out" + print( + "SKIP: OpenVINO publishes no build for this Python, so the openvino extra " + "installs no runtime and the delegate cannot load here." + ) + return + subprocess.run( + install + + [ + "-r", + str( + Path(test_base._repository_root_dir()) + / "backends/openvino/requirements.txt" + ), + ], + check=True, + ) + subprocess.run( + [sys.executable, str(Path(__file__).resolve()), "--openvino"], check=True + ) + importlib.invalidate_caches() + openvino = importlib.util.find_spec("openvino") + test_a_bare_name_does_not_load_from_the_current_folder( + Path(openvino.submodule_search_locations[0]) / "libs" + ) + + +def test_a_bare_name_does_not_load_from_the_current_folder(libs: Path) -> None: + """Run from a folder holding the real runtime and ask for it by bare name. + + The standard Windows search order includes the current folder, where a planted + openvino_c.dll would run, so the delegate has to refuse the copy found there. The + same copy asked for by its full path has to load, so the refusal comes from the + search order and not from a copy that cannot load at all. + """ + with tempfile.TemporaryDirectory() as work_dir: + for library in libs.glob("*.dll"): + shutil.copy2(library, work_dir) + for lib_path, expected in ( + (str(Path(work_dir) / "openvino_c.dll"), "True"), + ("openvino_c.dll", "False"), + ): + probe = subprocess.run( + [ + sys.executable, + "-c", + "from executorch.extension.pybindings.portable_lib import " + "_is_available; print(_is_available('OpenvinoBackend'))", + ], + cwd=work_dir, + env={**os.environ, "OPENVINO_LIB_PATH": lib_path}, + capture_output=True, + text=True, + check=False, + ) + assert probe.returncode == 0 and probe.stdout.split()[-1:] == [expected], ( + f"OPENVINO_LIB_PATH={lib_path}, run from a folder holding the runtime, " + f"should make _is_available return {expected}: the full path has to " + "load the copy there and the bare name must not, since a planted copy " + f"would run:\n{probe.stdout}{probe.stderr}" + ) + print( + "✓ a bare openvino_c.dll name does not load from the current folder, while " + "its full path does" + ) + + if __name__ == "__main__": + if sys.argv[1:] == ["--openvino"]: + assert _is_available( + "OpenvinoBackend" + ), "OpenvinoBackend is registered but cannot load the OpenVINO runtime" + print("✓ OpenvinoBackend loads the OpenVINO runtime") + test_base.test_a_model_runs_through_the_openvino_delegate(required=True) + sys.exit(0) + # Before anything else, because this is the check that fails the way a user # fails: with only the dependencies the wheel declares. with tempfile.TemporaryDirectory() as work_dir: @@ -79,6 +183,11 @@ def run_tests(model_tests: List[ModelTest]) -> None: if platform.system() == "Windows": registered = _get_registered_backend_names() + assert ( + "OpenvinoBackend" in registered + ), f"OpenvinoBackend not found in registered backends: {registered}" + print("✓ OpenvinoBackend is registered") + # Vulkan backend is optional: only present when the wheel was built with # EXECUTORCH_BUILD_VULKAN=1 and the Vulkan SDK (glslc) was available. if "VulkanBackend" in registered: @@ -108,3 +217,7 @@ def run_tests(model_tests: List[ModelTest]) -> None: ), ] ) + + # Last, because it installs packages the wheel does not declare. + if platform.system() == "Windows": + test_the_openvino_delegate_runs() diff --git a/README-wheel.md b/README-wheel.md index 083eb49fa9d..c76f6819010 100644 --- a/README-wheel.md +++ b/README-wheel.md @@ -34,9 +34,11 @@ to run ExecuTorch `.pte` files, with some restrictions: * \[macOS only] [Core ML](docs/source/backends/coreml/coreml-overview.md) and MLX backends are also linked into the prebuilt module. * \[Linux x86_64] [QNN](docs/source/backends-qualcomm.md) backend is linked into the prebuilt module. -* \[Linux] [OpenVINO](docs/source/build-run-openvino.md) backend is also linked into the +* \[Linux and Windows] [OpenVINO](docs/source/build-run-openvino.md) backend is also linked into the prebuilt module. OpenVINO requires the runtime to be installed separately: - `pip install executorch[openvino]` + `pip install executorch[openvino]`. For now the extra installs no runtime on Python 3.15, + or on Linux aarch64 with Python 3.14, because the OpenVINO versions it supports publish no + build there. On Linux, macOS and Windows the runtime, kernels and backends also ship as shared libraries with headers and a CMake package, so a C++ application can link them without building diff --git a/README.md b/README.md index 916ec777abb..f920bc41169 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,7 @@ is authoritative. See also the [Desktop guide](desktop/README.md). | iOS / iPadOS | [XNNPACK](https://docs.pytorch.org/executorch/main/backends/xnnpack/xnnpack-overview.html) CPU; [Core ML](https://docs.pytorch.org/executorch/main/backends/coreml/coreml-overview.html); [MLX](https://docs.pytorch.org/executorch/main/backends/mlx/mlx-overview.html) on physical devices *(experimental)* | | macOS | [XNNPACK](https://docs.pytorch.org/executorch/main/backends/xnnpack/xnnpack-overview.html); [Core ML](https://docs.pytorch.org/executorch/main/backends/coreml/coreml-overview.html); experimental [MLX](https://docs.pytorch.org/executorch/main/backends/mlx/mlx-overview.html), [Metal/AOTInductor](backends/apple/metal/README.md), and [WebGPU](https://docs.pytorch.org/executorch/main/backends/webgpu/webgpu-overview.html) paths | | Linux | [XNNPACK](https://docs.pytorch.org/executorch/main/backends/xnnpack/xnnpack-overview.html); [OpenVINO](https://docs.pytorch.org/executorch/main/build-run-openvino.html); experimental [CUDA/AOTInductor](https://docs.pytorch.org/executorch/main/backends/cuda/cuda-overview.html), [Vulkan](https://docs.pytorch.org/executorch/main/backends/vulkan/vulkan-overview.html), and [WebGPU](https://docs.pytorch.org/executorch/main/backends/webgpu/webgpu-overview.html) desktop paths | -| Windows | [XNNPACK](https://docs.pytorch.org/executorch/main/backends/xnnpack/xnnpack-overview.html); experimental [CUDA/AOTInductor](https://docs.pytorch.org/executorch/main/backends/cuda/cuda-overview.html) and [Vulkan](https://docs.pytorch.org/executorch/main/backends/vulkan/vulkan-overview.html) desktop paths | +| Windows | [XNNPACK](https://docs.pytorch.org/executorch/main/backends/xnnpack/xnnpack-overview.html); [OpenVINO](https://docs.pytorch.org/executorch/main/build-run-openvino.html); experimental [CUDA/AOTInductor](https://docs.pytorch.org/executorch/main/backends/cuda/cuda-overview.html) and [Vulkan](https://docs.pytorch.org/executorch/main/backends/vulkan/vulkan-overview.html) desktop paths | | Browser / WebAssembly | [Portable WebAssembly runtime](extension/wasm/README.md) and [WebGPU](https://docs.pytorch.org/executorch/main/backends/webgpu/webgpu-overview.html) *(both experimental)* | | Embedded / MCU | [Arm Cortex-M with CMSIS-NN](https://docs.pytorch.org/executorch/main/backends/arm-cortex-m/arm-cortex-m-overview.html) *(beta)*; [Arm Ethos-U](https://docs.pytorch.org/executorch/main/backends/arm-ethos-u/arm-ethos-u-overview.html); [NXP eIQ Neutron](https://docs.pytorch.org/executorch/main/backends/nxp/nxp-overview.html); [Cadence DSP](https://docs.pytorch.org/executorch/main/backends-cadence.html); [Zephyr](zephyr/README.md) and [Arduino](examples/arduino/README.md) integrations | diff --git a/backends/openvino/CMakeLists.txt b/backends/openvino/CMakeLists.txt index 0f3a04e97c2..4e9c087b648 100644 --- a/backends/openvino/CMakeLists.txt +++ b/backends/openvino/CMakeLists.txt @@ -28,8 +28,9 @@ set(COMMON_INCLUDE_DIRS ${EXECUTORCH_ROOT}/..) # Include utility CMake scripts from ExecuteTorch include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake) -# The backend resolves OpenVINO C API symbols via dlopen/dlsym at runtime, so -# there is no build-time dependency on the OpenVINO SDK. +# The backend resolves OpenVINO C API symbols at runtime (dlopen on Linux, +# LoadLibrary on Windows), so there is no build-time dependency on the OpenVINO +# SDK. # Shared when the wheel build asks for it, static otherwise. if(EXECUTORCH_BUILD_SHARED) @@ -39,8 +40,13 @@ else() endif() add_library(openvino_backend ${_openvino_backend_library_type}) -# Enable exceptions and RTTI for OpenVINO backend -target_compile_options(openvino_backend PRIVATE -frtti -fexceptions) +# Enable exceptions and RTTI for OpenVINO backend. MSVC is also set for +# clang-cl, which takes the cl spellings. +if(MSVC) + target_compile_options(openvino_backend PRIVATE /EHsc /GR) +else() + target_compile_options(openvino_backend PRIVATE -frtti -fexceptions) +endif() # Add source files for OpenVINO backend target_sources( diff --git a/backends/openvino/README.md b/backends/openvino/README.md index cdb94e03590..56d2f13383f 100644 --- a/backends/openvino/README.md +++ b/backends/openvino/README.md @@ -14,20 +14,30 @@ For more information on the supported hardware, please refer to [OpenVINO System ## Quick Start (pip wheel) -On Linux, the OpenVINO backend is included in the ExecuTorch pip wheel. Install the OpenVINO runtime to activate it: +On Linux and Windows, the OpenVINO backend is included in the ExecuTorch pip wheel. Install the OpenVINO runtime to activate it: ```bash pip install executorch[openvino] ``` -The backend automatically discovers the OpenVINO C library from the pip-installed package — no `LD_LIBRARY_PATH` setup is needed. +For now the extra installs no runtime on Python 3.15, or on Linux aarch64 with Python 3.14, because the OpenVINO versions it supports publish no build there. -If auto-discovery fails (e.g. non-standard install), you can point to the library explicitly: +The backend automatically discovers the OpenVINO C library from the pip-installed package, so no library search path setup is needed. + +If auto-discovery fails (e.g. non-standard install), set `OPENVINO_LIB_PATH` to the full path of the OpenVINO C library: a `libopenvino_c.so` file on Linux, or `openvino_c.dll` on Windows. For the pip package on Linux: ```bash -export OPENVINO_LIB_PATH=$(python3 -c "import openvino, os; print(os.path.join(os.path.dirname(openvino.__file__), 'libs', 'libopenvino_c.so'))") +export OPENVINO_LIB_PATH="$(python3 -c 'import glob, openvino, os; print(sorted(glob.glob(os.path.join(os.path.dirname(openvino.__file__), "libs", "libopenvino_c.so*")))[0])')" ``` +On Windows, in PowerShell: + +```powershell +$env:OPENVINO_LIB_PATH = python -c "import openvino, os; print(os.path.join(os.path.dirname(openvino.__file__), 'libs', 'openvino_c.dll'))" +``` + +On Windows, `PATH` and the current folder are not searched for the library or for its dependencies, such as `tbb12.dll`. Those must sit beside `openvino_c.dll`, as they do in the pip package, or in a folder added with `os.add_dll_directory` (`AddDllDirectory` in C++). Importing the `openvino` Python package adds its folders this way, including the ones Intel's `setupvars.bat` lists. + Verify the backend is available: ```python diff --git a/backends/openvino/runtime/OpenvinoApi.h b/backends/openvino/runtime/OpenvinoApi.h index 90403e24b95..d25e265db07 100644 --- a/backends/openvino/runtime/OpenvinoApi.h +++ b/backends/openvino/runtime/OpenvinoApi.h @@ -8,7 +8,6 @@ #pragma once -#include #include #include #include @@ -96,12 +95,9 @@ using ov_shape_create_fn = ov_status_e (*)(int64_t, const int64_t*, ov_shape_t*); using ov_shape_free_fn = ov_status_e (*)(ov_shape_t*); +// Defined in OpenvinoBackend.cpp, where the platform's loader API is included. struct DlCloser { - void operator()(void* handle) { - if (handle) { - dlclose(handle); - } - } + void operator()(void* handle); }; using DlHandle = std::unique_ptr; diff --git a/backends/openvino/runtime/OpenvinoBackend.cpp b/backends/openvino/runtime/OpenvinoBackend.cpp index 3cf87e6abfa..7fec8fd150f 100644 --- a/backends/openvino/runtime/OpenvinoBackend.cpp +++ b/backends/openvino/runtime/OpenvinoBackend.cpp @@ -10,6 +10,12 @@ #include #include +#ifdef _WIN32 +#include +#else +#include +#endif + #include #include #include @@ -23,7 +29,140 @@ namespace openvino { namespace { +#ifdef _WIN32 +constexpr const wchar_t* kDefaultLibName = L"openvino_c.dll"; +constexpr const char* kLibrarySearchPathHint = + "set OPENVINO_LIB_PATH to its full path, with its dependencies such as " + "tbb12.dll beside it or in a folder added with AddDllDirectory"; + +// A path with a character outside the active code page has no faithful narrow +// form, so read and load through the wide APIs and convert to UTF-8 only for +// logging. Unlike _wgetenv, which reads the C runtime's copy of the +// environment, GetEnvironmentVariableW also sees a value set with +// SetEnvironmentVariableW after startup. +std::wstring configured_lib_path() { + DWORD length = GetEnvironmentVariableW(L"OPENVINO_LIB_PATH", nullptr, 0); + if (length == 0) { + return std::wstring(kDefaultLibName); + } + std::wstring lib_path; + // Loop, since the value may grow between asking for its size and reading it. + while (length > lib_path.size()) { + lib_path.resize(length); + length = + GetEnvironmentVariableW(L"OPENVINO_LIB_PATH", lib_path.data(), length); + } + lib_path.resize(length); + return lib_path; +} + +// The standard search order includes the current folder, so a bare name would +// load an openvino_c.dll planted there. The LOAD_LIBRARY_SEARCH_* flags leave +// out the current folder and PATH. For a path they also search the library's +// own folder for its dependencies, such as openvino.dll and tbb12.dll, and +// they need the path to be absolute. +void* open_library(const std::wstring& path) { + if (path.find_first_of(L"\\/") == std::wstring::npos) { + return LoadLibraryExW( + path.c_str(), nullptr, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + } + const DWORD size = GetFullPathNameW(path.c_str(), 0, nullptr, nullptr); + std::wstring full(size, L'\0'); + const DWORD length = + GetFullPathNameW(path.c_str(), size, full.data(), nullptr); + if (length == 0 || length >= size) { + return nullptr; + } + full.resize(length); + return LoadLibraryExW( + full.c_str(), + nullptr, + LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); +} + +std::string to_utf8(const std::wstring& wide) { + if (wide.empty()) { + return std::string(); + } + int size = WideCharToMultiByte( + CP_UTF8, 0, wide.data(), (int)wide.size(), nullptr, 0, nullptr, nullptr); + std::string out(size, '\0'); + WideCharToMultiByte( + CP_UTF8, + 0, + wide.data(), + (int)wide.size(), + out.data(), + size, + nullptr, + nullptr); + return out; +} + +// The system's message for the calling thread's last error, with its code, +// since a code alone is hard to act on. Call it right after the failing call: +// later calls, logging included, may overwrite the last error. +std::string last_error() { + const DWORD err = GetLastError(); + LPSTR buffer = nullptr; + const DWORD flags = FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS; + const DWORD size = FormatMessageA( + flags, + nullptr, + err, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + reinterpret_cast(&buffer), + 0, + nullptr); + if (size == 0 || buffer == nullptr) { + return "error " + std::to_string(err); + } + std::string message(buffer, size); + LocalFree(buffer); + while (!message.empty() && + (message.back() == '\n' || message.back() == '\r')) { + message.pop_back(); + } + return message + " (error " + std::to_string(err) + ")"; +} + +template +FuncPtr load_symbol(void* handle, const char* name) { + FARPROC sym = GetProcAddress(static_cast(handle), name); + if (!sym) { + const std::string error = last_error(); + ET_LOG( + Error, + "OpenVINO: failed to resolve symbol '%s': %s", + name, + error.c_str()); + return nullptr; + } + return reinterpret_cast(sym); +} +#else constexpr const char* kDefaultLibName = "libopenvino_c.so"; +constexpr const char* kLibrarySearchPathHint = + "set OPENVINO_LIB_PATH or LD_LIBRARY_PATH"; + +std::string configured_lib_path() { + const char* lib_path = std::getenv("OPENVINO_LIB_PATH"); + return lib_path ? std::string(lib_path) : std::string(kDefaultLibName); +} + +void* open_library(const std::string& path) { + return dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); +} + +std::string to_utf8(const std::string& path) { + return path; +} + +std::string last_error() { + const char* err = dlerror(); + return err ? err : "unknown error"; +} template FuncPtr load_symbol(void* handle, const char* name) { @@ -36,26 +175,48 @@ FuncPtr load_symbol(void* handle, const char* name) { } return reinterpret_cast(sym); } +#endif } // namespace +void DlCloser::operator()(void* handle) { + if (!handle) { + return; + } +#ifdef _WIN32 + FreeLibrary(static_cast(handle)); +#else + dlclose(handle); +#endif +} + +OpenvinoBackend::~OpenvinoBackend() { +#ifdef _WIN32 + // The backend is a global, so in a DLL this runs inside DllMain when the DLL + // unloads, where FreeLibrary is not safe to call. Keep the runtime loaded. + (void)lib_handle_.release(); +#endif +} + // Loading is attempted exactly once via std::call_once. If the first attempt -// fails (e.g. library not on LD_LIBRARY_PATH), subsequent calls return false +// fails (e.g. library not on the search path), subsequent calls return false // without retrying. Users must fix their environment and restart the process. bool OpenvinoBackend::ensure_loaded() const { std::call_once(load_flag_, [this]() { - const char* lib_path = std::getenv("OPENVINO_LIB_PATH"); - const char* effective_path = lib_path ? lib_path : kDefaultLibName; + auto effective_path = configured_lib_path(); - void* handle = dlopen(effective_path, RTLD_NOW | RTLD_LOCAL); + void* handle = open_library(effective_path); if (!handle) { + const std::string error = last_error(); ET_LOG( Error, - "OpenVINO runtime not found (dlopen failed: %s). " - "Ensure 'libopenvino_c.so' is on your library search path " - "(set OPENVINO_LIB_PATH or LD_LIBRARY_PATH), or install with: " + "OpenVINO runtime not found (loading '%s' failed: %s). " + "Ensure '%s' can be found (%s), or install with: " "pip install \"openvino>=2025.1.0,<2026.0.0\"", - dlerror()); + to_utf8(effective_path).c_str(), + error.c_str(), + to_utf8(kDefaultLibName).c_str(), + kLibrarySearchPathHint); return; } lib_handle_.reset(handle); @@ -98,7 +259,7 @@ bool OpenvinoBackend::ensure_loaded() const { ET_LOG( Info, "OpenVINO: runtime loaded successfully from '%s'", - effective_path); + to_utf8(effective_path).c_str()); }); return loaded_; } diff --git a/backends/openvino/runtime/OpenvinoBackend.h b/backends/openvino/runtime/OpenvinoBackend.h index b0e60a3e068..72b0193dc2d 100644 --- a/backends/openvino/runtime/OpenvinoBackend.h +++ b/backends/openvino/runtime/OpenvinoBackend.h @@ -33,7 +33,7 @@ struct ExecutionHandle { class OpenvinoBackend final : public ::exr::BackendInterface { public: OpenvinoBackend() = default; - ~OpenvinoBackend() override = default; + ~OpenvinoBackend() override; bool is_available() const override; exr::Result init( diff --git a/docs/source/build-run-openvino.md b/docs/source/build-run-openvino.md index b560b9badef..d749e286c26 100644 --- a/docs/source/build-run-openvino.md +++ b/docs/source/build-run-openvino.md @@ -35,7 +35,7 @@ For more information on the supported hardware, please refer to [OpenVINO System ## Instructions for Building OpenVINO Backend -### Quick Start (pip wheel, Linux) +### Quick Start (pip wheel, Linux and Windows) The easiest way to use the OpenVINO backend is via the prebuilt pip wheel: @@ -43,6 +43,9 @@ The easiest way to use the OpenVINO backend is via the prebuilt pip wheel: pip install executorch[openvino] ``` +For now the extra installs no runtime on Python 3.15, or on Linux aarch64 with Python +3.14, because the OpenVINO versions it supports publish no build there. + The backend automatically discovers the OpenVINO C library from the pip-installed package — no environment variable setup is needed. diff --git a/docs/source/using-executorch-cpp.md b/docs/source/using-executorch-cpp.md index 72c9457bfd5..221acdafd63 100644 --- a/docs/source/using-executorch-cpp.md +++ b/docs/source/using-executorch-cpp.md @@ -199,7 +199,7 @@ These are the components the package provides: | `kernels_torchao` | The TorchAO low-bit quantized kernels | Linux and macOS, aarch64 only | | `backend_cuda` | The CUDA delegate | Linux, Windows (runs programs exported on Linux) | | `extension_cuda` | Shared CUDA allocator, stream and device helpers. Allocator headers require the CUDA toolkit. A program that calls the CUDA runtime itself links `CUDA::cudart`. | Linux, Windows | -| `backend_openvino` | The OpenVINO delegate | Linux | +| `backend_openvino` | The OpenVINO delegate | Linux, Windows | | `backend_coreml` | The Core ML delegate, for Apple GPU and Neural Engine execution | macOS | | `backend_mlx` | The MLX delegate, for Apple GPU execution | macOS, Apple Silicon | @@ -253,18 +253,28 @@ The quantized kernels are deliberately left out of that variable, because loadin stops the runtime. The OpenVINO delegate needs one more step. The wheel ships the adapter, not the OpenVINO runtime -itself, and the adapter opens `libopenvino_c.so` by name when the model is loaded. Python callers -get that path set for them on import; a standalone C++ program does not, so install the runtime and -point the program at it: +itself, and the adapter opens `libopenvino_c.so` (`openvino_c.dll` on Windows) by name when the +model is loaded. Python callers get that path set for them on import; a standalone C++ program does +not, so install the runtime and point the program at it: ```bash pip install "executorch[openvino]" export OPENVINO_LIB_PATH="$(python -c 'import glob, openvino, os; print(sorted(glob.glob(os.path.join(os.path.dirname(openvino.__file__), "libs", "libopenvino_c.so*")))[0])')" ``` +On Windows, in PowerShell: + +```powershell +pip install "executorch[openvino]" +$env:OPENVINO_LIB_PATH = python -c "import openvino, os; print(os.path.join(os.path.dirname(openvino.__file__), 'libs', 'openvino_c.dll'))" +``` + Without it the delegate still registers and the program still links, and the failure arrives later, when the model is loaded. +For now the openvino extra installs no runtime on Python 3.15, or on Linux aarch64 with +Python 3.14, because the OpenVINO versions it supports publish no build there. + The MLX delegate has a similar requirement. Its Metal kernels live in a separate `mlx.metallib` file, which MLX looks for next to whichever library holds MLX code. The wheel ships it beside the delegate, so a program that links the delegate where it sits needs nothing extra. A program that diff --git a/extension/pybindings/portable_lib.py b/extension/pybindings/portable_lib.py index 2fc920be5cd..b936acaeb2f 100644 --- a/extension/pybindings/portable_lib.py +++ b/extension/pybindings/portable_lib.py @@ -52,28 +52,29 @@ class _LightweightExperimentalWarning(DeprecationWarning): # Auto-discover the OpenVINO C library path from the pip-installed openvino -# package so the C++ backend's dlopen("libopenvino_c.so") works without the -# user having to set LD_LIBRARY_PATH or OPENVINO_LIB_PATH manually. +# package so the C++ backend can load it without the user having to set +# OPENVINO_LIB_PATH or the platform's library search path manually. if not os.environ.get("OPENVINO_LIB_PATH"): try: import glob import importlib.util + _ov_c_lib = "openvino_c.dll" if sys.platform == "win32" else "libopenvino_c.so" spec = importlib.util.find_spec("openvino") if spec is not None and spec.submodule_search_locations: _ov_dir = spec.submodule_search_locations[0] - _ov_libs = sorted( - glob.glob(os.path.join(_ov_dir, "libs", "libopenvino_c.so*")) - ) + _ov_libs = sorted(glob.glob(os.path.join(_ov_dir, "libs", _ov_c_lib + "*"))) if _ov_libs: os.environ["OPENVINO_LIB_PATH"] = _ov_libs[0] else: logger.warning( - "OpenVINO package found but libopenvino_c.so not in %s; " + "OpenVINO package found but %s not in %s; " "set OPENVINO_LIB_PATH manually if needed", + _ov_c_lib, os.path.join(_ov_dir, "libs"), ) del _ov_libs, _ov_dir, spec + del _ov_c_lib except Exception as e: logger.debug("OpenVINO auto-discovery failed: %s", e) diff --git a/pyproject.toml b/pyproject.toml index 28721d76cc2..9640001e415 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,9 @@ ethos_u = [ "tosa-tools==2026.5.0", ] openvino = [ - "openvino>=2025.1.0,<2026.0.0; platform_system == 'Linux'", + # The OpenVINO versions in this range publish no build for Python 3.15, or for Linux + # aarch64 on Python 3.14, so without these bounds pip fails to install the extra there. + "openvino>=2025.1.0,<2026.0.0; (platform_system == 'Linux' or (platform_system == 'Windows' and platform_machine == 'AMD64')) and python_version < '3.15' and (platform_machine != 'aarch64' or python_version < '3.14')", ] [project.urls] diff --git a/setup.py b/setup.py index 68aee9fc2d0..36d97b609a0 100644 --- a/setup.py +++ b/setup.py @@ -1411,6 +1411,12 @@ def _windows_import_libraries() -> List["BuiltFile"]: None, ["EXECUTORCH_BUILD_CUDA"], ), + ( + "backends/openvino/", + "executorch_backend_openvino", + None, + ["EXECUTORCH_BUILD_OPENVINO"], + ), ] return [ BuiltFile( @@ -2814,11 +2820,12 @@ def run(self): # noqa C901 f"-DQNN_SDK_ROOT={qnn_sdk_root}", ] - # Enable OpenVINO backend on Linux. The backend uses dlopen at - # runtime so it has no build-time SDK dependency. + # Enable the OpenVINO backend on Linux and Windows. The backend loads + # the OpenVINO runtime at run time, so it has no build-time SDK + # dependency. if ( not minimal_build - and sys.platform == "linux" + and sys.platform in ("linux", "win32") and install_utils.is_cmake_option_on( cmake_configuration_args, "EXECUTORCH_BUILD_OPENVINO", diff --git a/tools/cmake/executorch-wheel-config.cmake b/tools/cmake/executorch-wheel-config.cmake index 5be201fda8e..130dfac3c7b 100644 --- a/tools/cmake/executorch-wheel-config.cmake +++ b/tools/cmake/executorch-wheel-config.cmake @@ -89,8 +89,8 @@ # CUDAToolkit_ROOT or CMake's standard toolkit # search. A program that calls the CUDA runtime # itself links CUDA::cudart. -# executorch::backend_openvino The OpenVINO delegate. Linux only. Opens the -# OpenVINO runtime by name, which a C++ program +# executorch::backend_openvino The OpenVINO delegate. Linux and Windows. Opens +# the OpenVINO runtime by name, which a C++ program # installs and points OPENVINO_LIB_PATH at. # executorch::threadpool The shared thread pool. # executorch::etdump The profiler. Not on Windows.