From 3d2f3e49b728924e32dbdb514bb0b25523225021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Harboe?= Date: Wed, 4 Nov 2026 11:00:03 +0100 Subject: [PATCH] DependencyInstaller: the Python lock in a virtual environment on Ubuntu 23.04+ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Ubuntu 23.04 and later the system Python is externally managed (PEP 668), and the installer took Python packages from apt only: etc/requirements-common_lock.txt was never installed there. Ubuntu 24.04 and 26.04 are what the Docker images and CI run, so google-cloud-pubsub, which apt does not carry, was missing in them, and the other packages came at apt's versions, not the lock's. _installPipCommon installs the lock into the virtual environment VIRTUAL_ENV names, on any OS, and creates it if it does not exist. The dev image names one, /opt/orfs-python, and puts it first on PATH, so python3 and PYTHON_EXE are its interpreter in the images. Without VIRTUAL_ENV, Ubuntu 23.04+ keeps the apt packages, as before, and -common says how to get the rest. macOS already required a virtual environment; it now goes through the same lines. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Øyvind Harboe --- docker/Dockerfile.dev | 5 +++++ etc/DependencyInstaller.sh | 33 ++++++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index df31e072e7..2b9436711a 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -23,6 +23,11 @@ RUN sh /tmp/setup_compiler_wrappers.sh && rm /tmp/setup_compiler_wrappers.sh ENV PATH="/usr/local/bin/wrapped-cc:$PATH" +# The lock in requirements-common_lock.txt is installed into this virtual +# environment: the system Python is externally managed (PEP 668). +ENV VIRTUAL_ENV=/opt/orfs-python +ENV PATH="${VIRTUAL_ENV}/bin:$PATH" + # The last rm drops the unused net-imap default gem bundled with Ubuntu 22.04's # Ruby 3.0 (flagged by CVE scanners, never used by the flow). It is chained here # so the files don't persist in an earlier layer. No-op if not present. diff --git a/etc/DependencyInstaller.sh b/etc/DependencyInstaller.sh index dc3cb17cb3..d98925a0a3 100755 --- a/etc/DependencyInstaller.sh +++ b/etc/DependencyInstaller.sh @@ -154,17 +154,21 @@ _installPipCommon() { local lockfile piplock lockfile="${_script_dir}/requirements-common_lock.txt" piplock="${_script_dir}/requirements-pip_lock.txt" - if [[ "$OSTYPE" == "darwin"* ]]; then + if [[ -n "${VIRTUAL_ENV:-}" ]]; then + # The lock goes into the virtual environment, created if it does + # not exist yet; the Docker images name one with ENV VIRTUAL_ENV. + if [[ ! -x "${VIRTUAL_ENV}/bin/python3" ]]; then + python3 -m venv "${VIRTUAL_ENV}" + fi + "${VIRTUAL_ENV}/bin/python3" -m pip install --no-cache-dir --upgrade --require-hashes -r "$piplock" + "${VIRTUAL_ENV}/bin/python3" -m pip install --no-cache-dir -r "$lockfile" + elif [[ "$OSTYPE" == "darwin"* ]]; then if [[ "$EUID" -eq 0 ]]; then echo "Error: Do NOT run with sudo." exit 1 fi - if [[ -n "${VIRTUAL_ENV:-}" ]]; then - python3 -m pip install --no-cache-dir -r "$lockfile" - else - echo "Error: Activate a virtual environment on macOS." - exit 1 - fi + echo "Error: Activate a virtual environment on macOS." + exit 1 else # pip before 23.3 fails --require-hashes on requirements-common_lock.txt: # it does not match an extras-bearing pin such as @@ -726,11 +730,22 @@ case "${os}" in _installPipCommon fi else - # Python packages come from apt (PEP 668), so they belong - # with the other system packages in -base. + # The system Python is externally managed (PEP 668): its + # packages come from apt, in -base with the other system + # packages. The lock, which also has packages apt does not + # carry, goes into a virtual environment when one is named. if [[ "${option}" == "base" || "${option}" == "all" ]]; then _installPipSystem fi + if [[ "${option}" == "common" || "${option}" == "all" ]]; then + if [[ -n "${VIRTUAL_ENV:-}" ]]; then + _installPipCommon + else + echo "Python packages from apt only. For those in" \ + "etc/requirements-common_lock.txt that apt does" \ + "not carry, set VIRTUAL_ENV and run -common again." + fi + fi fi else echo "Skip pip packages for rodete"