run

#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Error out if a command fails or a variable is not defined
set -eu

#===============================================================================
# Default values for environment variables.
#===============================================================================

init_globals() {
    # Paths
    if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then
        CALLER=$(readlink -f "$0")
    fi
    export RUN_SCRIPT_FILE="$(readlink -f "${BASH_SOURCE[0]}")"
    export TOP=$(dirname "${RUN_SCRIPT_FILE}")

    # Versions
    export SUPPORTED_CUDA_VERSIONS=("12" "13")
    export CUDA_DEFAULT_VERSION="12"

    # Executables
    export HOLOSCAN_PY_EXE=${HOLOSCAN_PY_EXE:-"python3"}
    export HOLOSCAN_DOCKER_EXE=${HOLOSCAN_DOCKER_EXE:-"docker"}

    # Options
    export DO_DRY_RUN="false"  # print commands but do not execute them. Used by run_command
    export DO_STANDALONE="false"  # do not run prerequisite functions
    export NO_CACHE="" # by default, use cache

    # Define default img and dir names
    export SDK_BUILD_IMG="holoscan-sdk-build"
    export SDK_RUN_IMG="holoscan-sdk-run"
    export SDK_BUILD_DIR="build"
    export SDK_INSTALL_DIR="install"

    # Use buildkit for docker build
    export DOCKER_BUILDKIT=1

    # Only enabled TTY if supported
    [ -t 1 ] && export USE_TTY="--tty" || export USE_TTY=""
}

#===============================================================================
# Utility functions
#===============================================================================

# Build Type ===================================================================

get_buildtype_str() {
    local build_type="${1:-}"
    local build_type_str

    case "${build_type}" in
        debug|Debug)
            build_type_str="Debug"
            ;;
        release|Release)
            build_type_str="Release"
            ;;
        rel-debug|RelWithDebInfo)
            build_type_str="RelWithDebInfo"
            ;;
        *)
            build_type_str="${CMAKE_BUILD_TYPE:-Release}"
            ;;
    esac

    echo -n "${build_type_str}"
}

# Boolean ======================================================================

get_boolean() {
    local bool_in="${1:-}"
    local bool_out

    case "${bool_in}" in
        true|yes|1)
            bool_out=true
            ;;
        false|no|0)
            bool_out=false
            ;;
        *)
            return 1
    esac

    echo -n "${bool_out}"
}

# Archs, GPUs, CUDA =================================================================

checkif_x86_64() {
    if [ $(get_host_arch) == "x86_64" ]; then
        return 0
    else
        return 1
    fi
}

checkif_aarch64() {
    if [ $(get_host_arch) == "aarch64" ]; then
        return 0
    else
        return 1
    fi
}

get_host_arch() {
    echo -n "$(uname -m)"
}

is_cross_compiling() {
    [ "$ARCH" != $(get_host_arch) ]
}

get_gnu_arch_str() {
    local arch="${1:-}"
    local arch_str

    case "${arch}" in
        amd64|x86_64|x86|linux/amd64)
            arch_str="x86_64"
            ;;
        arm64|aarch64|arm|linux/arm64)
            arch_str="aarch64"
            ;;
        *)
            return 1
    esac

    echo -n "${arch_str}"
}

get_docker_arch_str() {
    local arch="${1:-}"
    local arch_str

    case "${arch}" in
        amd64|x86_64|x86|linux/amd64)
            arch_str="amd64"
            ;;
        arm64|aarch64|arm|linux/arm64)
            arch_str="arm64"
            ;;
        *)
            return 1
    esac

    echo -n "${arch_str}"
}

get_platform_str() {
    echo -n "linux/$(get_docker_arch_str $ARCH)"
}

get_gpu_str() {
    local gpu="${1:-}"
    local gpu_str=$gpu

    case "${gpu}" in
        igpu|iGPU|integrated|Jetson)
            gpu_str="igpu"
            ;;
        dgpu|dGPU|discrete|RTX)
            gpu_str="dgpu"
            ;;
    esac

    echo -n "${gpu_str}"
}

get_host_gpu() {
    if ! command -v nvidia-smi >/dev/null; then
        c_echo_err Y "Could not find any GPU drivers on host. Defaulting build to target dGPU/CPU stack."
        echo -n "dgpu"
    elif nvidia-smi  2>/dev/null | grep nvgpu -q; then
        echo -n "igpu"
    else
        echo -n "dgpu"
    fi
}

is_cross_gpu() {
    [ "$GPU" != $(get_host_gpu) ]
}

get_default_cuda_major_version() {
    echo -n "${CUDA_DEFAULT_VERSION}"
}

validate_cuda_major_version() {
    local cuda_major="${1:-}"
    # Check if cuda_major is in the supported list
    local found=0
    for version in "${SUPPORTED_CUDA_VERSIONS[@]}"; do
        if [[ "$cuda_major" == "$version" ]]; then
            found=1
            break
        fi
    done

    if [[ $found -eq 0 ]]; then
        c_echo_err R "Invalid CUDA version: '${cuda_major}'. Supported versions are: ${SUPPORTED_CUDA_VERSIONS[*]}"
        return 1
    fi

    echo -n "${cuda_major}"
}

get_build_flavor_str() {
    local suffix="cu${CUDA_MAJOR}"
    if [ "$ARCH" = "aarch64" ]; then
        suffix+="-$ARCH-$GPU"
    else
        suffix+="-$ARCH"
    fi
    echo -n "$suffix"
}

get_build_img_name() {
    echo -n "${SDK_BUILD_IMG}-$(get_build_flavor_str)"
}

get_run_img_name() {
    echo -n "${SDK_RUN_IMG}-$(get_build_flavor_str)"
}

get_build_dir() {
    echo -n "${SDK_BUILD_DIR}-$(get_build_flavor_str)"
}

get_install_dir() {
    echo -n "${SDK_INSTALL_DIR}-$(get_build_flavor_str)"
}

get_driver_version() {
    if command -v nvidia-smi >/dev/null 2>&1; then
        # Pick the first unique driver version reported (covers multi-GPU setups).
        nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null \
            | head -n1 | tr -d ' \n'
    else
        echo -n ""   # No driver found – let the caller decide what to do.
    fi
}

# CUDA =========================================================================

get_cuda_archs() {
    local cuda_archs="${1:-}"

    # match https://cmake.org/cmake/help/latest/prop_tgt/CUDA_ARCHITECTURES.html
    case "${cuda_archs}" in
        native|NATIVE)
            cuda_archs_str="native"
            ;;
        all|ALL)
            cuda_archs_str="all"
            ;;
        all-major|ALL-MAJOR|major|MAJOR)
            cuda_archs_str="all-major"
            ;;
        *)
            cuda_archs_str="${1:-}"
            ;;
    esac

    echo -n "${cuda_archs_str}"
}

# ID ===========================================================================

get_group_id() {
    local group=$1
    cat /etc/group | grep $group | cut -d: -f3
}

# Version ======================================================================

get_git_sha() {
    echo -n "$(git rev-parse --short=9 HEAD)"
}

get_git_branch() {
    # Can't use "/" in tag, convert to "-"
    echo -n "$(git branch --show-current | sed 's|/|_|g')"
}

get_git_tag() {
    echo -n "$(git tag --points-at HEAD)"
}

get_version() {
    echo -n "$(cat $TOP/VERSION)"
}

get_image_tag_flags() {
    local img_name=$1
    local tags=("latest" $(get_version) $(get_git_branch) $(get_git_tag) $(get_git_sha))
    local flags=""
    for tag in "${tags[@]}"; do
        flags+="-t ${img_name}:${tag} "
    done
    echo -n "$flags"
}

#===============================================================================
# Section: CLI
#===============================================================================

install_python_dev_deps() {
    if [ -n "${VIRTUAL_ENV-}" ] || [ -n "${CONDA_PREFIX-}" ]; then
        run_command ${HOLOSCAN_PY_EXE} -m pip install -q -r ${TOP}/python/requirements.cu${CUDA_MAJOR}.txt
        run_command ${HOLOSCAN_PY_EXE} -m pip install -q -r ${TOP}/python/requirements.dev.txt
    else
        c_echo_err R "You must be in a virtual environment to install dependencies."
        if [ ! -e "$TOP/.venv/dev/bin/python3" ]; then
            c_echo_err W "Installing a virtual environment at " G "$TOP/.venv/dev" W " ..."
            run_command ${HOLOSCAN_PY_EXE} -m venv "$TOP/.venv/dev"
        fi

        c_echo_err W "Please activate the virtual environment at " G "$TOP/.venv/dev" W " and execute this command again."
        c_echo_err
        c_echo_err G "  source $TOP/.venv/dev/bin/activate"
        c_echo_err G "  $0 $@"
        exit 1
    fi
}

setup_cli_dev_desc() { c_echo 'Setup development environment for Holoscan CLI
'
}
setup_cli_dev() {
    c_echo W "Setup Holoscan CLI development environment..."

    if [ -f "/.dockerenv" ]; then
        c_echo_err "WARNING: devcontainer is not supported for CLI development"
        exit
    fi

    install_python_dev_deps

    local build_path=$TOP/$(get_build_dir)/python/lib/holoscan
    if [ -d "${build_path}" ]; then
        c_echo "Found build directory: $build_path"
    else
        c_echo_err "Build directory ${build_path} not found, please run ./run build first"
        exit
    fi

    DEST=$TOP/python/holoscan
    c_echo "Removing existing symlinks from $DEST/**/*.so"
    find $DEST -name *.so -delete
    c_echo "Creating symlinks from $build_path/* to $DEST"
    cp -as $build_path/* $DEST &> /dev/null
}

#===============================================================================
# Section: Build
#===============================================================================

clear_cache_desc() { c_echo 'Clear cache folders (including build/install folders)
'
}
clear_cache() {
    c_echo W "Clearing cache..."
    run_command rm -rf ${TOP}/build
    run_command rm -rf ${TOP}/install
    run_command rm -rf ${TOP}/build-*
    run_command rm -rf ${TOP}/install-*

    run_command rm -rf ${TOP}/src/core/distributed/generated/*

    run_command rm -rf ${TOP}/.cache/ccache
    run_command rm -rf ${TOP}/.cache/cpm
    run_command rm -rf ${TOP}/.cache/gxf

    # Clean up old build images, keeping only IDs associated with a ':latest' tag
    c_echo W "Cleaning up old '${SDK_BUILD_IMG}-*' images and docker buildx cache..."
    local base_img_pattern="${SDK_BUILD_IMG}-*"
    local latest_ids=$(docker images -q --filter="reference=${base_img_pattern}:latest" | sort -u)
    local all_ids=$(docker images -q --filter="reference=${base_img_pattern}" | sort -u)
    local ids_to_delete=$(echo "$all_ids" | grep -v -f <(echo "$latest_ids"))
    if [ -n "$ids_to_delete" ]; then
        run_command docker rmi -f $ids_to_delete || true
    fi
    run_command docker buildx prune -f --filter 'type=regular' # don't delete cache mounts
}

check_system_deps_desc() { c_echo 'Check system dependencies

Ensure that the system has all adequate prerequisites to be
able to build the Holoscan SDK.
'
}
check_system_deps() {
    c_echo W "Setup development environment..."

    if ! command -v ${HOLOSCAN_DOCKER_EXE} > /dev/null; then
        fatal G "${HOLOSCAN_DOCKER_EXE}" W " doesn't exists. Please install Docker:

        https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository"
    fi

    if ! $(${HOLOSCAN_DOCKER_EXE} buildx version &> /dev/null) ; then
        fatal G "${HOLOSCAN_DOCKER_EXE} buildx plugin" W " is missing. Please install " G "docker-buildx-plugin" W ":

        https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository"
    fi

    if ! groups | grep -e docker -e root -q; then
        c_echo_err G "groups" W " doesn't contain 'docker' group. Please add 'docker' group to your user."
        fatal G "groups" W " doesn't contain 'docker' group. Please add 'docker' group to your user." B '
    # Create the docker group.
    sudo groupadd docker
    # Add your user to the docker group.
    sudo usermod -aG docker $USER
    newgrp docker
    docker run hello-world'
    fi

    if checkif_x86_64 && is_cross_compiling && [ ! -f /proc/sys/fs/binfmt_misc/qemu-aarch64 ]; then
        fatal G "qemu-aarch64" W " doesn't exists. Please install qemu with binfmt-support to run Docker container with aarch64 platform" B '
    # Install the qemu packages
    sudo apt-get install qemu binfmt-support qemu-user-static
    # Execute the registering scripts
    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes'
    fi

    # Check NVIDIA CTK version
    min_ctk_version="1.12.0"
    recommended_ctk_version="1.14.1"
    if ! command -v nvidia-ctk > /dev/null; then
        fatal G "nvidia-ctk" W " not found. Please install the NVIDIA Container Toolkit."
    fi
    ctk_version_output=$(nvidia-ctk --version | grep version)
    ctk_version_pattern="([0-9]+\.[0-9]+\.[0-9]+)"
    if [[ "$ctk_version_output" =~ $ctk_version_pattern ]]; then
        ctk_version="${BASH_REMATCH[1]}"
        if [[ "$(echo -e "$ctk_version\n$min_ctk_version" | sort -V | head -n1)" == "$ctk_version" ]]; then
            fatal "Found nvidia-ctk Version $ctk_version. Version $min_ctk_version+ is required ($recommended_ctk_version+ recommended)."
        fi
    else
        c_echo_err R "Could not extract nvidia-ctk version number."
        c_echo_err "  Version $min_ctk_version+ required."
        c_echo_err "  Version $recommended_ctk_version+ recommended."
    fi
}

build_image_desc() { c_echo 'Build the image where we can build the SDK

This will generate an image will the following tags:
- `latest`
- from VERSION file
- from git branch name
- from git tag, if any
- from git commit sha - used in the build and launch methods
'
}
build_image() {
    # Prerequisite steps
    if [ "$DO_STANDALONE" != "true" ]; then
        check_system_deps
    fi

    # Error if requesting iGPU and x86_64 when cross-compiling
    if [ "$ARCH" = "x86_64" ] && [ "$GPU" = "igpu" ]; then
        fatal R "Can't combine 'x86_64' arch with 'igpu' GPU type"
    fi

    # Docker tags (-t name:versions)
    local docker_tags=$(get_image_tag_flags $(get_build_img_name))

    # Docker build
    local extra_args="$@"
    run_command $HOLOSCAN_DOCKER_EXE build \
        --build-arg BUILDKIT_INLINE_CACHE=1 \
        --build-arg GPU_TYPE=${GPU} \
        --build-arg CUDA_MAJOR=${CUDA_MAJOR} \
        --platform $(get_platform_str) \
        --network=host \
        ${NO_CACHE} \
        ${docker_tags} \
        ${extra_args} \
        ${TOP}
}

build_desc() { c_echo 'Build the SDK with CMake

This command will build the SDK with optional configuration arguments.

Arguments:
  --build_libtorch [true | false] : Build the SDK with libtorch support
        Default: true
        Associated environment variable: HOLOSCAN_BUILD_LIBTORCH
  --config : Use this option to specify additional CMake configuration options.
        Example: --config "-DHOLOSCAN_BUILD_EXAMPLES:BOOL=ON"
  --cudaarchs [native | all | <custom_arch_list>]
        Default: "native" when building for host architecture, "all" otherwise
        Associated environment variable: CMAKE_CUDA_ARCHITECTURES
  --type [debug | release | rel-debug] : Specify the type of build
        Default: release
        Associated environment variable: CMAKE_BUILD_TYPE
  --parallel, -j <njobs> : Specify the maximum number of concurrent processes to be used when building
        Default: maximum (number of CPUs, using 'nproc')
        Associated environment variable: CMAKE_BUILD_PARALLEL_LEVEL
  --buildpath, -d <build_directory> : Change the build path.
        Default: build-<arch>[-<gpu>]
        Associated environment variable: CMAKE_BUILD_PATH
  --installprefix, -i <install_directory> : Specify the install directory
        Default: install-<arch>[-<gpu>]
        Associated environment variable: CMAKE_INSTALL_PREFIX
  --reconfigure, -f: Force reconfiguration of the CMake project. By default CMake is only run if
        a CMakeCache.txt does not exist in the build directory or if CMake detects a reconfigure
        is needed.
        Default: false
  --tidy [true | false] : Build the SDK with clang-tidy (will be slower)
        Default: false
        Associated environment variable: HOLOSCAN_ENABLE_CLANG_TIDY
  --tidy-headers [true | false] : Also check project headers with clang-tidy (requires --tidy true)
        Default: false
        Associated environment variable: HOLOSCAN_CLANG_TIDY_CHECK_HEADERS
  --build-python [true | false] : Build the SDK without python support
        Default: true
        Associated environment variable: HOLOSCAN_BUILD_PYTHON
  --sccache [true | false] : Use sccache for build caching (speeds up rebuilds)
        Default: true
        Associated environment variable: HOLOSCAN_USE_SCCACHE
        Optional: Set SCCACHE_MEMCACHED_ENDPOINT for distributed caching (e.g., server:11211)
'
}
build() {
    # Prerequisite steps
    if [ "$DO_STANDALONE" != "true" ]; then
        build_image
    fi

    # Default cuda architecture
    local default_cuda_archs="native"
    if is_cross_compiling || is_cross_gpu; then
        default_cuda_archs="all"
    fi

    # Parse env variables first or set default values
    local cuda_archs=$(get_cuda_archs "${CMAKE_CUDA_ARCHITECTURES:-$default_cuda_archs}")
    local build_type=$(get_buildtype_str "${CMAKE_BUILD_TYPE:-release}")
    local build_njobs="${CMAKE_BUILD_PARALLEL_LEVEL:-$(nproc)}"
    local build_path="${CMAKE_BUILD_PATH:-$(get_build_dir)}"
    local install_prefix="${CMAKE_INSTALL_PREFIX:-$(get_install_dir)}"
    local reconfigure=false
    local build_libtorch="${HOLOSCAN_BUILD_LIBTORCH:-ON}"
    local build_python="${HOLOSCAN_BUILD_PYTHON:-ON}"
    local enable_clang_tidy="${HOLOSCAN_ENABLE_CLANG_TIDY:-OFF}"
    local clang_tidy_check_headers="${HOLOSCAN_CLANG_TIDY_CHECK_HEADERS:-OFF}"
    local use_sccache="${HOLOSCAN_USE_SCCACHE:-ON}"
    local config_args=""

    # Mount sccache directory to container.
    local sccache_rel_dir="/.cache/sccache"
    local sccache_host_dir
    if [[ -n "${SCCACHE_DIR:-}" ]]; then
        sccache_host_dir="$SCCACHE_DIR"
    else
        sccache_host_dir="${HOME}${sccache_rel_dir}"
    fi
    local sccache_dir_opt=""
    if [[ "$use_sccache" == "ON" ]]; then
        mkdir -p "$sccache_host_dir"
        sccache_dir_opt="-v $sccache_host_dir:$sccache_rel_dir -e SCCACHE_DIR=$sccache_rel_dir"
    fi

    # Check if memcached endpoint is reachable before passing it to container
    local sccache_memcached_opt=""
    if [ -n "${SCCACHE_MEMCACHED_ENDPOINT-}" ]; then
        local endpoint="${SCCACHE_MEMCACHED_ENDPOINT}"
        local host="${endpoint%:*}"
        local port="${endpoint##*:}"

        # Test if the endpoint is reachable (timeout after 5 seconds)
        if timeout 5 bash -c "echo -n > /dev/tcp/${host}/${port}" 2>/dev/null; then
            sccache_memcached_opt="-e SCCACHE_MEMCACHED_ENDPOINT"
        else
            c_echo_err Y "Warning: SCCACHE_MEMCACHED_ENDPOINT is set to ${endpoint} but server is not reachable."
            c_echo_err Y "         Proceeding without distributed caching."
        fi
    fi

    # Parse args
    local extra_args=""
    while [[ $# -gt 0 ]]; do
    case $1 in
        --build-python)
            build_python_val=$(get_boolean "$2")
            if [ "$build_python_val" == "false" ]; then
                build_python='OFF'
            fi
            reconfigure=true
            shift
            shift
        ;;
        --build_libtorch)
            build_libtorch_val=$(get_boolean "$2")
            if [ "$build_libtorch_val" == "false" ]; then
                build_libtorch='OFF'
            fi
            reconfigure=true
            shift
            shift
        ;;
        --tidy)
            tidy_val=$(get_boolean "$2")
            if [ "$tidy_val" == "true" ]; then
                enable_clang_tidy='ON'
            fi
            reconfigure=true
            shift
            shift
        ;;
        --tidy-headers)
            tidy_headers_val=$(get_boolean "$2")
            if [ "$tidy_headers_val" == "true" ]; then
                clang_tidy_check_headers='ON'
            fi
            reconfigure=true
            shift
            shift
        ;;
        --sccache)
            sccache_val=$(get_boolean "$2")
            if [ "$sccache_val" == "true" ]; then
                use_sccache='ON'
            else
                use_sccache='OFF'
            fi
            reconfigure=true
            shift
            shift
        ;;
        --cudaarchs)
            cuda_archs=$(get_cuda_archs "$2")
            reconfigure=true
            shift
            shift
        ;;
        --type)
            build_type=$(get_buildtype_str "$2")
            reconfigure=true
            shift
            shift
        ;;
        --parallel|-j)
            build_njobs="$2"
            shift
            shift
        ;;
        --buildpath|-d)
            build_path="$2"
            shift
            shift
        ;;
        --installprefix|-i)
            install_prefix="$2"
            shift
            shift
        ;;
        --reconfigure|-f)
            reconfigure=true
            shift
        ;;
        --config | -c)
            config_args+=("$2")
            reconfigure=true
            shift
            shift
            ;;
        *)
            extra_args+=("$1")
            shift
        ;;
    esac
    done

    # Error if requesting native cuda arch explicitly when cross-compiling
    if [ "$cuda_archs" = "native" ] && (is_cross_compiling || is_cross_gpu); then
        fatal Y "Cross-compiling " W "(host: $(get_host_arch), target: $ARCH)" R " does not support 'native' cuda architecture."
    fi

    # Native means we need the container to access the GPU for CMake to choose the architecture
    # to use for nvcc.
    if [ "$cuda_archs" = "native" ]; then
        runtime=nvidia
    else
        runtime=runc
    fi

    # DOCKER PARAMETERS
    #
    # --rm
    #   Deletes the container after the command runs
    #
    # --net=host
    #   Provide access to the host network
    #
    # --ulimit memlock=-1
    # --ulimit stack=67108864
    #   Set resource limits for memory locking (unlimited) and stack size (64MB), to prevent
    #   out-of-memory issues on systems with high core counts during parallel builds.
    #
    # --interactive (-i)
    #   Making the container interactive allows cancelling the build
    #
    # ${USE_TTY}
    #   Set --tty (-t) when TTY is possible only
    #
    # --entrypoint=bash
    #   Override the default entrypoint to hide the regular message from the base image
    #
    # --runtime ${runtime}
    #   Docker runtime. Should ideally be runc for build (no need for drivers).
    #
    # --platform $(get_platform_str)
    #   Platform to build
    #
    # -u $(id -u):$(id -g)
    #   Ensures the generated files (build, install...) are owned by $USER and not root
    #
    # -v ${TOP}:/workspace/holoscan-sdk
    #   Mount the source directory
    #
    # -w /workspace/holoscan-sdk
    #   Start in the source directory
    #
    #
    # CMAKE PARAMETERS
    #
    # -S . -B ${build_path}
    #   Generic configuration
    #
    # -D CMAKE_BUILD_TYPE=${build_type}
    #   Define the build type (release, debug...). Can be passed as env to docker also.
    #
    # -D CMAKE_CUDA_ARCHITECTURES=${cuda_archs}
    #   Define the cuda architectures to build for (native, all, all-major, custom). Apart from
    #   native, the last arch will be used for real and virtual architectures (PTX, forward
    #   compatible) while the previous archs will be real only.
    #
    # CMAKE BUILD COMMAND
    #
    # cmake --build ${build_path} -j ${build_njobs}
    #   Builds the SDK using the build path at ${build_path} with number of concurrent
    #   jobs given by -j ${build_njobs}
    #
    img="$(get_build_img_name):$(get_git_sha)"
    config_args="${config_args[@]}"
    run_command $HOLOSCAN_DOCKER_EXE run \
        --rm \
        --net=host \
        --ulimit memlock=-1 \
        --ulimit stack=67108864 \
        --interactive \
        ${USE_TTY} \
        --entrypoint=bash \
        --runtime=${runtime} \
        --platform $(get_platform_str) \
        -u $(id -u):$(id -g) \
        -v ${TOP}:/workspace/holoscan-sdk \
        -w /workspace/holoscan-sdk \
        ${sccache_dir_opt} \
        ${sccache_memcached_opt} \
        ${extra_args[@]} \
        $img \
        -c "set -e
            if [ ! -f '${build_path}/build.ninja' ] || ${reconfigure} ; then
                cmake -S . -B ${build_path} -G Ninja \
                    -D HOLOSCAN_CUDA_MAJOR_VERSION=${CUDA_MAJOR} \
                    -D HOLOSCAN_BUILD_LIBTORCH=${build_libtorch} \
                    -D HOLOSCAN_BUILD_PYTHON=${build_python} \
                    -D HOLOSCAN_PYTHON_EXAMPLES=${build_python} \
                    -D HOLOSCAN_ENABLE_CLANG_TIDY=${enable_clang_tidy} \
                    -D HOLOSCAN_CLANG_TIDY_CHECK_HEADERS=${clang_tidy_check_headers} \
                    -D HOLOSCAN_USE_SCCACHE=${use_sccache} \
                    -D CMAKE_CUDA_ARCHITECTURES='${cuda_archs}' \
                    -D CMAKE_BUILD_TYPE=${build_type} \
                    ${config_args}
            fi
            cmake --build ${build_path} -j ${build_njobs}
            if [ '${use_sccache}' = 'ON' ]; then
                sccache --show-stats > ${build_path}/sccache-stats.txt
            fi

            # Install Holoscan SDK and its dependencies
            cmake --install ${build_path} --prefix ${install_prefix} --component holoscan
            cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-core
            cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-gxf_extensions
            cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-examples
            cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-gxf_libs
            cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-gxf_bins
            cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-modules
            cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-dependencies
            cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-python_libs
            cmake --install ${build_path} --prefix ${install_prefix} --component cli11
            cmake --install ${build_path} --prefix ${install_prefix} --component concurrentqueue
            cmake --install ${build_path} --prefix ${install_prefix} --component dlpack
            cmake --install ${build_path} --prefix ${install_prefix} --component fmt_core
            cmake --install ${build_path} --prefix ${install_prefix} --component matx
            cmake --install ${build_path} --prefix ${install_prefix} --component nvtx3
            cmake --install ${build_path} --prefix ${install_prefix} --component rapids_logger
            cmake --install ${build_path} --prefix ${install_prefix} --component rmm
            cmake --install ${build_path} --prefix ${install_prefix} --component spdlog
            cmake --install ${build_path} --prefix ${install_prefix} --component tl-expected
            cmake --install ${build_path} --prefix ${install_prefix} --component ucxx
            # CCCL libraries
            cmake --install ${build_path} --prefix ${install_prefix} --component CCCL
            cmake --install ${build_path} --prefix ${install_prefix} --component CUB
            cmake --install ${build_path} --prefix ${install_prefix} --component cudax
            cmake --install ${build_path} --prefix ${install_prefix} --component libcudacxx
            cmake --install ${build_path} --prefix ${install_prefix} --component Thrust

            # Install Holoscan Sensor Bridge (if enabled)
            if grep -q "HOLOSCAN_BUILD_HOLOLINK:BOOL=ON" ${build_path}/CMakeCache.txt; then
                hololink_install_prefix=${install_prefix}/../install-hololink
                mkdir -p \${hololink_install_prefix}
                hololink_install_prefix=\$(realpath \${hololink_install_prefix})
                for component in \
                    hololink-camera-sensor \
                    hololink-cmake \
                    hololink-common \
                    hololink-core \
                    hololink-emulation \
                    hololink-emulation-sensors \
                    hololink-examples \
                    hololink-module \
                    hololink-modules \
                    hololink-module-tools \
                    hololink-operators \
                    hololink-python_libs \
                    hololink-python_tools \
                    hololink-roce-operators \
                    hololink-scripts \
                    hololink-sensors \
                    hololink-tools
                do
                    cmake --install ${build_path} --prefix \${hololink_install_prefix} --component \$component
                done
             else
                echo \"Hololink is not built, skipping installation\"
            fi
        "
}

build_run_image_desc() { c_echo 'Build the runtime image

Build a lightweight docker image meant for running applications only.

Arguments:
  --cpp : Only build with C++ dependencies
  --cpp-no-mkl : (x86_64 only) Only build with C++ dependencies apart from MKL (torch dependency)
'
}

build_run_image() {
    # Prerequisite steps
    if [ "$DO_STANDALONE" != "true" ]; then
        build $@
    fi

    # Parse args
    local stage="runtime_cpp_py"
    local extra_args=""
    while [[ $# -gt 0 ]]; do
    case $1 in
        --cpp)
            stage="runtime_cpp"
            shift
        ;;
        --cpp-no-mkl)
            stage="runtime_cpp_no_mkl"
            shift
        ;;
        *)
            extra_args+=("$1")
            shift
        ;;
    esac
    done

    # Current build image to copy from some stages
    local current_build_img="$(get_build_img_name):$(get_git_sha)"

    # Run image tags (-t name:versions)
    local docker_tags=$(get_image_tag_flags $(get_run_img_name))

    # Build the runtime image
    run_command $HOLOSCAN_DOCKER_EXE build \
        --build-arg BUILDKIT_INLINE_CACHE=1 \
        --build-arg BUILD_IMAGE=${current_build_img} \
        --build-arg HOST_INSTALL_DIR=$(get_install_dir) \
        --build-arg GPU_TYPE=${GPU} \
        --platform $(get_platform_str) \
        --network=host \
        --target ${stage} \
        ${NO_CACHE} \
        ${docker_tags} \
        ${extra_args[@]} \
        -f ${TOP}/runtime_docker/Dockerfile \
        ${TOP}
}

#===============================================================================
# Section: Documentation
#===============================================================================

build_docs_desc() { c_echo 'Build and validate the Fern user guide

Runs the repository documentation pipeline from the current checkout. Any
remaining arguments are passed to docs/scripts/build_holoscan_docs.py.
With no arguments and no FERN_TOKEN, generated library MDX is skipped; when no
local API tree exists, validation omits the API-reference section.

Examples:
  ./public/run build_docs
  ./run build_docs                 # public repository layout
  ./public/run build_docs --skip-library-mdx
'
}
build_docs() {
    local docs_args=("$@")
    if [[ $# -eq 0 && -z "${FERN_TOKEN:-}" ]]; then
        docs_args=(--skip-library-mdx)
    fi
    run_command "${HOLOSCAN_PY_EXE}" \
        "${TOP}/docs/scripts/build_holoscan_docs.py" "${docs_args[@]}"
}

live_docs_desc() { c_echo 'Start a foreground local Fern user-guide preview

Runs the same documentation pipeline with --preview. Any remaining arguments
are passed to docs/scripts/build_holoscan_docs.py after --preview.
With no arguments and no FERN_TOKEN, the local preview omits the API-reference
section when no generated library MDX exists.

Examples:
  ./public/run live_docs
  ./run live_docs                   # public repository layout
  ./public/run live_docs --skip-library-mdx
'
}
live_docs() {
    local docs_args=(--preview "$@")
    if [[ $# -eq 0 && -z "${FERN_TOKEN:-}" ]]; then
        docs_args+=(--skip-library-mdx)
    fi
    build_docs "${docs_args[@]}"
}

#===============================================================================
# Section: Test
#===============================================================================

lint_desc() { c_echo 'Lint the repository (via pre-commit)

Runs all pre-commit hooks (copyright, ruff, cpplint, cmakelint, codespell, markdownlint)
on the whole repo. Configuration is in .pre-commit-config.yaml at repo root.

pre-commit is resolved automatically:
  1. uvx pre-commit  -- if uvx (from uv) is available (isolated, no pip pollution)
  2. pre-commit      -- if already installed
  3. pip install      -- as a last resort

Arguments:
  When no arguments are given, runs all hooks with --all-files.
  When arguments are given, they are passed directly to pre-commit run.

Examples:
  ./run lint                                    # all hooks, all files
  ./run lint update-notice-files --all-files    # single hook, all files
  ./run lint ruff-check --all-files             # single hook, all files
'
}
lint() {
    local REPO_ROOT
    REPO_ROOT="$(git -C "${TOP}" rev-parse --show-toplevel 2>/dev/null)" || REPO_ROOT="$(dirname "${TOP}")"
    if [ ! -f "${REPO_ROOT}/.pre-commit-config.yaml" ]; then
        c_echo_err R "Not found: ${REPO_ROOT}/.pre-commit-config.yaml"
        return 1
    fi
    local pre_commit_cmd
    if command -v uvx &>/dev/null; then
        pre_commit_cmd="uvx pre-commit"
    else
        if ! command -v pre-commit &>/dev/null; then
            c_echo W "Neither pre-commit nor uvx available, installing pre-commit via pip..."
            run_command ${HOLOSCAN_PY_EXE} -m pip install pre-commit
        fi
        pre_commit_cmd="pre-commit"
    fi
    c_echo W "Running pre-commit (copyright + lint hooks) on all files"
    if [ $# -gt 0 ]; then
        (cd "${REPO_ROOT}" && run_command ${pre_commit_cmd} run "$@")
    else
        (cd "${REPO_ROOT}" && run_command ${pre_commit_cmd} run --all-files)
    fi
}

test_desc() { c_echo 'Execute test cases with CTest

This command will run the tests using CTest. It will run headless
if no DISPLAY is found.

Arguments:

  --name, -n <regex>        Name of test(s) to execute (regular expression)
                            Default: all tests
  --verbose, -v             Print the test outputs
  --timeout, -t <seconds>   CTest timeout in seconds
                            Default: 480
  --options, -o <opts>      Extra flags to pass to CTest
                            Default: None
'
}
test() {
    local test_regex_flag=""
    local verbose=""
    local timeout=480
    local options=""

    # Parse CLI arguments next
    local extra_args=""
    while [[ $# -gt 0 ]]; do
    case $1 in
        --name|-n)
            local test_regex="$2"
            local test_regex_uppercase="${test_regex^^}"
            test_regex_flag="-R '$test_regex|$test_regex_uppercase'"
            shift
            shift
        ;;
        --verbose|-v)
            verbose="--verbose"
            shift
        ;;
        --timeout|-t)
            timeout="$2"
            shift
            shift
        ;;
        --options|-o)
            options="$2"
            shift
            shift
        ;;
        --no-xvfb)
            local no_xvfb="true"
            shift
        ;;
        *)
            extra_args+=("$1")
            shift
        ;;
    esac
    done

    # Workaround for a bug of the 550 driver where TLS resources are not released and the Vulkan
    # driver fails to initialize after a certain number of calls to vkCreateInstance.
    local preload=""
    if checkif_aarch64; then
        local driver_version=$(get_driver_version)
        local driver_major_version=$(echo "$driver_version" | cut -d'.' -f1)
        if [ "$driver_major_version" = 550 ]; then
            local candidate="/usr/lib/aarch64-linux-gnu/libnvidia-tls.so.${driver_version}"
            if [ -e "$candidate" ]; then
                preload="LD_PRELOAD=$candidate"
            fi
        fi
    fi

    # --init
    #   Use tini as entrypoint to forward proper signals to xvfb (below)
    #
    # --cap-drop=NET_BIND_SERVICE
    #   needed for SYSTEM_TEST that depends on privileged ports being unbindable
    #
    # $run_headless (xvfb-run -a)
    #   creates a display buffer for Holoviz use, the `-a` flag automatically finds
    #   an open display port, and sets the DISPLAY env var. This is installed inside
    #   the container at runtime so it isn't installed in the public container.
    #   Additionally, `xvfb-run -a docker run ...` is not used, as no solution was found
    #   that could reliably set the `-e DISPLAY` parameter using this approach.
    #
    if [ -z ${no_xvfb-} ]; then
        local run_headless=$([ -z ${DISPLAY-} ] && echo "xvfb-run -a")
    else
        local run_headless=""
    fi

    # Give the ctest container its own /dev/shm (overridable via the environment).
    # launch()'s default --ipc=host shares the CI Docker-in-Docker host's 64 MB
    # /dev/shm and ignores --shm-size, so the integrated HSB (Holoscan Sensor
    # Bridge) Python tests exhaust it on import (ENOSPC, masquerading as a full
    # disk). A private namespace with --shm-size mirrors HSB's standalone CI
    # (ci/ci.sh: --shm-size=1gb, no --ipc=host); UCX still works via CAP_SYS_PTRACE.
    : "${HOLOSCAN_DOCKER_IPC:=private}"
    : "${HOLOSCAN_DOCKER_SHM_SIZE:=1gb}"
    export HOLOSCAN_DOCKER_IPC HOLOSCAN_DOCKER_SHM_SIZE

    # Run ctest and capture output while also displaying it in real-time
    local ctest_output
    local ctest_exit_code

    # Use a temporary file to capture output while still displaying it
    local temp_output=$(mktemp)

    # Run ctest and tee output to both console and temp file
    launch $(get_build_dir) \
        --init \
        --cap-drop=NET_BIND_SERVICE \
        ${extra_args[@]} \
        --run-cmd "$preload $run_headless ctest . $test_regex_flag --timeout $timeout --output-on-failure $verbose $options" 2>&1 | tee "$temp_output"

    ctest_exit_code=${PIPESTATUS[0]}
    ctest_output=$(cat "$temp_output")
    rm -f "$temp_output"

    # Check if ctest output contains "No tests were found"
    if echo "$ctest_output" | grep -q "No tests were found"; then
        c_echo_err R "Error: No tests were found!"
        return 1
    fi

    # Return the original ctest exit code
    return $ctest_exit_code
}

#===============================================================================
# Section: Launch
#===============================================================================

launch_desc() { c_echo 'Launch the Holoscan build container

Note: any extraneous arguments not listed below will be added to `docker run`

Arguments:
    $1 - Working directory (e.g, "install" => "/workspace/holoscan-sdk/install")
         Default: build-<arch>[-<gpu>]
    --mount-point - Specifies the mount point (default is the directory of this script)
    --run-cmd - Specifies a command to run in the container instead of running interactively.
                This is the equivalent of what you would put after `bash -c` with `docker run`.
    --ssh-x11 : [DEPRECATED] X11/Wayland forwarding is now auto-detected from DISPLAY/WAYLAND_DISPLAY
'
}
launch() {
    # Prerequisite steps
    if [ "$DO_STANDALONE" != "true" ]; then
        build_image
    fi

    local container_mount="/workspace/holoscan-sdk"
    local mount_point="${TOP}"
    local mount_device_opt=""
    local display_server_opt=""
    local display_xauth_file=""
    local extra_args=""
    local run_cmd="bash"
    local ssh_x11=0

    # Skip the first argument to pass the remaining arguments to the docker command.
    local working_dir=${1:-$(get_build_dir)}
    if [ -n "${1-}" ]; then
        shift
    fi

    # Parse CLI arguments next
    while [[ $# -gt 0 ]]; do
    case $1 in
        --mount-point)
            mount_point="$2"
            shift
            shift
        ;;
        --run-cmd)
            run_cmd="$2"
            shift
            shift
        ;;
        --ssh-x11)
            c_echo_err Y "Warning: --ssh-x11 is deprecated and ignored; X11/Wayland forwarding now happens automatically when DISPLAY or WAYLAND_DISPLAY is set."
            ssh_x11=1
            shift
        ;;
        *)
            extra_args+=("$1")
            shift
        ;;
    esac
    done

    local host_mount_to_top=$(realpath --relative-to="$mount_point" "$TOP")
    local container_top="${container_mount}/${host_mount_to_top}"

    # Mount V4L2 device nodes
    for video_dev in $(find /dev -regex '/dev/video[0-9]+'); do
        mount_device_opt+=" --device $video_dev"
    done

    # Mount Tegra's Video Input unit (capture data from CSI) device nodes
    for capture_dev in $(find /dev -regex '/dev/capture-vi-channel[0-9]+'); do
        mount_device_opt+=" --device $capture_dev"
    done

    # Mount ConnectX device nodes
    if [ -e /dev/infiniband/rdma_cm ]; then
        mount_device_opt+=" --device /dev/infiniband/rdma_cm"
    fi
    for uverbs_dev in $(find /dev -regex '/dev/infiniband/uverbs[0-9]+'); do
        mount_device_opt+=" --device $uverbs_dev"
    done

    # The device nodes under /dev/dri are owned by the root user and video and render groups,
    # so using a non-root user in the container requires to access these groups. Adding the group
    # names to `docker run --group-add` might not work as the group ids in the container and on the
    # host device might not match. Mounting `/etc/group` doesn't address it since the mount occurs
    # after the user and its groups are setup. Instead, we pass the group ids which we compute on
    # the host before starting the container.
    groups=""
    video_id=$(get_group_id video)
    if [ -n "$video_id" ]; then
        groups+=" --group-add $video_id"
    fi
    render_id=$(get_group_id render)
    if [ -n "$render_id" ]; then
        groups+=" --group-add $render_id"
    fi

    # Add docker group to enable DooD
    groups+=" --group-add $(get_group_id docker)"

    # Forward graphical applications when the host exposes a display.
    if [ -z "${DISPLAY-}" ] && [ -z "${WAYLAND_DISPLAY-}" ]; then
        info "No DISPLAY or WAYLAND_DISPLAY set; skipping display forwarding."
    else
        if [ -n "${XDG_SESSION_TYPE-}" ]; then
            display_server_opt+=" -e XDG_SESSION_TYPE"
        fi

        # Required by Vulkan, dconf, pipewire, etc. on both X11 and Wayland.
        if [ -n "${XDG_RUNTIME_DIR-}" ] && [ -d "${XDG_RUNTIME_DIR}" ]; then
            display_server_opt+=" -e XDG_RUNTIME_DIR"
            display_server_opt+=" -v ${XDG_RUNTIME_DIR}:${XDG_RUNTIME_DIR}"
        fi

        if [ -n "${WAYLAND_DISPLAY-}" ]; then
            display_server_opt+=" -e WAYLAND_DISPLAY"
        fi

        if [ -n "${DISPLAY-}" ]; then
            case "${DISPLAY}" in
                :*|unix:*|unix/:*)
                    if [ -d /tmp/.X11-unix ]; then
                        display_server_opt+=" -v /tmp/.X11-unix:/tmp/.X11-unix:ro"
                    fi
                    ;;
                *)
                    ;;
            esac
            display_server_opt+=" -e DISPLAY"

            # If the X server already accepts no-cookie connections (typically
            # SI:localuser peer-credential auth), skip the cookie setup. The
            # container runs as $(id -u) via -u, so the same auth family will
            # accept its socket connection.
            local peer_creds_sufficient=0
            if command -v xset >/dev/null \
                && XAUTHORITY=/dev/null timeout 2 xset -display "${DISPLAY}" q >/dev/null 2>&1; then
                peer_creds_sufficient=1
            fi
            if [ "${peer_creds_sufficient}" -eq 1 ]; then
                : # peer-creds is sufficient; no xauth setup needed
            elif ! command -v xauth >/dev/null; then
                c_echo_err Y "Warning: xauth not found on host; install xauth (or x11-xauth) so X11 applications can authenticate inside the container."
            else
                local xauth_entries=""
                if ! xauth_entries=$(xauth nlist "${DISPLAY}" 2>/dev/null) || [ -z "${xauth_entries}" ]; then
                    c_echo_err Y "Warning: xauth nlist returned no entries for DISPLAY=${DISPLAY}; X11 may not authenticate inside the container."
                else
                    local xauth_file=""
                    # umask 077 ensures the cookie file is created with mode 0600
                    # atomically; mktemp on POSIX already does this, but we set
                    # umask explicitly so the guarantee does not depend on the
                    # mktemp implementation.
                    xauth_file=$(umask 077 && mktemp "${TMPDIR:-/tmp}/.docker.xauth.XXXXXX")
                    if printf "%s\n" "${xauth_entries}" | sed -e 's/^..../ffff/' | xauth -f "${xauth_file}" nmerge - 2>/dev/null; then
                        display_xauth_file="${xauth_file}"
                        # Register cleanup so the cookie is removed even on SIGINT/SIGTERM/SIGHUP.
                        # shellcheck disable=SC2064
                        trap "rm -f '${display_xauth_file}'" EXIT INT TERM HUP
                        display_server_opt+=" -v ${display_xauth_file}:${display_xauth_file}:ro"
                        display_server_opt+=" -e XAUTHORITY=${display_xauth_file}"
                    else
                        rm -f "${xauth_file}"
                        c_echo_err Y "Warning: xauth nmerge failed for DISPLAY=${DISPLAY}; X11 may not authenticate inside the container."
                    fi
                fi
            fi
        fi
    fi

    # DOCKER PARAMETERS
    #
    # --rm
    #   Deletes the container after the command runs
    #
    # --net=host
    #   Provide access to the host network
    #
    # --interactive (-i)
    #   The container needs to be interactive to be able to interact with the X11 windows
    #
    # ${USE_TTY}
    #   Set --tty when TTY is possible only
    #
    # --entrypoint=bash
    #   Override the default entrypoint to hide the regular message from the base image
    #
    # -u $(id -u):$(id -g)
    # -v /etc/group:/etc/group:ro
    # -v /etc/passwd:/etc/passwd:ro
    # ${groups} (--group-add render; --group-add video)
    #   Run the container as a non-root user. See details above for `groups` variable.
    #
    # -v /var/run/docker.sock:/var/run/docker.sock
    #    Enable the use of Holoscan CLI with Docker outside of Docker (DooD) for packaging and
    #    running applications inside the container.
    #
    # -v ${TOP}:/workspace/holoscan-sdk
    #   Mount the source directory
    #
    # -w /workspace/holoscan-sdk/${working_dir}
    #   Start in the build or install directory
    #
    # --runtime=nvidia \
    #   Enable GPU acceleration with access to all GPUs by default
    #
    # ${display_server_opt}
    #   Enable graphical applications (X11 or Wayland)
    #
    # ${mount_device_opt}
    #   --device /dev/video${i}
    #     Mount video capture devices for V4L2
    #
    #   --device /dev/capture-vi-channel${i}
    #     Mount Tegra's Video Input unit to capture from CSI
    #
    #   --device /dev/infiniband/...
    #     Mount ConnectX SmartNIC for MOFED
    #
    # -e PYTHONPATH
    # -e HOLOSCAN_LIB_PATH
    # -e HOLOSCAN_INPUT_PATH
    #   Define paths needed by the python applications
    #
    # -e CUPY_CACHE_DIR
    #   Define path for cupy' kernel cache, needed since $HOME does
    #   not exist when running with `-u id:group`
    #
    # --ipc=${HOLOSCAN_DOCKER_IPC:-host}
    #   IPC namespace for the container. Defaults to `host` so UCX can share
    #   memory with processes outside the container. Override with
    #   HOLOSCAN_DOCKER_IPC (e.g. `private`) to give the container its own
    #   /dev/shm whose size is then controlled by --shm-size (see below) instead
    #   of inheriting the host's /dev/shm. The `test` command sets this to a
    #   private namespace by default; see the note there.
    #
    # ${HOLOSCAN_DOCKER_SHM_SIZE:+--shm-size=...}
    #   Size of the container's /dev/shm. Only takes effect with a non-`host`
    #   IPC namespace (Docker ignores --shm-size under --ipc=host, where the
    #   host's /dev/shm is used directly). Unset by default; set via
    #   HOLOSCAN_DOCKER_SHM_SIZE.
    #
    # --cap-add=CAP_SYS_PTRACE
    #   Allow process tracing to enable shared memory transport for UCX
    #   (UCX's cross-memory-attach relies on CAP_SYS_PTRACE, not on --ipc=host)
    #
    # --cap-add=CAP_SYS_NICE
    # --ulimit rtprio=10
    #   Allow setting process priorities and scheduling policies for real-time scheduling
    #
    # --ulimit memlock=-1
    # --ulimit stack=67108864
    #   Set resource limits for memory locking (unlimited), stack size (64MB), to prevent
    #   out-of-memory issues on systems with high core counts.
    #
    # Container launch command steps:
    # - Append the Holoscan bin folder to the existing `PATH` before running
    # - Add the GXF module to PYTHONPATH before running

    img="$(get_build_img_name):$(get_git_sha)"
    pythonpath=$(docker inspect ${img} --format '{{.Config.Env}}' | grep -oP 'PYTHONPATH=\K[^ ]+' || true)
    ld_library_path=$(docker inspect ${img} --format '{{.Config.Env}}' | grep -oP 'LD_LIBRARY_PATH=\K[^ ]+' || true)

    # Workaround: Required to support cuda-compat-13-0 on Jetpack 7.0 platform.
    # Expected library mount path from Jetpack 7.0 with nvidia-container-toolkit
    if [ "${CUDA_MAJOR}" -eq 13 ] && [ -e /usr/lib/aarch64-linux-gnu/nvidia ]; then
        ld_library_path="/usr/lib/aarch64-linux-gnu/nvidia:${ld_library_path}"
    fi

    local launch_status=0
    run_command $HOLOSCAN_DOCKER_EXE run \
        --rm \
        --net=host \
        --interactive \
        ${USE_TTY} \
        --entrypoint=bash \
        -u $(id -u):$(id -g) \
        -v /etc/group:/etc/group:ro \
        -v /etc/passwd:/etc/passwd:ro \
        ${groups} \
        -v /var/run/docker.sock:/var/run/docker.sock \
        -v ${mount_point}:${container_mount} \
        -w ${container_top}/${working_dir} \
        --runtime=nvidia \
        ${display_server_opt} \
        ${mount_device_opt} \
        -e LD_LIBRARY_PATH=${ld_library_path} \
        -e PYTHONPATH=${pythonpath}:${container_top}/${working_dir}/python/lib \
        -e HOLOSCAN_LIB_PATH=${container_top}/${working_dir}/lib \
        -e HOLOSCAN_INPUT_PATH=${container_top}/data \
        -e HOLOSCAN_TESTS_DATA_PATH=${container_top}/tests/data \
        -e CUPY_CACHE_DIR=${container_top}/.cupy/kernel_cache \
        --ipc=${HOLOSCAN_DOCKER_IPC:-host} \
        ${HOLOSCAN_DOCKER_SHM_SIZE:+--shm-size=${HOLOSCAN_DOCKER_SHM_SIZE}} \
        --cap-add=CAP_SYS_PTRACE \
        --cap-add=CAP_SYS_NICE \
        --ulimit rtprio=10 \
        --ulimit memlock=-1 \
        --ulimit stack=67108864 \
        ${extra_args[@]} \
        $img -c "
            export PATH=\$PATH:${container_top}/${working_dir}/bin;
            $run_cmd" || launch_status=$?

    if [ -n "${display_xauth_file}" ]; then
        rm -f "${display_xauth_file}"
        trap - EXIT INT TERM HUP
    fi

    return ${launch_status}
}

vscode_desc() { c_echo 'Launch VSCode in DevContainer

Launch a VSCode instance in a Docker container with the development environment.
If `cursor` command is available, Cursor will be launched instead.

Arguments:
  --parallel, -j <njobs> : Specify the maximum number of concurrent processes to be used when building
        Default: maximum (number of CPUs, using 'nproc')
        Associated environment variable: CMAKE_BUILD_PARALLEL_LEVEL
  --ide <ide_name> : Specify the IDE to use
        Default: vscode
        Supported IDEs: vscode, vscode-insiders, cursor
  --cmd <path> : Specify the command or path to the IDE binary (such as 'code')
  --code: Launch VSCode for DevContainer. (using `code` command)
  --cursor: Launch Cursor for DevContainer. (using `cursor` command)
'
}
vscode() {
    local build_njobs="${CMAKE_BUILD_PARALLEL_LEVEL:-$(nproc)}"
    local ide_name="vscode"
    local ide_bin="code"

    # Check if cursor command is available
    if command -v cursor &> /dev/null; then
        ide_name="cursor"
        ide_bin="cursor"
    fi

    # Parse CLI arguments next
    local args=("$@")
    while [[ $# -gt 0 ]]; do
        case $1 in
            --parallel|-j)
                build_njobs="$2"
                shift 2
                ;;
            --ide)
                ide_name="$2"
                shift 2
                if [ "$ide_name" == "vscode" ]; then
                    ide_bin="code"
                elif [ "$ide_name" == "vscode-insiders" ]; then
                    ide_bin="code-insiders"
                elif [ "$ide_name" == "cursor" ]; then
                    ide_bin="cursor"
                else
                    ide_bin="$ide_name"
                    echo_c W "Using binary '${ide_bin}' for VSCode-based IDE"
                fi
                ;;
            --cmd)
                ide_bin="$2"
                shift 2
                ;;
            --code)
                ide_name="vscode"
                ide_bin="code"
                shift
                ;;
            --cursor)
                ide_name="cursor"
                ide_bin="cursor"
                shift
                ;;
            *)
                # If $1 starts with a dash, it's an unknown option
                if [[ $1 == -* ]]; then
                    c_echo_err R "Unknown option: $1"
                fi
                shift
                ;;
        esac
    done

    if ! command -v "${ide_bin}" > /dev/null; then
        fatal R "No '${ide_bin}' available. Please install " W "VSCode or Cursor" R " to use VSCode DevContainer. Follow the instructions in https://code.visualstudio.com/Download"
    fi

    local workspace_path=$(git rev-parse --show-toplevel 2> /dev/null || dirname $(realpath -s $0))
    local workspace_path_hex

    if [ "${workspace_path}" != "${TOP}" ]; then
        # Set the environment variable 'HOLOSCAN_PUBLIC_FOLDER' to the resolved path relative to the
        # workspace path if the workspace path is different than the TOP folder so that
        # the DevContainer can build files under the Holoscan source root folder while mounting the
        # workspace folder.
        run_command export HOLOSCAN_PUBLIC_FOLDER=$(realpath --relative-to=${workspace_path} ${TOP})
    fi

    # These environment variables will be passed to the container via `devcontainer.json`
    # during the creation of the VSCode Docker dev container.
    # - CMAKE_BUILD_PARALLEL_LEVEL: without this setting, `make` will not limit the number of jobs
    # running concurrently, which could lead to memory exhaustion.
    # - HOLOSCAN_BUILD_IMAGE: name of the base image of the docker dev container
    export CMAKE_BUILD_PARALLEL_LEVEL=${build_njobs}
    export HOLOSCAN_BUILD_IMAGE="$(get_build_img_name):$(get_git_sha)"

    # Install the VSCode Remote Development extension.
    # Checking if the extension is already installed is slow, so we always install it so that
    # the installation can be skipped if the extension is already installed.
    if [ "${ide_name}" == "vscode" ] || [ "${ide_name}" == "vscode-insiders" ]; then
        run_command "${ide_bin}" --force --install-extension ms-vscode-remote.vscode-remote-extensionpack
    elif [ "${ide_name}" == "cursor" ]; then
        run_command "${ide_bin}" --force --install-extension anysphere.remote-containers
        # Install C/C++ themes as a host extension
        run_command "${ide_bin}" --force --install-extension ms-vscode.cpptools-themes
    fi

    # For now, there is no easy way to launch DevContainer directly from the command line.
    # ('devcontainer' CLI is only available after manually executing 'Remote-Containers: Install devcontainer CLI' command)
    # Here, we launch VSCode with .devcontainer/devcontainer.json with '--folder-uri' option to open the current folder in the container.
    # See https://github.com/microsoft/vscode-remote-release/issues/2133
    workspace_path_hex=$(echo -n ${workspace_path} | xxd -p)
    workspace_path_hex="${workspace_path_hex//[[:space:]]/}"
    c_echo B "Local workspace path: " W "${workspace_path} (${workspace_path_hex})"
    run_command "${ide_bin}" --folder-uri "vscode-remote://dev-container+${workspace_path_hex}/workspace/holoscan-sdk"
}

vscode_remote_desc() { c_echo 'Attach to the existing VSCode DevContainer

This command is useful when you want to attach to the existing VSCode DevContainer from the remote machine.

1) Launch VSCode DevContainer on the local machine with "./run vscode" command.
2) In the remote machine, launch VSCode Remote-SSH extension to connect to the local machine.
3) Launch this command on the remote machine to attach to the existing VSCode DevContainer in the local machine.
'
}
vscode_remote() {
    if ! command -v code > /dev/null; then
        fatal R "Please install " W "VSCode" R " to use VSCode DevContainer. Follow the instructions in https://code.visualstudio.com/Download"
    fi

    local workspace_path=$(git rev-parse --show-toplevel 2> /dev/null || dirname $(realpath -s $0))

    local container_name=$(run_command $HOLOSCAN_DOCKER_EXE ps --filter "label=devcontainer.local_folder=${workspace_path}" --format "{{.Names}}" | head -1)
    if [ -z "${container_name}" ]; then
        fatal R "No DevContainer (with the label 'devcontainer.local_folder=${workspace_path}') is found in the local machine. Please launch VSCode DevContainer with " W "./run vscode" R " command first."
    fi
    c_echo W "Container name: " y "${container_name}"

    # The following command is based on the information of the following issue:
    #   https://github.com/microsoft/vscode-remote-release/issues/2133#issuecomment-618328138
    python - > /tmp/holoscan_remote_uri.txt << EOF
import json
from urllib.parse import urlunparse
from subprocess import check_output
hostname = check_output("hostname" , shell=True).decode('utf-8').strip()
remote_description = json.dumps({"containerName": f"/${container_name}", "settings":{"host": f"ssh://{hostname}"}})
remote_uri = urlunparse(('vscode-remote', f'attached-container+{remote_description.encode("utf8").hex()}', '/workspace/holoscan-sdk', '', '', ''))
print(remote_uri)
EOF
    local remote_uri=$(cat /tmp/holoscan_remote_uri.txt)
    c_echo W "Remote URI: " y "${remote_uri}"

    # VSCode's attached-container mode does not support forwarding SSH keys to the container well.
    # For the reason, we need to copy SSH keys to the container so that the container can access
    # the local machine's SSH keys.
    c_echo W "Copying SSH keys to the container..."
    for i in identity id_rsa id_ed25519 id_ed25519_sk id_dsa id_cdsa id_ecdsa_sk; do
        if [ -e ~/.ssh/${i} ]; then
            run_command $HOLOSCAN_DOCKER_EXE cp ~/.ssh/${i} ${container_name}:/home/holoscan/.ssh/
        fi
    done;

    run_command code --folder-uri ${remote_uri}
}

#===============================================================================
# Logging utils
#===============================================================================

my_cat_prefix() {
    local IFS
    local prefix="$1"
    local line
    while IFS= read -r line; do
        echo "${prefix}${line}" # -e option doesn't work in 'sh' so disallow escaped characters
    done <&0
}

c_str() {
    local old_color=39
    local old_attr=0
    local color=39
    local attr=0
    local text=""
    local mode="color"
    if [ "${1:-}" = "color" ]; then
        mode="color"
        shift
    elif [ "${1:-}" = "nocolor" ]; then
        mode="nocolor"
        shift
    fi

    for i in "$@"; do
        case "$i" in
            r|R)
                color=31
                ;;
            g|G)
                color=32
                ;;
            y|Y)
                color=33
                ;;
            b|B)
                color=34
                ;;
            p|P)
                color=35
                ;;
            c|C)
                color=36
                ;;
            w|W)
                color=37
                ;;

            z|Z)
                color=0
                ;;
        esac
        case "$i" in
            l|L|R|G|Y|B|P|C|W)
                attr=1
                ;;
            n|N|r|g|y|b|p|c|w)
                attr=0
                ;;
            z|Z)
                attr=0
                ;;
            *)
                text="${text}$i"
        esac
        if [ "${mode}" = "color" ]; then
            if [ ${old_color} -ne ${color} ] || [ ${old_attr} -ne ${attr} ]; then
                text="${text}\033[${attr};${color}m"
                old_color=$color
                old_attr=$attr
            fi
        fi
    done
    /bin/echo -en "$text"
}

c_echo() {
    # Select color/nocolor based on the first argument
    local mode="color"
    if [ "${1:-}" = "color" ]; then
        mode="color"
        shift
    elif [ "${1:-}" = "nocolor" ]; then
        mode="nocolor"
        shift
    else
        if [ ! -t 1 ]; then
            mode="nocolor"
        fi
    fi

    local old_opt="$(shopt -op xtrace)" # save old xtrace option
    set +x # unset xtrace

    if [ "${mode}" = "color" ]; then
        local text="$(c_str color "$@")"
        /bin/echo -e "$text\033[0m"
    else
        local text="$(c_str nocolor "$@")"
        /bin/echo -e "$text"
    fi
    eval "${old_opt}" # restore old xtrace option
}

echo_err() {
    >&2 echo "$@"
}

c_echo_err() {
    >&2 c_echo "$@"
}

printf_err() {
    >&2 printf "$@"
}

get_unused_ports() {
    local num_of_ports=${1:-1}
    local start=${2:-49152}
    local end=${3:-61000}
    comm -23 \
    <(seq ${start} ${end} | sort) \
    <(ss -tan | awk '{print $4}' | while read line; do echo ${line##*\:}; done | grep '[0-9]\{1,5\}' | sort -u) \
    | shuf | tail -n ${num_of_ports} # use tail instead head to avoid broken pipe in VSCode terminal
}

newline() {
    echo
}

info() {
    c_echo_err W "$(date -u '+%Y-%m-%d %H:%M:%S') [INFO] " Z "$@"
}

error() {
    c_echo_err R "$(date -u '+%Y-%m-%d %H:%M:%S') [ERROR] " Z "$@"
}

fatal() {
    if [ -n "$*" ]; then
        c_echo_err R "$(date -u '+%Y-%m-%d %H:%M:%S') [FATAL] " Z "$@"
        echo_err
    fi
    if [ -z "${CALLER-}" ]; then
        return 1
    else
        kill -INT $$  # kill the current process instead of exit in shell environment.
    fi
}

#===============================================================================
# Driver functions
#===============================================================================

run_command() {
    local status=0
    local cmd="$*"

    if [ "${DO_DRY_RUN}" != "true" ]; then
        c_echo_err B "$(date -u '+%Y-%m-%d %H:%M:%S') " W "\$ " G "${cmd}"
    else
        c_echo_err B "$(date -u '+%Y-%m-%d %H:%M:%S') " C "[dryrun] " W "\$ " G "${cmd}"
    fi

    [ "$(echo -n "$@")" = "" ] && return 1 # return 1 if there is no command available

    if [ "${DO_DRY_RUN}" != "true" ]; then
        "$@"
        status=$?
    fi

    return $status
}

#######################################
# Get list of available commands from a given input file.
#
# Available commands and command summary are extracted by checking a pattern
# "_desc() { c_echo '".
# Section title is extracted by checking a pattern "# Section: ".
# This command is used for listing available commands in CLI.
#
# e.g.)
#   "# Section: String/IO functions"
#     => "# String/IO functions"
#   "to_lower_desc() { c_echo 'Convert to lower case"
#     => "to_lower ----------------- Convert to lower case"
#
# Arguments:
#   $1 - input file that defines commands
# Returns:
#   Print list of available commands from $1
#######################################
get_list_of_available_commands() {
    local mode="color"
    if [ "${1:-}" = "color" ]; then
        mode="color"
        shift
    elif [ "${1:-}" = "nocolor" ]; then
        mode="nocolor"
        shift
    fi

    local file_name="$1"
    if [ ! -e "$1" ]; then
        echo "$1 doesn't exist!"
    fi

    local line_str='--------------------------------'
    local IFS= cmd_lines="$(IFS= cat "$1" | grep -E -e "^(([[:alpha:]_[:digit:]]+)_desc\(\)|# Section: )" | sed "s/_desc() *{ *c_echo [\"']/ : /")"
    local line
    while IFS= read -r line; do
        local cmd=$(echo "$line" | cut -d":" -f1)
        local desc=$(echo "$line" | cut -d":" -f2-)
        if [ "$cmd" = "# Section" ]; then
            c_echo ${mode} B "${desc}"
        else
            # there is no substring operation in 'sh' so use 'cut'
            local dash_line="$(echo "${line_str}" | cut -c ${#cmd}-)"  #  = "${line_str:${#cmd}}"
             c_echo ${mode} Y "   ${cmd}" w " ${dash_line} ${desc}"
        fi
        # use <<EOF, not '<<<"$cmd_lines"' to be executable in sh
    done <<EOF
$cmd_lines
EOF
}

get_list_of_global_flags() {
    echo_err
    c_echo_err W "Global Arguments"
    echo_err "
    --help, -h                  : Print help messages for [command]
    --dryrun                    : Print commands to screen without running
    --standalone, -s            : Do not run prerequisite steps for build-related commands
    --no-cache, -n              : Do not use cache when building docker images
    --arch, -a [amd64 | arm64]  : Specify the platform (for cross-compilation)
        Default: current host architecture ($(get_host_arch))
        Associated environment variable: HOLOSCAN_BUILD_ARCH
    --gpu, -g [igpu | dgpu]     : Specify the GPU stack (integrated vs discrete) for arm64
        Default: current host GPU ($(get_host_gpu))
    --cuda, -cu [12 | 13]       : Specify the CUDA major version
        Associated environment variable: HOLOSCAN_BUILD_GPU_TYPE
        Default: ${CUDA_DEFAULT_VERSION}
        Associated environment variable: HOLOSCAN_CUDA_MAJOR_VERSION
    "
}

print_usage() {
    set +x
    echo_err
    echo_err "USAGE: $0 [command] [arguments]..."
    echo_err
    c_echo_err W "Command List"
    echo_err "$(get_list_of_available_commands color "${RUN_SCRIPT_FILE}" | my_cat_prefix " ")"
    echo_err
    echo_err "$(get_list_of_global_flags)"
}

print_cmd_help_messages() {
    local cmd="${1-}"
    if [ -n "${cmd}" ]; then
        if type ${cmd}_desc > /dev/null 2>&1; then
            ${cmd}_desc
            get_list_of_global_flags
            return 0
        else
            c_echo_err R "Command '${cmd}' doesn't exist!"
            print_usage
            return 1
        fi
    fi
    print_usage
}

main() {
    local ret=0

    # Parse command
    if [ -z "${1-}" ]; then
        print_usage
        return 0
    fi
    local cmd=$1
    shift

    # Parse args
    local args=()
    unset ARCH GPU CUDA_MAJOR
    while [[ $# -gt 0 ]]; do
    case $1 in
        -h|--help)
            print_cmd_help_messages $cmd
            return 0
        ;;
        --dryrun|--dry-run)
            DO_DRY_RUN="true"
            shift
        ;;
        -s|--standalone)
            DO_STANDALONE="true"
            shift
        ;;
        -n|--no-cache)
            NO_CACHE="--no-cache"
            shift
        ;;
        -a|--arch)
            ARCH=$(get_gnu_arch_str $2)
            shift
            shift
        ;;
        -g|--gpu)
            GPU=$(get_gpu_str $2)
            shift
            shift
        ;;
        -cu|--cuda)
            CUDA_MAJOR=$(validate_cuda_major_version $2)
            shift
            shift
        ;;
        *)
            args+=("$1")
            shift
        ;;
    esac
    done

    # Ensure arch and gpu are defined with defaults if not passed explicitly
    if [ -z "${ARCH-}" ]; then
        ARCH=$(get_gnu_arch_str ${HOLOSCAN_BUILD_ARCH:-$(get_host_arch)})
    fi
    if [ -z "${GPU-}" ]; then
        GPU=$(get_gpu_str ${HOLOSCAN_BUILD_GPU_TYPE:-$(get_host_gpu)})
    fi
    if [ -z "${CUDA_MAJOR-}" ]; then
        CUDA_MAJOR=$(validate_cuda_major_version ${HOLOSCAN_CUDA_MAJOR_VERSION:-${CUDA_DEFAULT_VERSION}})
    fi

    # Run command
    case "$cmd" in
        ''|main|help)
            print_usage
            ;;
        *)
            if type ${cmd} > /dev/null 2>&1; then
                "$cmd" "${args[@]}"
            else
                c_echo_err R "Command '${cmd}' doesn't exist!"
                print_usage
                return 1
            fi
            ;;
    esac
    ret=$?
    if [ -z "${CALLER-}" ]; then
        return $ret
    fi
}

init_globals

if [ -z "${CALLER-}" ]; then
    main "$@"
fi

#===============================================================================
# Description template
#===============================================================================
# Globals:
#   HOLOSCAN_OS
#   HOLOSCAN_TARGET
#   HOLOSCAN_USER (used if HOLOSCAN_OS is "linux")
#   HOLOSCAN_HOST (used if HOLOSCAN_OS is "linux")
# Arguments:
#   Command line to execute
# Returns:
#   Outputs print messages during the execution (stdout->stdout, stderr->stderr).
#
#   Note:
#     This command removes "\r" characters from stdout.
#
#   Exit code:
#     exit code returned from executing a given command