_llm_server.yml

name: LLM Server Tests

on:
  workflow_call:
    inputs:
      docker-image:
        description: Name of the docker image to use, without registry or tag suffix.
        required: false
        type: string
        default: ci-image:executorch-ubuntu-22.04-clang12

jobs:
  docker-image:
    name: Resolve CI docker image
    uses: ./.github/workflows/_docker-image.yml

  linux:
    needs: docker-image
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    with:
      runner: mt-l-x86iavx512-8-64
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/${{ inputs.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: recursive
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 60
      script: |
        set -eux

        eval "$(conda shell.bash hook)"
        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
        conda activate "${CONDA_ENV}"

        python -m pip install --progress-bar off \
          -r examples/llm_server/python/requirements.txt \
          httpx \
          pytest \
          tokenizers

        export PYTHONPATH="$(dirname "${PWD}"):${PYTHONPATH:-}"
        export PYTHONDONTWRITEBYTECODE=1

        python -m pytest -q \
          examples/llm_server/python/tests \
          examples/models/muse-glimmer/tests/test_serve.py \
          examples/models/muse-glimmer/tests/test_serve_batching.py \
          examples/models/muse-glimmer/tests/test_stream_parser.py

        cmake -S . -B cmake-out \
          -DCMAKE_BUILD_TYPE=Release \
          -DCMAKE_INSTALL_PREFIX=cmake-out \
          -DEXECUTORCH_BUILD_EXTENSION_LLM=ON \
          -DEXECUTORCH_BUILD_EXTENSION_LLM_RUNNER=ON
        cmake --build cmake-out --target install -j "$(nproc)"

        cmake -S examples/llm_server/cpp -B cmake-out/examples/llm_server/cpp \
          -DCMAKE_PREFIX_PATH="${PWD}/cmake-out"
        cmake --build cmake-out/examples/llm_server/cpp \
          --target test_worker_prefill_plan test_worker_loop test_multiplexed_worker test_batching_worker \
          -j "$(nproc)"
        ctest --test-dir cmake-out/examples/llm_server/cpp --output-on-failure

        export EXECUTORCH_BATCHING_TEST_WORKER="${PWD}/cmake-out/examples/llm_server/cpp/test_batching_worker"
        test -x "${EXECUTORCH_BATCHING_TEST_WORKER}"
        python -m pytest -q examples/llm_server/python/tests/test_batching_end_to_end.py