build.yaml

name: Tests

on:
  push:
  pull_request:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read
  packages: read

jobs:
  blazingmq-dependency:
    name: Build BlazingMQ as a dependency
    runs-on: ubuntu-24.04
    outputs:
      deps_cache_key: ${{ steps.cache-key.outputs.deps_cache_key }}
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Compute dependency cache key
        id: cache-key
        run: echo "deps_cache_key=deps-${{ hashFiles('bin/clone-dependencies.sh') }}" >> $GITHUB_OUTPUT

      - name: Try to get cached BlazingMQ build artifacts
        id: cache-restore
        uses: actions/cache/restore@v6
        with:
          path: blazingmq_artifacts.tar.gz
          key: ${{ steps.cache-key.outputs.deps_cache_key }}

      - name: Set up dependencies
        if: steps.cache-restore.outputs.cache-hit != 'true'
        run: |
            sudo apt-get update
            sudo apt-get install -qy build-essential \
              gdb \
              curl \
              python3.10 \
              cmake \
              ninja-build \
              pkg-config \
              bison \
              libfl-dev \
              libbenchmark-dev \
              libgtest-dev \
              libgmock-dev \
              libz-dev

      - name: Create install directory for BlazingMQ and its dependencies
        if: steps.cache-restore.outputs.cache-hit != 'true'
        run: mkdir -p blazingmq_artifacts

      - name: Skip building bison, google-benchmark, and googletest
        if: steps.cache-restore.outputs.cache-hit != 'true'
        run: |
          mkdir -p thirdparty/{bison,google-benchmark,googletest}
          mkdir -p build/{bison,google-benchmark,googletest}
          touch thirdparty/bison/.complete
          touch build/google-benchmark/.complete
          touch build/googletest/.complete

      - name: Build and install BlazingMQ and its dependencies
        if: steps.cache-restore.outputs.cache-hit != 'true'
        env:
          DIR_INSTALL: 'blazingmq_artifacts'
        run: /bin/bash bin/build-manylinux.sh

      - name: Save built BlazingMQ build artifacts
        if: steps.cache-restore.outputs.cache-hit != 'true'
        run: tar czf blazingmq_artifacts.tar.gz blazingmq_artifacts

      - name: Cache built BlazingMQ build artifacts
        id: cache-save
        if: steps.cache-restore.outputs.cache-hit != 'true'
        uses: actions/cache/save@v6
        with:
          path: blazingmq_artifacts.tar.gz
          key: ${{ steps.cache-key.outputs.deps_cache_key }}

  linux-check:
    name: Test on Linux
    needs: blazingmq-dependency
    runs-on: ubuntu-24.04
    strategy:
      matrix:
        python-version:
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
          - "3.13"
          - "3.14"
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false
      - name: Try to get cached BlazingMQ build artifacts
        id: cache-restore
        uses: actions/cache/restore@v6
        with:
          path: blazingmq_artifacts.tar.gz
          key: ${{ needs.blazingmq-dependency.outputs.deps_cache_key }}
      - name: Restore cached BlazingMQ build artifacts
        run: tar xzf blazingmq_artifacts.tar.gz
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v7
        with:
          python-version: ${{ matrix.python-version }}
      - name: Create virtual environment
        run: |
          python3 -m venv venv
      - name: Install Python dependencies
        run: |
          ./venv/bin/python -m pip install --upgrade pip
          ./venv/bin/python -m pip install -r requirements-dev.txt
      - name: Install package manager dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential \
            gdb \
            curl \
            cmake \
            cmake \
            ninja-build \
            pkg-config \
            bison \
            libfl-dev \
            libbenchmark-dev \
            libgtest-dev \
            libgmock-dev \
            libz-dev
      - name: Log in to GHCR
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Start BlazingMQ broker
        run: |
          docker run -d --name bmqbrkr \
            --network host \
            -v ${{ github.workspace }}/tests/broker-config:/broker-config:ro \
            ghcr.io/bloomberg/blazingmq:latest \
            /usr/local/bin/bmqbrkr /broker-config
          timeout 30 bash -c 'until nc -z localhost 30114; do sleep 1; done' \
            || (docker logs bmqbrkr; exit 1)
      - name: Setup core dumps
        run: |
          sudo mkdir -p /cores
          sudo chmod 777 /cores
          echo "/cores/%e.%p.%s.%t" | sudo tee /proc/sys/kernel/core_pattern
      - name: Build and run tests
        env:
          BMQ_BROKER_URI: tcp://localhost:30114
          PREFIX: blazingmq_artifacts
          PYTHON: ./venv/bin/python
          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig:./blazingmq_artifacts/lib64/pkgconfig
        run: |
          ulimit -c unlimited
          make test-build && make test-install && make check
      - name: Print backtrace from core dump
        if: failure()
        run: |
          for core in /cores/*; do
            if [ -f "$core" ]; then
              echo "=== Backtrace from $core ==="
              gdb -batch \
                -ex "set print frame-arguments all" \
                -ex "bt full" \
                -ex "echo \n=== All threads ===\n" \
                -ex "thread apply all bt" \
                ./venv/bin/python "$core"
            fi
          done
      - name: Stop BlazingMQ broker
        if: always()
        run: docker rm -f bmqbrkr

  lint-docs:
    name: Lint and Docs
    needs: blazingmq-dependency
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false
      - name: Try to get cached BlazingMQ build artifacts
        id: cache-restore
        uses: actions/cache/restore@v6
        with:
          path: blazingmq_artifacts.tar.gz
          key: ${{ needs.blazingmq-dependency.outputs.deps_cache_key }}
      - name: Restore cached BlazingMQ build artifacts
        run: tar xzf blazingmq_artifacts.tar.gz
      - name: Set up Python 3.14
        uses: actions/setup-python@v7
        with:
          python-version: "3.14"
      - name: Create virtual environment
        run: |
          python3 -m venv venv
      - name: Install Python dependencies
        run: |
          ./venv/bin/python -m pip install --upgrade pip
          ./venv/bin/python -m pip install -r requirements-dev.txt
      - name: Set up dependencies
        run:
          sudo apt-get install clang-format
      - name: Install Package
        run: |
          ./venv/bin/python -m pip install -e .
        env:
          PREFIX: blazingmq_artifacts
          PYTHON: ./venv/bin/python
          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig:./blazingmq_artifacts/lib64/pkgconfig
      - name: Lint sources
        env:
          PYTHON: ./venv/bin/python
        run: |
          make lint
      - name: Build docs
        run: |
          ./venv/bin/python -m towncrier build --version 99.99 --name blazingmq --keep
          make docs
        env:
          PYTHON: ./venv/bin/python
          SPHINXBUILD: ../venv/bin/sphinx-build

  coverage:
    name: Coverage
    needs: blazingmq-dependency
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false
      - name: Try to get cached BlazingMQ build artifacts
        id: cache-restore
        uses: actions/cache/restore@v6
        with:
          path: blazingmq_artifacts.tar.gz
          key: ${{ needs.blazingmq-dependency.outputs.deps_cache_key }}
      - name: Restore cached BlazingMQ build artifacts
        run: tar xzf blazingmq_artifacts.tar.gz
      - name: Set up Python
        uses: actions/setup-python@v7
        with:
          python-version: "3.14"
      - name: Create virtual environment
        run: |
          python3 -m venv venv
      - name: Install Python dependencies
        run: |
          ./venv/bin/python3 -m pip install --upgrade pip
          ./venv/bin/python3 -m pip install -r requirements-dev.txt
      - name: Setup Core Dumps config
        run: |
          sudo mkdir /cores
          sudo chmod 777 /cores
          echo "/cores/%e.%p.%s.%t" | sudo tee /proc/sys/kernel/core_pattern
      - name: Log in to GHCR
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Start BlazingMQ broker
        run: |
          docker run -d --name bmqbrkr \
            --network host \
            -v ${{ github.workspace }}/tests/broker-config:/broker-config:ro \
            ghcr.io/bloomberg/blazingmq:latest \
            /usr/local/bin/bmqbrkr /broker-config
          timeout 30 bash -c 'until nc -z localhost 30114; do sleep 1; done' \
            || (docker logs bmqbrkr; exit 1)
      - name: Run tests with coverage
        env:
          BMQ_BROKER_URI: tcp://localhost:30114
          PREFIX: blazingmq_artifacts
          PYTHON: ./venv/bin/python
          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig:./blazingmq_artifacts/lib64/pkgconfig
        run: |
          # Allow core dumps
          ulimit -c unlimited
          make coverage-install && make coverage
      - name: Stop BlazingMQ broker
        if: always()
        run: docker rm -f bmqbrkr
      - name: Output code coverage summary
        uses: irongut/CodeCoverageSummary@v1.3.0
        with:
          filename: coverage*.xml
      - name: Upload broker logs as artifacts
        if: failure()
        run: |
          mkdir -p bmq/logs
          docker logs bmqbrkr > bmq/logs/broker_stdout.log 2> bmq/logs/broker_stderr.log || true
          docker cp bmqbrkr:/var/local/bmq/logs/. bmq/logs/ || true
      - uses: actions/upload-artifact@v7
        if: failure()
        with:
          name: broker_logs
          path: ./bmq/logs
          retention-days: 5
          compression-level: 9
      - name: Upload broker core dump as artifacts
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: core_dumps
          path: /cores
          retention-days: 5