From cb8c8d63a35756efc9888411e0d1c7e8651f0d1c Mon Sep 17 00:00:00 2001
From: Amp <amp@ampcode.com>
Date: Fri, 7 Aug 2026 00:39:24 +0000
Subject: [PATCH 1/3] ci: benchmark isolated pytest changes

Amp-Thread-ID: https://ampcode.com/threads/T-019fd7f4-525c-74fc-9a8e-d12146d0665d
Co-authored-by: Isaac Miller <isaac@cmpnd.ai>
---
 .github/workflows/paired_ci_benchmarks.yml | 72 ++++++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 .github/workflows/paired_ci_benchmarks.yml

diff --git a/.github/workflows/paired_ci_benchmarks.yml b/.github/workflows/paired_ci_benchmarks.yml
new file mode 100644
index 0000000000..0c74935c0f
--- /dev/null
+++ b/.github/workflows/paired_ci_benchmarks.yml
@@ -0,0 +1,72 @@
+name: Paired CI Performance Benchmarks
+
+on:
+  pull_request:
+
+permissions: {}
+
+jobs:
+  benchmark:
+    name: Benchmark ${{ matrix.change }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - change: memory-cache
+            candidate: perf/test-memory-cache
+          - change: litellm-server
+            candidate: perf/reuse-litellm-test-server
+          - change: asyncify
+            candidate: perf/deterministic-asyncify-test
+          - change: unbatchify
+            candidate: perf/reduce-unbatchify-test-timeout
+    steps:
+      - name: Check out exact parent
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          ref: 485ce445432af70317f7cc60768517356b3ded4d
+          path: baseline
+          persist-credentials: false
+      - name: Check out isolated candidate
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          ref: ${{ matrix.candidate }}
+          path: candidate
+          persist-credentials: false
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.11"
+      - name: Install uv with caching
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+        with:
+          enable-cache: true
+          cache-dependency-glob: |
+            baseline/pyproject.toml
+            baseline/uv.lock
+      - name: Install dependencies
+        run: |
+          uv sync --project baseline --dev -p baseline/.venv --extra dev
+          uv sync --project candidate --dev -p candidate/.venv --extra dev
+      - name: Run paired full-suite benchmarks
+        shell: bash
+        run: |
+          run_suite() {
+            local label="$1"
+            local directory="$2"
+            local start end elapsed
+            start=$(date +%s)
+            (cd "$directory" && uv run -p .venv pytest -q -n auto --dist worksteal tests/)
+            end=$(date +%s)
+            elapsed=$((end - start))
+            echo "CI_BENCHMARK change=${{ matrix.change }} label=$label elapsed_seconds=$elapsed"
+          }
+
+          # Reverse the order in round two so runner warmup and filesystem cache
+          # effects do not consistently favor either revision.
+          run_suite baseline-r1 baseline
+          run_suite candidate-r1 candidate
+          run_suite candidate-r2 candidate
+          run_suite baseline-r2 baseline

From 436acfad74a696bba59e695eff8c335a4a1e0d61 Mon Sep 17 00:00:00 2001
From: Amp <amp@ampcode.com>
Date: Fri, 7 Aug 2026 00:40:25 +0000
Subject: [PATCH 2/3] ci: create benchmark virtual environments

Amp-Thread-ID: https://ampcode.com/threads/T-019fd7f4-525c-74fc-9a8e-d12146d0665d
Co-authored-by: Isaac Miller <isaac@cmpnd.ai>
---
 .github/workflows/paired_ci_benchmarks.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/paired_ci_benchmarks.yml b/.github/workflows/paired_ci_benchmarks.yml
index 0c74935c0f..01628a9dc3 100644
--- a/.github/workflows/paired_ci_benchmarks.yml
+++ b/.github/workflows/paired_ci_benchmarks.yml
@@ -48,6 +48,8 @@ jobs:
             baseline/uv.lock
       - name: Install dependencies
         run: |
+          uv venv baseline/.venv
+          uv venv candidate/.venv
           uv sync --project baseline --dev -p baseline/.venv --extra dev
           uv sync --project candidate --dev -p candidate/.venv --extra dev
       - name: Run paired full-suite benchmarks

From b77e37b634bf75fcc86f0ab7164c6692345e5e42 Mon Sep 17 00:00:00 2001
From: Amp <amp@ampcode.com>
Date: Fri, 7 Aug 2026 00:42:50 +0000
Subject: [PATCH 3/3] ci: match benchmark command to CI

Amp-Thread-ID: https://ampcode.com/threads/T-019fd7f4-525c-74fc-9a8e-d12146d0665d
Co-authored-by: Isaac Miller <isaac@cmpnd.ai>
---
 .github/workflows/paired_ci_benchmarks.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/paired_ci_benchmarks.yml b/.github/workflows/paired_ci_benchmarks.yml
index 01628a9dc3..f9fcdea8a7 100644
--- a/.github/workflows/paired_ci_benchmarks.yml
+++ b/.github/workflows/paired_ci_benchmarks.yml
@@ -5,6 +5,10 @@ on:
 
 permissions: {}
 
+concurrency:
+  group: paired-ci-benchmarks-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
 jobs:
   benchmark:
     name: Benchmark ${{ matrix.change }}
@@ -60,7 +64,7 @@ jobs:
             local directory="$2"
             local start end elapsed
             start=$(date +%s)
-            (cd "$directory" && uv run -p .venv pytest -q -n auto --dist worksteal tests/)
+            (cd "$directory" && uv run -p .venv pytest -vv -n auto --dist worksteal tests/)
             end=$(date +%s)
             elapsed=$((end - start))
             echo "CI_BENCHMARK change=${{ matrix.change }} label=$label elapsed_seconds=$elapsed"