ci(experimental): give the GraalPy leg room to finish (#2739)

.github/workflows/experimental.yml

@@ -155,8 +155,8 @@ jobs:
   # tests/project/test_ktlo_legs_stay_green_1672.py pins the caps and the
   # minutes they leave the rest.
   #
-  # The tests' cap is sized for GraalPy (#1732), whose leg may still end on the
-  # 110-minute `timeout` in `Run unit tests`, which `Containment summary`
+  # The tests' cap is sized for GraalPy (#1732, #2739), whose leg may still end on
+  # the 180-minute `timeout` in `Run unit tests`, which `Containment summary`
   # reports as a warning that blocks nothing.
   experimental:
     name: Python ${{ matrix.python-version }} (experimental)
@@ -169,7 +169,7 @@ jobs:
       && needs.changes.outputs.test-only != 'true' }}
     needs: [current, changes, python-versions]
     runs-on: ${{ matrix.os || 'ubuntu-latest' }}
-    timeout-minutes: 140
+    timeout-minutes: 210
     continue-on-error: true
     # #1938: `Run unit tests` reads the pull request, as `current` does.
     permissions:
@@ -334,14 +334,14 @@ jobs:
       # - #1963: util/pytest_shards.py runs pytest once per shard `SHARDS`
       #   names, a directory or a glob of one directory's entries, then once
       #   over the rest, so every run starts its workers afresh and none
-      #   outlives its heap. #2713: the shards that lost workers are split by
-      #   their tests' JUnit seconds on GraalPy, each to run in under 5
-      #   minutes; the run over the rest is left whole, as it took about 6.7
-      #   minutes in job 125639681253 and lost no worker. The script runs
+      #   outlives its heap. #2713 split the shards that lost workers in job
+      #   125639681253 by their tests' JUnit seconds there, and #2739 the two
+      #   that ran 11-14 minutes in job 125752017215 by its; the run over the
+      #   rest is left whole, as it lost no worker in either. The script runs
       #   inside the timeout, scope and watcher below, so they hold over the
       #   whole tier; on the timeout's SIGINT no later run starts. Each shard's
       #   run writes a JUnit file of its own beside junit.xml.
-      # - `timeout` sends SIGINT at 110 minutes, under the step's cap, so pytest
+      # - `timeout` sends SIGINT at 180 minutes, under the step's cap, so pytest
       #   still writes the JUnit XML, and exits 124; SIGKILL 3 minutes on.
       #   #2020: GNU's `timeout`, which Ubuntu 26.04 installs as `gnutimeout`:
       #   its `timeout` is uutils' (0.10.0), whose `--kill-after` sends SIGKILL with the
@@ -387,7 +387,7 @@ jobs:
         id: tests
         if: ${{ steps.install.outcome == 'success' }}
         continue-on-error: true
-        timeout-minutes: 120
+        timeout-minutes: 190
         env:
           GRAAL_PYTHON_VM_ARGS: ${{ startsWith(matrix.python-version, 'graalpy') && '--vm.Xmx2g --vm.XX:+ExitOnOutOfMemoryError' || '' }}
           # #2571: 3.14t, which installs psutil, takes `logical` as unit-tests.yml `test` does.
@@ -397,8 +397,9 @@ jobs:
             ${{ startsWith(matrix.python-version, 'graalpy') && 'tests/const
             tests/corekit/test_fields_n* tests/corekit/test_[a-h]* tests/corekit/test_i* tests/corekit
             tests/foundation/reassembly tests/foundation/test_* tests/foundation
-            tests/protocols/internet/test_ipv4* tests/protocols/internet/test_ipv6* tests/protocols/internet
-            tests/protocols/application tests/protocols/link tests/protocols/misc/test_pcapng* tests/protocols/misc
+            tests/protocols/internet/test_ipv4* tests/protocols/internet/test_ipv6* tests/protocols/internet/test_igmp*
+            tests/protocols/internet tests/protocols/application/test_tls* tests/protocols/application
+            tests/protocols/link tests/protocols/misc/test_pcapng* tests/protocols/misc
             tests/protocols/transport/test_tcp* tests/protocols/transport tests/protocols/test_t* tests/protocols
             tests/toolkit' || '' }}
           GH_TOKEN: ${{ github.token }}
@@ -425,7 +426,7 @@ jobs:
           fi
           # #2020: GNU's `timeout` on any image, `gnutimeout` where it exists.
           gnu_timeout=$(command -v gnutimeout || command -v timeout)
-          set -- prlimit --fsize=2147483648 "$gnu_timeout" --signal=INT --kill-after=3m 110m "$@"
+          set -- prlimit --fsize=2147483648 "$gnu_timeout" --signal=INT --kill-after=3m 180m "$@"
           scoped() {
             unit=$1
             shift
@@ -528,7 +529,7 @@ jobs:
       # summary's (tests/project/test_ktlo_legs_stay_green_1672.py). Each cause
       # gets a warning of its own: no 12G scope, an OOM kill, a MemoryError, a
       # worker xdist replaced (#1753), and a timeout -- exit 124, or 137 with
-      # no OOM kill after 110 minutes, `--kill-after` ending a teardown SIGINT
+      # no OOM kill after 180 minutes, `--kill-after` ending a teardown SIGINT
       # did not; an OOM kill of pytest itself also exits 137. Nothing here
       # fails. #2077: a run the watcher stopped as superseded exits 0, so the
       # watcher's `stopped=true` makes the closing line say the tests were
@@ -566,9 +567,9 @@ jobs:
             warn "tests lost a process to the OOM killer" "${oom} OOM kill(s) while the tests ran, scope ${scope}, exit ${code}, ${elapsed} s; ${last}."
           fi
           if [ "$code" = 124 ]; then
-            warn "tests timed out at 110 minutes" "timeout sent SIGINT (exit 124), scope ${scope}, ${elapsed} s; ${last}."
-          elif [ "$code" = 137 ] && [ "$oom" -eq 0 ] 2>/dev/null && [ "$elapsed" -ge 6600 ] 2>/dev/null; then
-            warn "tests timed out at 110 minutes" "SIGINT did not end them, so timeout sent SIGKILL 3 minutes on (exit 137, no OOM kill), scope ${scope}, ${elapsed} s; ${last}."
+            warn "tests timed out at 180 minutes" "timeout sent SIGINT (exit 124), scope ${scope}, ${elapsed} s; ${last}."
+          elif [ "$code" = 137 ] && [ "$oom" -eq 0 ] 2>/dev/null && [ "$elapsed" -ge 10800 ] 2>/dev/null; then
+            warn "tests timed out at 180 minutes" "SIGINT did not end them, so timeout sent SIGKILL 3 minutes on (exit 137, no OOM kill), scope ${scope}, ${elapsed} s; ${last}."
           fi
           if [ "$errors" -gt 0 ] 2>/dev/null; then
             warn "tests raised MemoryError" "${errors} MemoryError line(s) in the pytest output, exit ${code}, scope ${scope}; ${last}."

docs/source/contributing/workflows.rst

@@ -689,7 +689,7 @@ CI Timing Review (:issue:`1538`)
 or on dispatch with a ``days`` window and an optional ``concurrency_hours`` runner-pool
 measurement. Through the API it lists every completed run in the window of Unit Tests,
 Python Compatibility, Lint, CodeQL, GitHub Pages and Experimental Tests (:issue:`1732`, so
-its two-hour legs stay measured), a day per query because a filtered listing stops
+its long legs stay measured), a day per query because a filtered listing stops
 at 1,000 runs (Pages completed 1,154 in the week to 2026-10-09). It then reads the
 jobs of an evenly spaced sample -- 150 push and 150 pull-request runs of Unit Tests,
 60 runs of each other workflow -- and the summary says how many runs each sample
@@ -1159,8 +1159,8 @@ though the leg's check run still reads ``failure``. A job past its
 ``timeout-minutes`` is cancelled, and that page does not say whether the job-level
 guard covers a cancellation. So setup-python, the install, the watcher's setup-python
 and ``requests`` (:issue:`1938`,
-below) and the tests have caps of their own, of 5, 10, 1, 1 and 120 minutes, under
-the job's 140, which leaves 3 for the rest: a step at its cap
+below) and the tests have caps of their own, of 5, 10, 1, 1 and 190 minutes, under
+the job's 210, which leaves 3 for the rest: a step at its cap
 fails rather than cancels, and the guard holds (actions/runner's
 ``StepsRunner.RunStepAsync``). The runner's ``has timed out`` error
 stays, changing no conclusion, as ``Process completed with exit code N.`` does.
@@ -1170,7 +1170,7 @@ job, and none on a step. Promoting one whose leg stays green is a hand edit, whi
 :file:`util/python_versions.py` refuses until the build has a wheel, a conda build
 and a ``Compat`` leg.
 
-The tests' 120 minutes are sized for GraalPy, the slowest of the legs that share
+The tests' 190 minutes are sized for GraalPy, the slowest of the legs that share
 them: its leg reached 26% of the unit tier in 25 minutes (job 115915976024), where
 3.14t's ran the whole tier in 5.8 (run 38581834697). The experimental legs run the
 ``ALL`` test tier alone, as every leg but ``latest`` does (see `Test Tiers`_ below):
@@ -1180,7 +1180,13 @@ passes each module it printed as an ``--ignore``. The
 round-trip sweeps of :issue:`1732`
 took 8% of the unit tier's time in a serial 3.14 run (86 of 1,031 s), and 10-11% on
 the 3.14, 3.15 and 3.14t legs of run 38581834697, so the rest should take GraalPy
-88-113 minutes (estimate).
+88-113 minutes (estimate). That was low. In job 125752017215, on ``ab7be2fb0``, the
+shards below took 6386 s, and the ``timeout``, of 110 minutes at that commit, ended the
+run over the rest, which had taken 404 s in job 125639681253, on a runner 1.46-1.86
+times as fast (see below). With that run at 589-750 s on the slower runner, and
+70-100 s of start-up for each of the two runs the later split adds, the leg comes to
+7110-7340 s there (estimate). So the ``timeout`` is 180 minutes, about 1.5 times that,
+and the step's cap 10 minutes more (:issue:`2739`).
 
 GraalPy's leg still loses its runner 18-30 minutes into its tests, with ``The runner
 has received a shutdown signal`` (:issue:`1740`),
@@ -1242,14 +1248,21 @@ warnings:
   sized from that job's ``junit-experimental-graalpy-25.4`` artifact: a shard's
   estimate is its tests' JUnit seconds over the 3 workers, plus 50 s for the 46-57 s
   that each of the job's shards that lost no worker took beyond that, start to start.
-  Every shard split this way comes to under 5 minutes, and the tier to about 4750 s
-  (estimate). The run over the rest is left whole: in that job it took about 6.7
-  minutes and lost no worker. The runs share the step's ``timeout``, scope and
+  That put every shard split this way under 5 minutes, and the tier at about 4750 s,
+  on that job's runner (estimate). The run over the rest is left whole: in that job it
+  took about 6.7 minutes and lost no worker. In job 125752017215, on ``ab7be2fb0``, no
+  worker went down, but on a runner whose unchanged shards each took 1.46-1.86 times
+  as long, ``tests/protocols/internet`` and ``tests/protocols/application`` ran 11.5
+  and 13.9 minutes, their largest worker reaching 2.25 and 2.48 GiB RSS, and the
+  other shards of that split 2.3-8.1 (:issue:`2739`). So those two are split again by
+  that job's JUnit seconds, at ``test_igmp*`` and ``test_tls*``, which leaves every
+  other shard fewer JUnit seconds there than ``tests/const`` took, whose largest
+  worker peaked at 2.05 GiB. The runs share the step's ``timeout``, scope and
   watcher, so the caps hold over the whole tier; on ``timeout``'s SIGINT the run in
   progress writes its JUnit file and no later one starts. Each shard's run writes a
   JUnit file of its own beside ``junit.xml``, such as ``junit-tests-protocols.xml``,
   and ``Upload test timings`` takes them all. 3.14t's leg runs pytest once.
-* ``timeout --signal=INT --kill-after=3m 110m`` ends pytest under the step's 120-minute
+* ``timeout --signal=INT --kill-after=3m 180m`` ends pytest under the step's 190-minute
   cap, so it still writes the junit XML, and exits 124; ``prlimit --fsize=2147483648``
   caps any file at 2 GiB. The ``timeout`` is GNU's, ``gnutimeout`` where it exists
   (:issue:`2020`): on Ubuntu 26.04 ``timeout`` is uutils', whose ``--kill-after`` sends SIGKILL
@@ -1269,20 +1282,21 @@ probe's error, the seconds the tests took, the OOM kills while they ran
 (``/proc/vmstat``), the lines in their output that report a raised ``MemoryError`` (#1962),
 not those that mention one, such as a docstring a traceback prints, and xdist's ``node
 down`` lines. From it and the monitor's last sample, ``Containment summary`` warns
-``No memory cap``, ``tests lost a process to the OOM killer``, ``tests timed out at 110
+``No memory cap``, ``tests lost a process to the OOM killer``, ``tests timed out at 180
 minutes``, ``tests raised MemoryError`` or ``tests lost an xdist worker``, each a
 warning of its own. Its closing line gives the exit status, but where the watcher
 stopped the run and wrote ``stopped=true`` it reads ``tests were stopped as
 superseded``, not ``tests exited 0``, and adds no warning, since the ``Superseded``
 notice already says why (`#2077
 <https://github.com/JarryShaw/PyPCAPKit/issues/2077>`__). A timeout is exit 124, or 137 with no OOM kill
-after 110 minutes: ``--kill-after``'s SIGKILL to a teardown SIGINT did not end, where an
+after 180 minutes: ``--kill-after``'s SIGKILL to a teardown SIGINT did not end, where an
 OOM kill of pytest exits 137 too. ``Experimental summary``, the job's last step but the job summary's and the
 KTLO summary's script, still warns of every failed step, ``tests`` included. The 2 GiB,
 ``-n 3`` and 12G are estimates for a 16 GB, 4-CPU runner until a run shows them. At 3
-workers rather than 4, GraalPy's tests should take about 117-151 minutes (an inference
-from the 88-113 above, not measured), so its leg may end on the 110-minute timeout as a
-warning: that is expected, blocks nothing, and leaves the caps as they are.
+workers rather than 4, GraalPy's tests were to take about 117-151 minutes (an inference
+from the 88-113 above, not measured), and job 125752017215's came to about 119-122 in
+the shards above (estimate). A leg that still ends on the 180-minute timeout does so as
+a warning that blocks nothing.
 
 **The Windows leg.** :file:`experimental.yml`'s ``windows`` job runs the ``ALL`` tier
 on ``windows-latest`` under ``latest``, the newest CPython the conda builds target, as

tests/project/test_ci_timing.py

@@ -275,7 +275,7 @@ def test_the_real_unit_tests_workflow(self) -> 'None':
             with self.subTest(job=name):
                 experimental = ci_timing.match_job(own, name)
                 self.assertEqual((experimental['id'], experimental['timeout'],
-                                  ci_timing.step_cap(experimental, ci_timing.TESTS_STEP)), ('experimental', 140, 120))
+                                  ci_timing.step_cap(experimental, ci_timing.TESTS_STEP)), ('experimental', 210, 190))
 
     def test_every_step_of_interest_names_a_real_step(self) -> 'None':
         """#2230: each alternative of :data:`STEPS_OF_INTEREST` matches a step some workflow has.
@@ -1037,7 +1037,7 @@ class TestNonBlockingLegs(unittest.TestCase):
     """#1732: a cap finding on a leg nothing waits on is a warning, not HARD."""
 
     #: (workflow, job name, its cap in the real workflow file).
-    EXPERIMENTAL = ('experimental.yml', 'Python graalpy-25.4 (experimental)', 140)
+    EXPERIMENTAL = ('experimental.yml', 'Python graalpy-25.4 (experimental)', 210)
     KTLO = ('unit-tests.yml', 'Python 3.10 (KTLO)', 45)
     BLOCKING = ('unit-tests.yml', 'Python 3.14', 45)
 

tests/project/test_gnu_timeout_2020.py

@@ -41,7 +41,7 @@
 
 #: How the tests step resolves GNU's ``timeout``, and how it runs it.
 RESOLVE = 'gnu_timeout=$(command -v gnutimeout || command -v timeout)\n'
-RUN = '"$gnu_timeout" --signal=INT --kill-after=3m 110m "$@"\n'
+RUN = '"$gnu_timeout" --signal=INT --kill-after=3m 180m "$@"\n'
 #: ``timeout`` run as a command and given an option, which only GNU's honours as GNU does.
 OPTIONED = re.compile(r'(?<![\w$"/.-])timeout\s+-')
 #: A shell comment, a whole line of one or one after a space.
@@ -197,7 +197,7 @@ def test_the_step_runs_gnutimeout_where_it_exists(self) -> None:
         self.assertEqual(result.returncode, 0, result.stderr)
         calls = self.calls.read_text(encoding='utf-8').splitlines()
         self.assertEqual(len(calls), 1, calls)
-        self.assertTrue(calls[0].startswith('gnutimeout --signal=INT --kill-after=3m 110m python -m pytest '), calls)
+        self.assertTrue(calls[0].startswith('gnutimeout --signal=INT --kill-after=3m 180m python -m pytest '), calls)
 
 
 if __name__ == '__main__':

tests/project/test_graalpy_shards_1963.py

@@ -35,18 +35,20 @@
 from util import pytest_shards
 
 #: GraalPy's shards, in the order they run; the rest of the tier runs last. #2713 split the
-#: shards that lost workers in job 125639681253 by their JUnit seconds there.
+#: shards that lost workers in job 125639681253 by their JUnit seconds there, and #2739
+#: split the two that ran 11-14 minutes in job 125752017215 by its.
 SHARDS = ('tests/const tests/corekit/test_fields_n* tests/corekit/test_[a-h]* tests/corekit/test_i* tests/corekit '
           'tests/foundation/reassembly tests/foundation/test_* tests/foundation '
-          'tests/protocols/internet/test_ipv4* tests/protocols/internet/test_ipv6* tests/protocols/internet '
-          'tests/protocols/application tests/protocols/link tests/protocols/misc/test_pcapng* tests/protocols/misc '
+          'tests/protocols/internet/test_ipv4* tests/protocols/internet/test_ipv6* tests/protocols/internet/test_igmp* '
+          'tests/protocols/internet tests/protocols/application/test_tls* tests/protocols/application '
+          'tests/protocols/link tests/protocols/misc/test_pcapng* tests/protocols/misc '
           'tests/protocols/transport/test_tcp* tests/protocols/transport tests/protocols/test_t* tests/protocols '
           'tests/toolkit')
 SCRIPT = ROOT / 'util' / 'pytest_shards.py'
 #: The lines that build the command the step's caps run, in order.
 PYTEST = 'set -- python -m pytest "$@"\n'
 SHARDED = 'set -- /usr/bin/python3 util/pytest_shards.py "$SHARDS" -- "$@"\n'
-CAPPED = 'set -- prlimit --fsize=2147483648 "$gnu_timeout" --signal=INT --kill-after=3m 110m "$@"\n'
+CAPPED = 'set -- prlimit --fsize=2147483648 "$gnu_timeout" --signal=INT --kill-after=3m 180m "$@"\n'
 
 
 def rest_ignores(shards: 'str') -> 'list[str]':
@@ -184,7 +186,7 @@ def test_without_shards_pytest_runs_once(self) -> None:
 
     def test_the_timeout_ends_the_run_in_progress_and_starts_no_other(self) -> None:
         # pytest ends on SIGINT, which timeout sends to its whole process group, with exit 2.
-        script = step(TESTS)['run'].replace('--kill-after=3m 110m', '--kill-after=5s 1s')
+        script = step(TESTS)['run'].replace('--kill-after=3m 180m', '--kill-after=5s 1s')
         self.assertNotEqual(script, step(TESTS)['run'])
         result = self.run_step("trap 'echo interrupted; exit 2' INT\nsleep 30 &\nwait $!\nexit 0\n", script)
         self.assertEqual(result.returncode, 124, result.stderr)

tests/project/test_ktlo_legs_stay_green_1672.py

@@ -173,10 +173,12 @@
 #: ``experimental``'s steps with a ``timeout-minutes`` of their own (#1660), by ``id``:
 #: setup-python, the install, the watcher's setup-python and requests (#1938) and the
 #: tests. #1732: the tests' is GraalPy's unit tier less the round-trip sweeps, 88-113
-#: minutes (estimate) from the 26% its leg reached in 25 (job 115915976024).
-EXPERIMENTAL_CAPS = {'python': 5, 'install': 10, 'watcher-python': 1, 'watcher': 1, 'tests': 120}
+#: minutes (estimate) from the 26% its leg reached in 25 (job 115915976024). #2739: its
+#: 180-minute timeout is about 1.5 times the leg in job 125752017215, whose last run is
+#: estimated.
+EXPERIMENTAL_CAPS = {'python': 5, 'install': 10, 'watcher-python': 1, 'watcher': 1, 'tests': 190}
 #: ``experimental``'s own ``timeout-minutes`` (#1732), and ``test``'s.
-EXPERIMENTAL_JOB_CAP = 140
+EXPERIMENTAL_JOB_CAP = 210
 JOB_CAP = 45
 #: The job's minutes those caps leave for the rest of an experimental leg: the steps
 #: without a cap of their own -- checkout, the parallelism report, the resource