From 90fd72db55635c6b1e663bb1ab94b58d873f412f Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 3 Nov 2026 03:16:33 -0500 Subject: [PATCH] ci(experimental): give the GraalPy leg room to finish (#2739) On ab7be2fb0, job 125752017215 lost no GraalPy worker but hit the 110-minute timeout during the rest: its shards took 6386 s, on a runner 1.46-1.86 times slower than #2713's estimates assumed, and tests/protocols/internet and tests/protocols/application ran 11.5 and 13.9 minutes, their largest worker at 2.25 and 2.48 GiB RSS. - Run unit tests: timeout 110m -> 180m, step cap 120 -> 190, job cap 140 -> 210, about 1.5 times the leg's 7110-7340 s there (estimate); Containment summary's timeout warning and 137 threshold follow. - SHARDS: split those two at test_igmp* and test_tls* by that job's JUnit seconds, so no shard but tests/const takes as many. - Pins of the caps and the timeout in tests/project follow; the #1963 and timeout notes in workflows.rst gain the measurements. Collecting the tier through util/pytest_shards.py on CPython gives the same 3058 node ids, none twice, for the old and new SHARDS. tests/project passes, each module in its own process; the moved pins fail on the base workflow. --- .github/workflows/experimental.yml | 33 ++++++++------- docs/source/contributing/workflows.rst | 42 ++++++++++++------- tests/project/test_ci_timing.py | 4 +- tests/project/test_gnu_timeout_2020.py | 4 +- tests/project/test_graalpy_shards_1963.py | 12 +++--- .../project/test_ktlo_legs_stay_green_1672.py | 8 ++-- tests/project/test_resource_monitor_1740.py | 38 ++++++++--------- tests/project/test_superseded_runs_1938.py | 2 +- .../test_sweeps_on_latest_only_1732.py | 9 ++-- 9 files changed, 86 insertions(+), 66 deletions(-) diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index e45b543253..2bc146512f 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -155,8 +155,8 @@ jobs: # tests/project/test_ktlo_legs_stay_green_1672.py pins the caps and the # minutes they leave the rest. # - # The tests' cap is sized for GraalPy (#1732), whose leg may still end on the - # 110-minute `timeout` in `Run unit tests`, which `Containment summary` + # The tests' cap is sized for GraalPy (#1732, #2739), whose leg may still end on + # the 180-minute `timeout` in `Run unit tests`, which `Containment summary` # reports as a warning that blocks nothing. experimental: name: Python ${{ matrix.python-version }} (experimental) @@ -169,7 +169,7 @@ jobs: && needs.changes.outputs.test-only != 'true' }} needs: [current, changes, python-versions] runs-on: ${{ matrix.os || 'ubuntu-latest' }} - timeout-minutes: 140 + timeout-minutes: 210 continue-on-error: true # #1938: `Run unit tests` reads the pull request, as `current` does. permissions: @@ -334,14 +334,14 @@ jobs: # - #1963: util/pytest_shards.py runs pytest once per shard `SHARDS` # names, a directory or a glob of one directory's entries, then once # over the rest, so every run starts its workers afresh and none - # outlives its heap. #2713: the shards that lost workers are split by - # their tests' JUnit seconds on GraalPy, each to run in under 5 - # minutes; the run over the rest is left whole, as it took about 6.7 - # minutes in job 125639681253 and lost no worker. The script runs + # outlives its heap. #2713 split the shards that lost workers in job + # 125639681253 by their tests' JUnit seconds there, and #2739 the two + # that ran 11-14 minutes in job 125752017215 by its; the run over the + # rest is left whole, as it lost no worker in either. The script runs # inside the timeout, scope and watcher below, so they hold over the # whole tier; on the timeout's SIGINT no later run starts. Each shard's # run writes a JUnit file of its own beside junit.xml. - # - `timeout` sends SIGINT at 110 minutes, under the step's cap, so pytest + # - `timeout` sends SIGINT at 180 minutes, under the step's cap, so pytest # still writes the JUnit XML, and exits 124; SIGKILL 3 minutes on. # #2020: GNU's `timeout`, which Ubuntu 26.04 installs as `gnutimeout`: # its `timeout` is uutils' (0.10.0), whose `--kill-after` sends SIGKILL with the @@ -387,7 +387,7 @@ jobs: id: tests if: ${{ steps.install.outcome == 'success' }} continue-on-error: true - timeout-minutes: 120 + timeout-minutes: 190 env: GRAAL_PYTHON_VM_ARGS: ${{ startsWith(matrix.python-version, 'graalpy') && '--vm.Xmx2g --vm.XX:+ExitOnOutOfMemoryError' || '' }} # #2571: 3.14t, which installs psutil, takes `logical` as unit-tests.yml `test` does. @@ -397,8 +397,9 @@ jobs: ${{ startsWith(matrix.python-version, 'graalpy') && 'tests/const tests/corekit/test_fields_n* tests/corekit/test_[a-h]* tests/corekit/test_i* tests/corekit tests/foundation/reassembly tests/foundation/test_* tests/foundation - tests/protocols/internet/test_ipv4* tests/protocols/internet/test_ipv6* tests/protocols/internet - tests/protocols/application tests/protocols/link tests/protocols/misc/test_pcapng* tests/protocols/misc + tests/protocols/internet/test_ipv4* tests/protocols/internet/test_ipv6* tests/protocols/internet/test_igmp* + tests/protocols/internet tests/protocols/application/test_tls* tests/protocols/application + tests/protocols/link tests/protocols/misc/test_pcapng* tests/protocols/misc tests/protocols/transport/test_tcp* tests/protocols/transport tests/protocols/test_t* tests/protocols tests/toolkit' || '' }} GH_TOKEN: ${{ github.token }} @@ -425,7 +426,7 @@ jobs: fi # #2020: GNU's `timeout` on any image, `gnutimeout` where it exists. gnu_timeout=$(command -v gnutimeout || command -v timeout) - set -- prlimit --fsize=2147483648 "$gnu_timeout" --signal=INT --kill-after=3m 110m "$@" + set -- prlimit --fsize=2147483648 "$gnu_timeout" --signal=INT --kill-after=3m 180m "$@" scoped() { unit=$1 shift @@ -528,7 +529,7 @@ jobs: # summary's (tests/project/test_ktlo_legs_stay_green_1672.py). Each cause # gets a warning of its own: no 12G scope, an OOM kill, a MemoryError, a # worker xdist replaced (#1753), and a timeout -- exit 124, or 137 with - # no OOM kill after 110 minutes, `--kill-after` ending a teardown SIGINT + # no OOM kill after 180 minutes, `--kill-after` ending a teardown SIGINT # did not; an OOM kill of pytest itself also exits 137. Nothing here # fails. #2077: a run the watcher stopped as superseded exits 0, so the # watcher's `stopped=true` makes the closing line say the tests were @@ -566,9 +567,9 @@ jobs: warn "tests lost a process to the OOM killer" "${oom} OOM kill(s) while the tests ran, scope ${scope}, exit ${code}, ${elapsed} s; ${last}." fi if [ "$code" = 124 ]; then - warn "tests timed out at 110 minutes" "timeout sent SIGINT (exit 124), scope ${scope}, ${elapsed} s; ${last}." - elif [ "$code" = 137 ] && [ "$oom" -eq 0 ] 2>/dev/null && [ "$elapsed" -ge 6600 ] 2>/dev/null; then - warn "tests timed out at 110 minutes" "SIGINT did not end them, so timeout sent SIGKILL 3 minutes on (exit 137, no OOM kill), scope ${scope}, ${elapsed} s; ${last}." + warn "tests timed out at 180 minutes" "timeout sent SIGINT (exit 124), scope ${scope}, ${elapsed} s; ${last}." + elif [ "$code" = 137 ] && [ "$oom" -eq 0 ] 2>/dev/null && [ "$elapsed" -ge 10800 ] 2>/dev/null; then + warn "tests timed out at 180 minutes" "SIGINT did not end them, so timeout sent SIGKILL 3 minutes on (exit 137, no OOM kill), scope ${scope}, ${elapsed} s; ${last}." fi if [ "$errors" -gt 0 ] 2>/dev/null; then warn "tests raised MemoryError" "${errors} MemoryError line(s) in the pytest output, exit ${code}, scope ${scope}; ${last}." diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 4fb86f0816..a6ebc628d2 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -689,7 +689,7 @@ CI Timing Review (:issue:`1538`) or on dispatch with a ``days`` window and an optional ``concurrency_hours`` runner-pool measurement. Through the API it lists every completed run in the window of Unit Tests, Python Compatibility, Lint, CodeQL, GitHub Pages and Experimental Tests (:issue:`1732`, so -its two-hour legs stay measured), a day per query because a filtered listing stops +its long legs stay measured), a day per query because a filtered listing stops at 1,000 runs (Pages completed 1,154 in the week to 2026-10-09). It then reads the jobs of an evenly spaced sample -- 150 push and 150 pull-request runs of Unit Tests, 60 runs of each other workflow -- and the summary says how many runs each sample @@ -1159,8 +1159,8 @@ though the leg's check run still reads ``failure``. A job past its ``timeout-minutes`` is cancelled, and that page does not say whether the job-level guard covers a cancellation. So setup-python, the install, the watcher's setup-python and ``requests`` (:issue:`1938`, -below) and the tests have caps of their own, of 5, 10, 1, 1 and 120 minutes, under -the job's 140, which leaves 3 for the rest: a step at its cap +below) and the tests have caps of their own, of 5, 10, 1, 1 and 190 minutes, under +the job's 210, which leaves 3 for the rest: a step at its cap fails rather than cancels, and the guard holds (actions/runner's ``StepsRunner.RunStepAsync``). The runner's ``has timed out`` error stays, changing no conclusion, as ``Process completed with exit code N.`` does. @@ -1170,7 +1170,7 @@ job, and none on a step. Promoting one whose leg stays green is a hand edit, whi :file:`util/python_versions.py` refuses until the build has a wheel, a conda build and a ``Compat`` leg. -The tests' 120 minutes are sized for GraalPy, the slowest of the legs that share +The tests' 190 minutes are sized for GraalPy, the slowest of the legs that share them: its leg reached 26% of the unit tier in 25 minutes (job 115915976024), where 3.14t's ran the whole tier in 5.8 (run 38581834697). The experimental legs run the ``ALL`` test tier alone, as every leg but ``latest`` does (see `Test Tiers`_ below): @@ -1180,7 +1180,13 @@ passes each module it printed as an ``--ignore``. The round-trip sweeps of :issue:`1732` took 8% of the unit tier's time in a serial 3.14 run (86 of 1,031 s), and 10-11% on the 3.14, 3.15 and 3.14t legs of run 38581834697, so the rest should take GraalPy -88-113 minutes (estimate). +88-113 minutes (estimate). That was low. In job 125752017215, on ``ab7be2fb0``, the +shards below took 6386 s, and the ``timeout``, of 110 minutes at that commit, ended the +run over the rest, which had taken 404 s in job 125639681253, on a runner 1.46-1.86 +times as fast (see below). With that run at 589-750 s on the slower runner, and +70-100 s of start-up for each of the two runs the later split adds, the leg comes to +7110-7340 s there (estimate). So the ``timeout`` is 180 minutes, about 1.5 times that, +and the step's cap 10 minutes more (:issue:`2739`). GraalPy's leg still loses its runner 18-30 minutes into its tests, with ``The runner has received a shutdown signal`` (:issue:`1740`), @@ -1242,14 +1248,21 @@ warnings: sized from that job's ``junit-experimental-graalpy-25.4`` artifact: a shard's estimate is its tests' JUnit seconds over the 3 workers, plus 50 s for the 46-57 s that each of the job's shards that lost no worker took beyond that, start to start. - Every shard split this way comes to under 5 minutes, and the tier to about 4750 s - (estimate). The run over the rest is left whole: in that job it took about 6.7 - minutes and lost no worker. The runs share the step's ``timeout``, scope and + That put every shard split this way under 5 minutes, and the tier at about 4750 s, + on that job's runner (estimate). The run over the rest is left whole: in that job it + took about 6.7 minutes and lost no worker. In job 125752017215, on ``ab7be2fb0``, no + worker went down, but on a runner whose unchanged shards each took 1.46-1.86 times + as long, ``tests/protocols/internet`` and ``tests/protocols/application`` ran 11.5 + and 13.9 minutes, their largest worker reaching 2.25 and 2.48 GiB RSS, and the + other shards of that split 2.3-8.1 (:issue:`2739`). So those two are split again by + that job's JUnit seconds, at ``test_igmp*`` and ``test_tls*``, which leaves every + other shard fewer JUnit seconds there than ``tests/const`` took, whose largest + worker peaked at 2.05 GiB. The runs share the step's ``timeout``, scope and watcher, so the caps hold over the whole tier; on ``timeout``'s SIGINT the run in progress writes its JUnit file and no later one starts. Each shard's run writes a JUnit file of its own beside ``junit.xml``, such as ``junit-tests-protocols.xml``, and ``Upload test timings`` takes them all. 3.14t's leg runs pytest once. -* ``timeout --signal=INT --kill-after=3m 110m`` ends pytest under the step's 120-minute +* ``timeout --signal=INT --kill-after=3m 180m`` ends pytest under the step's 190-minute cap, so it still writes the junit XML, and exits 124; ``prlimit --fsize=2147483648`` caps any file at 2 GiB. The ``timeout`` is GNU's, ``gnutimeout`` where it exists (:issue:`2020`): on Ubuntu 26.04 ``timeout`` is uutils', whose ``--kill-after`` sends SIGKILL @@ -1269,20 +1282,21 @@ probe's error, the seconds the tests took, the OOM kills while they ran (``/proc/vmstat``), the lines in their output that report a raised ``MemoryError`` (#1962), not those that mention one, such as a docstring a traceback prints, and xdist's ``node down`` lines. From it and the monitor's last sample, ``Containment summary`` warns -``No memory cap``, ``tests lost a process to the OOM killer``, ``tests timed out at 110 +``No memory cap``, ``tests lost a process to the OOM killer``, ``tests timed out at 180 minutes``, ``tests raised MemoryError`` or ``tests lost an xdist worker``, each a warning of its own. Its closing line gives the exit status, but where the watcher stopped the run and wrote ``stopped=true`` it reads ``tests were stopped as superseded``, not ``tests exited 0``, and adds no warning, since the ``Superseded`` notice already says why (`#2077 `__). A timeout is exit 124, or 137 with no OOM kill -after 110 minutes: ``--kill-after``'s SIGKILL to a teardown SIGINT did not end, where an +after 180 minutes: ``--kill-after``'s SIGKILL to a teardown SIGINT did not end, where an OOM kill of pytest exits 137 too. ``Experimental summary``, the job's last step but the job summary's and the KTLO summary's script, still warns of every failed step, ``tests`` included. The 2 GiB, ``-n 3`` and 12G are estimates for a 16 GB, 4-CPU runner until a run shows them. At 3 -workers rather than 4, GraalPy's tests should take about 117-151 minutes (an inference -from the 88-113 above, not measured), so its leg may end on the 110-minute timeout as a -warning: that is expected, blocks nothing, and leaves the caps as they are. +workers rather than 4, GraalPy's tests were to take about 117-151 minutes (an inference +from the 88-113 above, not measured), and job 125752017215's came to about 119-122 in +the shards above (estimate). A leg that still ends on the 180-minute timeout does so as +a warning that blocks nothing. **The Windows leg.** :file:`experimental.yml`'s ``windows`` job runs the ``ALL`` tier on ``windows-latest`` under ``latest``, the newest CPython the conda builds target, as diff --git a/tests/project/test_ci_timing.py b/tests/project/test_ci_timing.py index a8d514d775..fee9104c78 100644 --- a/tests/project/test_ci_timing.py +++ b/tests/project/test_ci_timing.py @@ -275,7 +275,7 @@ def test_the_real_unit_tests_workflow(self) -> 'None': with self.subTest(job=name): experimental = ci_timing.match_job(own, name) self.assertEqual((experimental['id'], experimental['timeout'], - ci_timing.step_cap(experimental, ci_timing.TESTS_STEP)), ('experimental', 140, 120)) + ci_timing.step_cap(experimental, ci_timing.TESTS_STEP)), ('experimental', 210, 190)) def test_every_step_of_interest_names_a_real_step(self) -> 'None': """#2230: each alternative of :data:`STEPS_OF_INTEREST` matches a step some workflow has. @@ -1037,7 +1037,7 @@ class TestNonBlockingLegs(unittest.TestCase): """#1732: a cap finding on a leg nothing waits on is a warning, not HARD.""" #: (workflow, job name, its cap in the real workflow file). - EXPERIMENTAL = ('experimental.yml', 'Python graalpy-25.4 (experimental)', 140) + EXPERIMENTAL = ('experimental.yml', 'Python graalpy-25.4 (experimental)', 210) KTLO = ('unit-tests.yml', 'Python 3.10 (KTLO)', 45) BLOCKING = ('unit-tests.yml', 'Python 3.14', 45) diff --git a/tests/project/test_gnu_timeout_2020.py b/tests/project/test_gnu_timeout_2020.py index 151894bd1d..205beaa48f 100644 --- a/tests/project/test_gnu_timeout_2020.py +++ b/tests/project/test_gnu_timeout_2020.py @@ -41,7 +41,7 @@ #: How the tests step resolves GNU's ``timeout``, and how it runs it. RESOLVE = 'gnu_timeout=$(command -v gnutimeout || command -v timeout)\n' -RUN = '"$gnu_timeout" --signal=INT --kill-after=3m 110m "$@"\n' +RUN = '"$gnu_timeout" --signal=INT --kill-after=3m 180m "$@"\n' #: ``timeout`` run as a command and given an option, which only GNU's honours as GNU does. OPTIONED = re.compile(r'(? None: self.assertEqual(result.returncode, 0, result.stderr) calls = self.calls.read_text(encoding='utf-8').splitlines() self.assertEqual(len(calls), 1, calls) - self.assertTrue(calls[0].startswith('gnutimeout --signal=INT --kill-after=3m 110m python -m pytest '), calls) + self.assertTrue(calls[0].startswith('gnutimeout --signal=INT --kill-after=3m 180m python -m pytest '), calls) if __name__ == '__main__': diff --git a/tests/project/test_graalpy_shards_1963.py b/tests/project/test_graalpy_shards_1963.py index 4b729089c0..6bbc0179e6 100644 --- a/tests/project/test_graalpy_shards_1963.py +++ b/tests/project/test_graalpy_shards_1963.py @@ -35,18 +35,20 @@ from util import pytest_shards #: GraalPy's shards, in the order they run; the rest of the tier runs last. #2713 split the -#: shards that lost workers in job 125639681253 by their JUnit seconds there. +#: shards that lost workers in job 125639681253 by their JUnit seconds there, and #2739 +#: split the two that ran 11-14 minutes in job 125752017215 by its. SHARDS = ('tests/const tests/corekit/test_fields_n* tests/corekit/test_[a-h]* tests/corekit/test_i* tests/corekit ' 'tests/foundation/reassembly tests/foundation/test_* tests/foundation ' - 'tests/protocols/internet/test_ipv4* tests/protocols/internet/test_ipv6* tests/protocols/internet ' - 'tests/protocols/application tests/protocols/link tests/protocols/misc/test_pcapng* tests/protocols/misc ' + 'tests/protocols/internet/test_ipv4* tests/protocols/internet/test_ipv6* tests/protocols/internet/test_igmp* ' + 'tests/protocols/internet tests/protocols/application/test_tls* tests/protocols/application ' + 'tests/protocols/link tests/protocols/misc/test_pcapng* tests/protocols/misc ' 'tests/protocols/transport/test_tcp* tests/protocols/transport tests/protocols/test_t* tests/protocols ' 'tests/toolkit') SCRIPT = ROOT / 'util' / 'pytest_shards.py' #: The lines that build the command the step's caps run, in order. PYTEST = 'set -- python -m pytest "$@"\n' SHARDED = 'set -- /usr/bin/python3 util/pytest_shards.py "$SHARDS" -- "$@"\n' -CAPPED = 'set -- prlimit --fsize=2147483648 "$gnu_timeout" --signal=INT --kill-after=3m 110m "$@"\n' +CAPPED = 'set -- prlimit --fsize=2147483648 "$gnu_timeout" --signal=INT --kill-after=3m 180m "$@"\n' def rest_ignores(shards: 'str') -> 'list[str]': @@ -184,7 +186,7 @@ def test_without_shards_pytest_runs_once(self) -> None: def test_the_timeout_ends_the_run_in_progress_and_starts_no_other(self) -> None: # pytest ends on SIGINT, which timeout sends to its whole process group, with exit 2. - script = step(TESTS)['run'].replace('--kill-after=3m 110m', '--kill-after=5s 1s') + script = step(TESTS)['run'].replace('--kill-after=3m 180m', '--kill-after=5s 1s') self.assertNotEqual(script, step(TESTS)['run']) result = self.run_step("trap 'echo interrupted; exit 2' INT\nsleep 30 &\nwait $!\nexit 0\n", script) self.assertEqual(result.returncode, 124, result.stderr) diff --git a/tests/project/test_ktlo_legs_stay_green_1672.py b/tests/project/test_ktlo_legs_stay_green_1672.py index 3085fe46bd..3e58ad0c46 100644 --- a/tests/project/test_ktlo_legs_stay_green_1672.py +++ b/tests/project/test_ktlo_legs_stay_green_1672.py @@ -173,10 +173,12 @@ #: ``experimental``'s steps with a ``timeout-minutes`` of their own (#1660), by ``id``: #: setup-python, the install, the watcher's setup-python and requests (#1938) and the #: tests. #1732: the tests' is GraalPy's unit tier less the round-trip sweeps, 88-113 -#: minutes (estimate) from the 26% its leg reached in 25 (job 115915976024). -EXPERIMENTAL_CAPS = {'python': 5, 'install': 10, 'watcher-python': 1, 'watcher': 1, 'tests': 120} +#: minutes (estimate) from the 26% its leg reached in 25 (job 115915976024). #2739: its +#: 180-minute timeout is about 1.5 times the leg in job 125752017215, whose last run is +#: estimated. +EXPERIMENTAL_CAPS = {'python': 5, 'install': 10, 'watcher-python': 1, 'watcher': 1, 'tests': 190} #: ``experimental``'s own ``timeout-minutes`` (#1732), and ``test``'s. -EXPERIMENTAL_JOB_CAP = 140 +EXPERIMENTAL_JOB_CAP = 210 JOB_CAP = 45 #: The job's minutes those caps leave for the rest of an experimental leg: the steps #: without a cap of their own -- checkout, the parallelism report, the resource diff --git a/tests/project/test_resource_monitor_1740.py b/tests/project/test_resource_monitor_1740.py index 4127fc4664..8d97f74cd7 100644 --- a/tests/project/test_resource_monitor_1740.py +++ b/tests/project/test_resource_monitor_1740.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- """#1740: the experimental legs sample their runner while the tests run. -GraalPy's leg loses its runner 18-30 minutes into ``Run unit tests``, under a -120-minute cap, so experimental.yml records what the runner has left: a sampler +GraalPy's leg loses its runner 18-30 minutes into ``Run unit tests``, under its +step cap, so experimental.yml records what the runner has left: a sampler started before the tests, stopped ``if: always()`` after them, whose samples ``Run unit tests`` copies into its own log, each line led by ``[monitor]``, and a file uploaded ``if: always()`` with ``continue-on-error``. A lost runner runs no @@ -53,7 +53,7 @@ 'ps -eo pid,rss,vsz,etime,args --sort=-rss | head -6') #: The experimental job's caps and its check name, which the monitor leaves alone. #: The watcher's interpreter and its requests (#1938) are capped as the rest are. -CAPS = {'python': 5, 'install': 10, 'watcher-python': 1, 'watcher': 1, 'tests': 120} +CAPS = {'python': 5, 'install': 10, 'watcher-python': 1, 'watcher': 1, 'tests': 190} CHECK_NAME = 'Python ${{ matrix.python-version }} (experimental)' #: A sample's first line, ``[monitor] sample ``. HEADER = re.compile(r'^\[monitor\] \d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ sample (\d+)$') @@ -299,7 +299,7 @@ def test_the_timeout_ends_the_tests_before_the_step_cap(self) -> None: self.assertIsNotNone(match, script) assert match is not None kill_after, duration = map(int, match.groups()) - self.assertEqual((duration, kill_after), (110, 3)) + self.assertEqual((duration, kill_after), (180, 3)) self.assertLess(duration + kill_after, step(TESTS)['timeout-minutes']) def test_no_file_grows_past_2_gib(self) -> None: @@ -408,7 +408,7 @@ def test_the_scope_runs_the_tests_under_the_timeout_and_fsize_cap(self) -> None: self.assertTrue(probe.endswith(f' GRAAL_PYTHON_VM_ARGS={HEAP} true'), probe) self.assertTrue(run.startswith(SUDO_ARGV.format(unit='pcapkit-tests.scope', **ids)), run) # #2020: GNU's timeout, `gnutimeout` where the image has one, by its path. - self.assertIn(f' prlimit --fsize={FSIZE} {gnu_timeout(self.env)} --signal=INT --kill-after=3m 110m ' + self.assertIn(f' prlimit --fsize={FSIZE} {gnu_timeout(self.env)} --signal=INT --kill-after=3m 180m ' 'python -m pytest -q -n 3 ', run) self.assertIn('args: -m pytest -q -n 3 --dist load', result.stdout) self.assertIn(f'heap {HEAP}\n', result.stdout) @@ -463,7 +463,7 @@ def test_the_uncapped_tests_run_the_capped_argv(self) -> None: def test_a_timeout_exits_124(self) -> None: self.scope() self.stub('python', 'exec sleep 30\n') - script = step(TESTS)['run'].replace('--kill-after=3m 110m', '--kill-after=3s 1s') + script = step(TESTS)['run'].replace('--kill-after=3m 180m', '--kill-after=3s 1s') self.assertNotEqual(script, step(TESTS)['run']) result = self.run_step(script) self.assertEqual(result.returncode, 124, result.stderr) @@ -488,7 +488,7 @@ def test_the_summary_reports_each_cause_distinctly(self) -> None: encoding='utf-8') base = 'scope=12G\n' self.assertEqual(self.titles(base + 'exit=124\noom_kills=0\nmemory_errors=0\n'), - ['tests timed out at 110 minutes']) + ['tests timed out at 180 minutes']) self.assertEqual(self.titles(base + 'exit=137\noom_kills=1\nmemory_errors=0\n'), ['tests lost a process to the OOM killer']) self.assertEqual(self.titles(base + 'exit=1\noom_kills=0\nmemory_errors=3\n'), ['tests raised MemoryError']) @@ -501,21 +501,21 @@ def test_the_summary_reports_each_cause_distinctly(self) -> None: def test_a_hung_teardown_killed_after_the_timeout_is_a_timeout(self) -> None: """``--kill-after`` ends a teardown SIGINT did not with SIGKILL, exit 137, as an OOM kill does.""" base = 'scope=12G\nmemory_errors=0\n' - timeout = ['tests timed out at 110 minutes'] - self.assertEqual(self.titles(base + 'exit=137\noom_kills=0\nelapsed=6781\n'), timeout) - self.assertEqual(self.titles(base + 'exit=137\noom_kills=0\nelapsed=6600\n'), timeout) - # Before 110 minutes, or with an OOM kill, 137 is not the timeout's. - self.assertEqual(self.titles(base + 'exit=137\noom_kills=0\nelapsed=6599\n'), []) - self.assertEqual(self.titles(base + 'exit=137\noom_kills=1\nelapsed=6781\n'), + timeout = ['tests timed out at 180 minutes'] + self.assertEqual(self.titles(base + 'exit=137\noom_kills=0\nelapsed=10981\n'), timeout) + self.assertEqual(self.titles(base + 'exit=137\noom_kills=0\nelapsed=10800\n'), timeout) + # Before 180 minutes, or with an OOM kill, 137 is not the timeout's. + self.assertEqual(self.titles(base + 'exit=137\noom_kills=0\nelapsed=10799\n'), []) + self.assertEqual(self.titles(base + 'exit=137\noom_kills=1\nelapsed=10981\n'), ['tests lost a process to the OOM killer']) self.assertEqual(self.titles(base + 'exit=137\noom_kills=0\n'), []) - killed = self.summary(base + 'exit=137\noom_kills=0\nelapsed=6781\n')[0] - self.assertIn('timeout sent SIGKILL 3 minutes on (exit 137, no OOM kill), scope 12G, 6781 s', killed) + killed = self.summary(base + 'exit=137\noom_kills=0\nelapsed=10981\n')[0] + self.assertIn('timeout sent SIGKILL 3 minutes on (exit 137, no OOM kill), scope 12G, 10981 s', killed) def test_a_tests_step_killed_after_the_timeout_records_137_and_its_time(self) -> None: self.scope() self.stub('python', "trap '' INT\nsleep 5\n") - script = step(TESTS)['run'].replace('--kill-after=3m 110m', '--kill-after=1s 1s') + script = step(TESTS)['run'].replace('--kill-after=3m 180m', '--kill-after=1s 1s') self.assertNotEqual(script, step(TESTS)['run']) self.no_real_sudo() result = self.run_step(script) @@ -558,7 +558,7 @@ def test_only_the_experimental_job_runs_it(self) -> None: def test_the_caps_and_check_name_are_unchanged(self) -> None: spec = job(*EXPERIMENTAL) self.assertEqual(spec['name'], CHECK_NAME) - self.assertEqual(spec['timeout-minutes'], 140) + self.assertEqual(spec['timeout-minutes'], 210) self.assertEqual({entry.get('id'): entry['timeout-minutes'] for entry in spec['steps'] if 'timeout-minutes' in entry}, CAPS) @@ -589,10 +589,10 @@ def test_the_workflows_page_describes_the_monitor(self) -> None: def test_the_workflows_page_describes_the_containment(self) -> None: page = prose(ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst') - for phrase in (f'``GRAAL_PYTHON_VM_ARGS={HEAP}``', '``-n 3``', '``timeout --signal=INT --kill-after=3m 110m``', + for phrase in (f'``GRAAL_PYTHON_VM_ARGS={HEAP}``', '``-n 3``', '``timeout --signal=INT --kill-after=3m 180m``', f'``prlimit --fsize={FSIZE}``', '``Containment summary``', '``pcapkit-probe-.scope``', '117-151 minutes', '137', f'``MemoryMax={SCOPE_MEMORY}``', - '``No memory cap``', '``$RUNNER_TEMP/tests-status``', '``tests timed out at 110 minutes``', + '``No memory cap``', '``$RUNNER_TEMP/tests-status``', '``tests timed out at 180 minutes``', '``tests lost a process to the OOM killer``', '``tests raised MemoryError``'): with self.subTest(phrase=phrase): self.assertIn(phrase, page) diff --git a/tests/project/test_superseded_runs_1938.py b/tests/project/test_superseded_runs_1938.py index 869389cdea..8964643aaa 100644 --- a/tests/project/test_superseded_runs_1938.py +++ b/tests/project/test_superseded_runs_1938.py @@ -878,7 +878,7 @@ def test_a_current_run_keeps_the_tests_status(self) -> None: def test_the_watcher_runs_the_scope_and_names_it(self) -> None: """Every cap stays on pytest, and the watcher runs outside the scope, which it names to ``--scope``.""" script = step(TESTS)['run'] - capped = 'set -- prlimit --fsize=2147483648 "$gnu_timeout" --signal=INT --kill-after=3m 110m "$@"\n' + capped = 'set -- prlimit --fsize=2147483648 "$gnu_timeout" --signal=INT --kill-after=3m 180m "$@"\n' scoped = 'set -- scoped pcapkit-tests.scope "$@"\n' named = 'watched=pcapkit-tests.scope\n' watched = ('set -- "$WATCHER_PYTHON" util/superseded.py watch --interval 60 --grace 30 --scope "$watched" \\\n' diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index 667a5c5b97..56d47ce559 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -70,9 +70,10 @@ #: The workflows that wait on "Experimental Tests" completing, to post its report alone (#2030). REPORTERS = ('ci-reports.yml',) #: Each job's ``timeout-minutes``, and its capped steps', by ``id``: ``test``'s are -#: main's, ``experimental``'s are sized for GraalPy (88-113 minutes, estimate). +#: main's, ``experimental``'s are sized for GraalPy (#2739: about 1.5 times its leg in job +#: 125752017215, whose last run is estimated). #: The watcher's interpreter and its requests are capped too (#1938). -CAPS = {JOB: (45, {}), EXPERIMENTAL: (140, {'python': 5, 'install': 10, 'watcher-python': 1, 'watcher': 1, 'tests': 120})} +CAPS = {JOB: (45, {}), EXPERIMENTAL: (210, {'python': 5, 'install': 10, 'watcher-python': 1, 'watcher': 1, 'tests': 190})} #: A pytest-xdist worker count on a command line: ``-n X``, ``-nX``, ``--numprocesses X`` or #: ``--numprocesses=X``, with or without ``--dist`` after it. WORKERS_FLAG = re.compile(r'(?[^\s;|&]+)') @@ -670,7 +671,7 @@ def test_it_installs_what_test_installs(self) -> None: self.assertIn('test', [found.name for found in _dependency_gates.pytest_jobs()]) def test_ci_timing_still_reads_its_legs(self) -> None: - """The weekly CI Timing review reads the workflow, so its two-hour legs stay measured.""" + """The weekly CI Timing review reads the workflow, so its long legs stay measured.""" from util import ci_timing # pylint: disable=import-outside-toplevel self.assertIn('experimental.yml', ci_timing.OTHER_WORKFLOWS) specs = ci_timing.parse_workflow((ROOT / '.github' / 'workflows' / 'experimental.yml').read_text(encoding='utf-8')) @@ -678,7 +679,7 @@ def test_ci_timing_still_reads_its_legs(self) -> None: with self.subTest(python=version): spec = ci_timing.match_job(specs, f'Python {version} (experimental)') self.assertEqual((spec['id'], spec['timeout'], ci_timing.step_cap(spec, ci_timing.TESTS_STEP)), - ('experimental', 140, 120)) + ('experimental', 210, 190)) def test_no_required_context_names_a_leg(self) -> None: names = {f'Python {leg["python-version"]} (experimental)' for leg in legs(*EXPERIMENTAL)}