diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index 61dbf23035..e45b543253 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -334,11 +334,13 @@ jobs: # - #1963: util/pytest_shards.py runs pytest once per shard `SHARDS` # names, a directory or a glob of one directory's entries, then once # over the rest, so every run starts its workers afresh and none - # outlives its heap. Each shard is sized by its largest live heap after - # a full GC on GraalPy. The script runs inside the timeout, scope and - # watcher below, so they hold over the whole tier; on the timeout's - # SIGINT no later run starts. Each shard's run writes a JUnit file of - # its own beside junit.xml. + # outlives its heap. #2713: the shards that lost workers are split by + # their tests' JUnit seconds on GraalPy, each to run in under 5 + # minutes; the run over the rest is left whole, as it took about 6.7 + # minutes in job 125639681253 and lost no worker. The script runs + # inside the timeout, scope and watcher below, so they hold over the + # whole tier; on the timeout's SIGINT no later run starts. Each shard's + # run writes a JUnit file of its own beside junit.xml. # - `timeout` sends SIGINT at 110 minutes, under the step's cap, so pytest # still writes the JUnit XML, and exits 124; SIGKILL 3 minutes on. # #2020: GNU's `timeout`, which Ubuntu 26.04 installs as `gnutimeout`: @@ -394,7 +396,10 @@ jobs: SHARDS: >- ${{ startsWith(matrix.python-version, 'graalpy') && 'tests/const tests/corekit/test_fields_n* tests/corekit/test_[a-h]* tests/corekit/test_i* tests/corekit - tests/foundation tests/protocols/internet tests/protocols/[a-m]* tests/protocols + tests/foundation/reassembly tests/foundation/test_* tests/foundation + tests/protocols/internet/test_ipv4* tests/protocols/internet/test_ipv6* tests/protocols/internet + tests/protocols/application tests/protocols/link tests/protocols/misc/test_pcapng* tests/protocols/misc + tests/protocols/transport/test_tcp* tests/protocols/transport tests/protocols/test_t* tests/protocols tests/toolkit' || '' }} GH_TOKEN: ${{ github.token }} WATCHER: ${{ steps.watcher.outcome }} diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index ca0db328a0..4fb86f0816 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -1228,15 +1228,27 @@ warnings: names -- a directory, or a glob of one directory's entries such as ``tests/corekit/test_i*`` -- and once more over the rest, and every run starts its own workers. Each run also ignores what every earlier shard took, so a glob may - precede its own directory, which then takes the remainder. The shards are sized by - each one's largest live heap after a full GC, measured locally on GraalPy 25.4.4 at - ``-n 3``: run whole, the tier lost 8 workers, its heap reached the 2 GiB cap, and it - took 3559 s; in these shards none went down, no heap passed 1303 MB, and it - took 1879 s. The runs share the step's ``timeout``, scope and watcher, so the - caps hold over the whole tier; on ``timeout``'s SIGINT the run in progress writes - its JUnit file and no later one starts. Each shard's run writes a JUnit file of its - own beside ``junit.xml``, such as ``junit-tests-protocols.xml``, and ``Upload test - timings`` takes them all. 3.14t's leg runs pytest once. + precede its own directory, which then takes the remainder. The shards were first + sized by each one's largest live heap after a full GC, measured locally on GraalPy + 25.4.4 at ``-n 3``: run whole, the tier lost 8 workers, its heap reached the 2 GiB + cap, and it took 3559 s; in those shards none went down, no heap passed 1303 MB, and + it took 1879 s. The tier outgrew them (:issue:`2713`). On ``57a6c366d``, job + 125639681253 lost 8 workers, all in its runs of 10-14 minutes -- ``tests/foundation``, + ``tests/protocols/internet``, ``tests/protocols/[a-m]*`` and ``tests/protocols`` -- + and all 9.8-12 minutes into the run. The ``[monitor]`` samples in its ``Run unit + tests`` log put each of those runs' largest worker at 2.2 GiB RSS at 5 minutes old, + and at 2.65-2.78 GiB at its largest. Its runs of 2-7 minutes lost none, and its + tests took 4510 s of the ``timeout``'s 6600. So those shards are split by time, + sized from that job's ``junit-experimental-graalpy-25.4`` artifact: a shard's + estimate is its tests' JUnit seconds over the 3 workers, plus 50 s for the 46-57 s + that each of the job's shards that lost no worker took beyond that, start to start. + Every shard split this way comes to under 5 minutes, and the tier to about 4750 s + (estimate). The run over the rest is left whole: in that job it took about 6.7 + minutes and lost no worker. The runs share the step's ``timeout``, scope and + watcher, so the caps hold over the whole tier; on ``timeout``'s SIGINT the run in + progress writes its JUnit file and no later one starts. Each shard's run writes a + JUnit file of its own beside ``junit.xml``, such as ``junit-tests-protocols.xml``, + and ``Upload test timings`` takes them all. 3.14t's leg runs pytest once. * ``timeout --signal=INT --kill-after=3m 110m`` ends pytest under the step's 120-minute cap, so it still writes the junit XML, and exits 124; ``prlimit --fsize=2147483648`` caps any file at 2 GiB. The ``timeout`` is GNU's, ``gnutimeout`` where it exists diff --git a/tests/project/test_graalpy_shards_1963.py b/tests/project/test_graalpy_shards_1963.py index 89745a1c55..4b729089c0 100644 --- a/tests/project/test_graalpy_shards_1963.py +++ b/tests/project/test_graalpy_shards_1963.py @@ -34,9 +34,14 @@ from tests.project.test_resource_monitor_1740 import DEFAULT, EXPERIMENTAL, HEAP, TESTS, by_impl, sh, step from util import pytest_shards -#: GraalPy's shards, in the order they run; the rest of the tier runs last. +#: GraalPy's shards, in the order they run; the rest of the tier runs last. #2713 split the +#: shards that lost workers in job 125639681253 by their JUnit seconds there. SHARDS = ('tests/const tests/corekit/test_fields_n* tests/corekit/test_[a-h]* tests/corekit/test_i* tests/corekit ' - 'tests/foundation tests/protocols/internet tests/protocols/[a-m]* tests/protocols tests/toolkit') + 'tests/foundation/reassembly tests/foundation/test_* tests/foundation ' + 'tests/protocols/internet/test_ipv4* tests/protocols/internet/test_ipv6* tests/protocols/internet ' + 'tests/protocols/application tests/protocols/link tests/protocols/misc/test_pcapng* tests/protocols/misc ' + 'tests/protocols/transport/test_tcp* tests/protocols/transport tests/protocols/test_t* tests/protocols ' + 'tests/toolkit') SCRIPT = ROOT / 'util' / 'pytest_shards.py' #: The lines that build the command the step's caps run, in order. PYTEST = 'set -- python -m pytest "$@"\n'