|
| 1 | +# -*- coding: utf-8 -*- |
| 2 | +"""#1969: every pytest and unittest command in ``unit-tests.yml`` caps the files it writes. |
| 3 | +
|
| 4 | +A test that writes without bound fills the runner's disk (#1372's ``out.json``). |
| 5 | +:file:`util/cap_file_size.py` sets ``RLIMIT_FSIZE`` and execs the command, and is |
| 6 | +the one place the limit is written down. This module pins both halves: every step |
| 7 | +of :file:`.github/workflows/unit-tests.yml` that runs pytest, unittest or |
| 8 | +:file:`util/run_unittest_leg.py` does so through the script, so a new step without |
| 9 | +it fails here; and the script caps what its command writes, so that a write past |
| 10 | +the cap errors rather than passes. |
| 11 | +
|
| 12 | +""" |
| 13 | + |
| 14 | +from __future__ import annotations |
| 15 | + |
| 16 | +import errno |
| 17 | +import os |
| 18 | +import pathlib |
| 19 | +import re |
| 20 | +import signal |
| 21 | +import subprocess # nosec: B404 |
| 22 | +import sys |
| 23 | +import tempfile |
| 24 | +import unittest |
| 25 | + |
| 26 | +from tests._workflows import WORKFLOWS, load, loads |
| 27 | + |
| 28 | +try: |
| 29 | + import resource |
| 30 | +except ImportError: # pragma: no cover - not on Windows |
| 31 | + resource = None # type: ignore[assignment] |
| 32 | + |
| 33 | +ROOT = pathlib.Path(__file__).resolve().parents[2] |
| 34 | +SCRIPT = ROOT / 'util' / 'cap_file_size.py' |
| 35 | +DOC = ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst' |
| 36 | +WORKFLOW = 'unit-tests.yml' |
| 37 | +#: How a step runs a command under the cap. |
| 38 | +CAP = 'python util/cap_file_size.py ' |
| 39 | +#: A command that runs tests: pytest or unittest as a module, coverage over |
| 40 | +#: either, or util/run_unittest_leg.py. |
| 41 | +RUNS_TESTS = re.compile(r'-m (?:pytest|unittest|coverage run)\b|util/run_unittest_leg\.py\b') |
| 42 | +#: The jobs that run tests, each of which must be found running some. |
| 43 | +TEST_JOBS = frozenset({ |
| 44 | + 'test', 'test-ktlo', 'integration', 'integration-ktlo', 'engine-tests', |
| 45 | + 'pypcap-parity', 'pypcap-parity-ktlo', 'unittest-ordering', 'project-tests', 'gate', |
| 46 | +}) |
| 47 | + |
| 48 | +if str(ROOT / 'util') not in sys.path: |
| 49 | + sys.path.insert(0, str(ROOT / 'util')) |
| 50 | + |
| 51 | +import cap_file_size # noqa: E402 pylint: disable=wrong-import-position |
| 52 | + |
| 53 | + |
| 54 | +def commands(script: 'str') -> 'list[str]': |
| 55 | + """The logical lines of ``script``, continuations joined, comments and blanks dropped.""" |
| 56 | + lines = [] |
| 57 | + for line in script.replace('\\\n', ' ').splitlines(): |
| 58 | + line = line.strip() |
| 59 | + if line and not line.startswith('#'): |
| 60 | + lines.append(line) |
| 61 | + return lines |
| 62 | + |
| 63 | + |
| 64 | +def suite_commands(workflow: 'dict[str, object]') -> 'list[tuple[str, str, str]]': |
| 65 | + """Every ``(job, step, command)`` of ``workflow`` that runs tests.""" |
| 66 | + found = [] |
| 67 | + for job_id, spec in workflow['jobs'].items(): # type: ignore[attr-defined] |
| 68 | + for entry in spec.get('steps', ()): |
| 69 | + for line in commands(str(entry.get('run', ''))): |
| 70 | + if RUNS_TESTS.search(line): |
| 71 | + found.append((job_id, entry.get('name', entry.get('id', '?')), line)) |
| 72 | + return found |
| 73 | + |
| 74 | + |
| 75 | +def uncapped(workflow: 'dict[str, object]') -> 'list[tuple[str, str, str]]': |
| 76 | + """The test commands of ``workflow`` that do not run through :data:`CAP`.""" |
| 77 | + bad = [] |
| 78 | + for where in suite_commands(workflow): |
| 79 | + line = where[2] |
| 80 | + start = RUNS_TESTS.search(line).start() # type: ignore[union-attr] |
| 81 | + at = line.find(CAP) |
| 82 | + if at == -1 or at + len(CAP) > start or not line[at + len(CAP):].startswith('python '): |
| 83 | + bad.append(where) |
| 84 | + return bad |
| 85 | + |
| 86 | + |
| 87 | +class TestEveryTestStepIsCapped(unittest.TestCase): |
| 88 | + """Each pytest and unittest command in ``unit-tests.yml`` runs under the cap.""" |
| 89 | + |
| 90 | + def setUp(self) -> None: |
| 91 | + self.text = (WORKFLOWS / WORKFLOW).read_text(encoding='utf-8') |
| 92 | + self.workflow = load(WORKFLOW) |
| 93 | + |
| 94 | + def test_every_test_command_runs_through_the_script(self) -> None: |
| 95 | + self.assertEqual(uncapped(self.workflow), []) |
| 96 | + |
| 97 | + def test_every_test_job_is_seen(self) -> None: |
| 98 | + # The scan is not vacuous: each job that runs tests is found doing so, |
| 99 | + # the KTLO ones through their YAML alias, the engine cells inside the |
| 100 | + # script their step writes. |
| 101 | + seen = {job_id for job_id, _, _ in suite_commands(self.workflow)} |
| 102 | + self.assertLessEqual(TEST_JOBS, seen) |
| 103 | + |
| 104 | + def test_a_doctored_copy_without_the_cap_is_caught(self) -> None: |
| 105 | + bare = uncapped(loads(self.text.replace(CAP, ''))) |
| 106 | + self.assertLessEqual(TEST_JOBS, {job_id for job_id, _, _ in bare}) |
| 107 | + extra = loads(self.text.replace( |
| 108 | + ' - name: Run full test suite\n run: ' + CAP, |
| 109 | + ' - name: Run a new suite\n run: python -m pytest tests/project\n' |
| 110 | + ' - name: Run full test suite\n run: ' + CAP, 1)) |
| 111 | + self.assertEqual(uncapped(extra), [('gate', 'Run a new suite', 'python -m pytest tests/project')]) |
| 112 | + |
| 113 | + def test_the_cap_has_one_mechanism(self) -> None: |
| 114 | + for word in ('ulimit -f', 'prlimit'): |
| 115 | + self.assertEqual([line for line in commands(self.text) if word in line], [], word) |
| 116 | + |
| 117 | + |
| 118 | +@unittest.skipIf(resource is None, 'RLIMIT_FSIZE is POSIX') |
| 119 | +class TestTheScript(unittest.TestCase): |
| 120 | + """util/cap_file_size.py caps the file size of the command it execs.""" |
| 121 | + |
| 122 | + def setUp(self) -> None: |
| 123 | + self.tmp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with |
| 124 | + self.addCleanup(self.tmp.cleanup) |
| 125 | + self.path = os.path.join(self.tmp.name, 'out') |
| 126 | + |
| 127 | + def capped(self, limit: 'int', *command: 'str', **kwargs: 'object') -> 'subprocess.CompletedProcess[str]': |
| 128 | + """``command`` run by the script, with :data:`cap_file_size.LIMIT` set to ``limit``.""" |
| 129 | + main = ('import sys; sys.path.insert(0, sys.argv[1]); import cap_file_size; ' |
| 130 | + 'cap_file_size.LIMIT = int(sys.argv[2]); cap_file_size.main(sys.argv[3:])') |
| 131 | + return subprocess.run([sys.executable, '-c', main, str(SCRIPT.parent), str(limit), *command], # nosec: B603 |
| 132 | + cwd=self.tmp.name, text=True, check=False, timeout=60, **kwargs) # type: ignore[call-overload] |
| 133 | + |
| 134 | + def test_the_limit_is_two_gib(self) -> None: |
| 135 | + self.assertEqual(cap_file_size.LIMIT, 2 * 1024 ** 3) |
| 136 | + |
| 137 | + def test_the_command_runs_under_the_limit(self) -> None: |
| 138 | + _, hard = resource.getrlimit(resource.RLIMIT_FSIZE) |
| 139 | + expected = cap_file_size.LIMIT if hard == resource.RLIM_INFINITY else min(cap_file_size.LIMIT, hard) |
| 140 | + probe = 'import resource; print(*resource.getrlimit(resource.RLIMIT_FSIZE))' |
| 141 | + result = subprocess.run([sys.executable, str(SCRIPT), sys.executable, '-c', probe], # nosec: B603 |
| 142 | + capture_output=True, text=True, check=False, timeout=60) |
| 143 | + self.assertEqual(result.returncode, 0, result.stderr) |
| 144 | + self.assertEqual(result.stdout.split(), [str(expected)] * 2) |
| 145 | + |
| 146 | + def test_a_lower_hard_limit_is_kept(self) -> None: |
| 147 | + def lower() -> None: |
| 148 | + resource.setrlimit(resource.RLIMIT_FSIZE, (1 << 20, 1 << 20)) |
| 149 | + |
| 150 | + probe = 'import resource; print(*resource.getrlimit(resource.RLIMIT_FSIZE))' |
| 151 | + result = self.capped(1 << 30, sys.executable, '-c', probe, capture_output=True, |
| 152 | + preexec_fn=lower) # pylint: disable=subprocess-popen-preexec-fn |
| 153 | + self.assertEqual(result.returncode, 0, result.stderr) |
| 154 | + self.assertEqual(result.stdout.split(), [str(1 << 20)] * 2) |
| 155 | + |
| 156 | + def test_a_python_write_past_the_limit_raises_efbig(self) -> None: |
| 157 | + writer = ('import sys\nwith open(sys.argv[1], "wb") as file:\n' |
| 158 | + ' file.write(b"x" * 8192)\n') |
| 159 | + result = self.capped(4096, sys.executable, '-c', writer, self.path, capture_output=True) |
| 160 | + self.assertNotEqual(result.returncode, 0) |
| 161 | + self.assertIn(f'OSError: [Errno {errno.EFBIG}] File too large', result.stderr) |
| 162 | + self.assertEqual(os.path.getsize(self.path), 4096) |
| 163 | + |
| 164 | + def test_another_program_is_killed_by_sigxfsz(self) -> None: |
| 165 | + # CPython ignores SIGXFSZ, and an ignored signal stays ignored across |
| 166 | + # exec, so this holds only because the script restores the default. |
| 167 | + with open(self.path, 'wb') as file: |
| 168 | + result = self.capped(4096, 'head', '-c', '8192', '/dev/zero', stdout=file, |
| 169 | + stderr=subprocess.PIPE) |
| 170 | + self.assertEqual(result.returncode, -signal.SIGXFSZ, result.stderr) |
| 171 | + self.assertEqual(os.path.getsize(self.path), 4096) |
| 172 | + |
| 173 | + def test_no_command_is_a_usage_error(self) -> None: |
| 174 | + result = subprocess.run([sys.executable, str(SCRIPT)], capture_output=True, text=True, # nosec: B603 |
| 175 | + check=False, timeout=60) |
| 176 | + self.assertEqual(result.returncode, cap_file_size.USAGE) |
| 177 | + self.assertIn('usage:', result.stderr) |
| 178 | + |
| 179 | + |
| 180 | +class TestTheWorkflowsPage(unittest.TestCase): |
| 181 | + """docs/source/contributing/workflows.rst names the script for the cap.""" |
| 182 | + |
| 183 | + def test_the_page_names_the_script(self) -> None: |
| 184 | + text = DOC.read_text(encoding='utf-8') |
| 185 | + self.assertIn('``util/cap_file_size.py``', text) |
| 186 | + self.assertNotIn('ulimit -f', text) |
| 187 | + |
| 188 | + |
| 189 | +if __name__ == '__main__': |
| 190 | + unittest.main() |
0 commit comments