Stream pull request diffs to disk, and prune CI's uploaded runs []

.github/workflows/build-and-test-ci.yml

@@ -308,7 +308,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     container:
-      image: ghcr.io/edgcpp/ci-runner@sha256:565d7b6581912008ca91c5dbc940280c6e28a02c8cb6487d88ee5f0d7e8c51d7
+      image: ghcr.io/edgcpp/ci-runner@sha256:daae67adf04085b9ae54a816b35053b79086070434d2da4aafd6662f6ecb0bf7
     defaults:
       run:
         shell: edg-container-exec "bash -eo pipefail {0}"

.github/workflows/ci-toolchain.yml

@@ -66,7 +66,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     container:
-      image: &ci-runner-image ghcr.io/edgcpp/ci-runner@sha256:565d7b6581912008ca91c5dbc940280c6e28a02c8cb6487d88ee5f0d7e8c51d7
+      image: &ci-runner-image ghcr.io/edgcpp/ci-runner@sha256:daae67adf04085b9ae54a816b35053b79086070434d2da4aafd6662f6ecb0bf7
     defaults: &container-defaults
       run:
         shell: edg-container-exec "bash -eo pipefail {0}"
@@ -267,6 +267,17 @@ jobs:
 
           edg-prefix-env-loader .tmp/extensive-ci.edgivk
 
+      # Reviews (acknowledg-forge, edgy-review) read only the .rt.tar.gz
+      # files changes.elist names; edgy -A archives every test's. Pruning
+      # must never keep the results from being uploaded, hence
+      # continue-on-error.
+      - name: Prune .rt.tar.gz results changes.elist doesn't name
+        if: ${{ failure() }}
+        continue-on-error: true
+        env:
+          RUN_TIMESTAMP: ${{ inputs.run_timestamp }}
+        run: edg-prune-test-run "tests/runs/${RUN_TIMESTAMP}"
+
       - name: Upload partial elist, elog, and .rt.tar.gz results
         if: ${{ failure() }}
         uses: actions/upload-artifact@v7

.github/workflows/promote-test-cache.yml

@@ -72,7 +72,7 @@ jobs:
     # The test legs' image, so that actions/cache picks the same compression
     # (part of an entry's version) and the test legs can restore the entry.
     container:
-      image: ghcr.io/edgcpp/ci-runner@sha256:565d7b6581912008ca91c5dbc940280c6e28a02c8cb6487d88ee5f0d7e8c51d7
+      image: ghcr.io/edgcpp/ci-runner@sha256:daae67adf04085b9ae54a816b35053b79086070434d2da4aafd6662f6ecb0bf7
     strategy:
       fail-fast: false
       matrix:

.github/workflows/strict-build-check.yml

@@ -158,7 +158,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     container:
-      image: ghcr.io/edgcpp/ci-runner@sha256:565d7b6581912008ca91c5dbc940280c6e28a02c8cb6487d88ee5f0d7e8c51d7
+      image: ghcr.io/edgcpp/ci-runner@sha256:daae67adf04085b9ae54a816b35053b79086070434d2da4aafd6662f6ecb0bf7
     defaults:
       run:
         shell: edg-container-exec "bash -eo pipefail {0}"
@@ -185,7 +185,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     container:
-      image: ghcr.io/edgcpp/ci-runner@sha256:565d7b6581912008ca91c5dbc940280c6e28a02c8cb6487d88ee5f0d7e8c51d7
+      image: ghcr.io/edgcpp/ci-runner@sha256:daae67adf04085b9ae54a816b35053b79086070434d2da4aafd6662f6ecb0bf7
     defaults:
       run:
         shell: edg-container-exec "bash -eo pipefail {0}"
@@ -212,7 +212,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     container:
-      image: ghcr.io/edgcpp/ci-runner@sha256:565d7b6581912008ca91c5dbc940280c6e28a02c8cb6487d88ee5f0d7e8c51d7
+      image: ghcr.io/edgcpp/ci-runner@sha256:daae67adf04085b9ae54a816b35053b79086070434d2da4aafd6662f6ecb0bf7
     defaults:
       run:
         shell: edg-container-exec "bash -eo pipefail {0}"
@@ -239,7 +239,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     container:
-      image: ghcr.io/edgcpp/ci-runner@sha256:565d7b6581912008ca91c5dbc940280c6e28a02c8cb6487d88ee5f0d7e8c51d7
+      image: ghcr.io/edgcpp/ci-runner@sha256:daae67adf04085b9ae54a816b35053b79086070434d2da4aafd6662f6ecb0bf7
     defaults:
       run:
         shell: edg-container-exec "bash -eo pipefail {0}"

dev_tools/bin/edg-prune-test-run

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM
+# Exceptions.
+# See https://edgcpp.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+'''Prune a test run down to what a review of it reads, e.g., before CI
+uploads the run:
+
+  edg-prune-test-run tests/runs/<run timestamp>
+
+Reviews of a run read only the .rt.tar files its changes.elist files name.
+The replays directory is removed whole.  In each configuration, a suite
+with no test named there is removed whole (its directory, and its
+suite-<name> logs), as is every suite of a configuration without a
+changes.elist; of the other suites, the .rt.tar files not named there are
+removed.
+'''
+
+import argparse
+import sys
+
+import edgtest
+import edgutil
+
+from pathlib import Path
+
+def _parse_args() -> argparse.Namespace:
+  parser = argparse.ArgumentParser(
+    description = (
+      "Remove the .rt.tar files of a test run that no configuration's "
+      'changes.elist names.'
+    )
+  )
+  parser.add_argument(
+    'run_dir',
+    type = Path,
+    help = 'the run directory (e.g., tests/runs/<run timestamp>)'
+  )
+  return parser.parse_args()
+
+def main() -> int:
+  args = _parse_args()
+  if not args.run_dir.is_dir():
+    edgutil.eprint(f"error: {args.run_dir} is not a directory")
+    return 1
+
+  # Counts: of .rt.tar files kept and removed, and of suites removed.
+  kept, removed, suites = edgtest.prune_run_archives(args.run_dir)
+  print(
+    f"Kept {kept} .rt.tar files named by changes.elist files; removed "
+    f"{removed} others, and {suites} suites with no changes."
+  )
+  return 0
+
+if __name__ == '__main__':
+  sys.exit(main())

dev_tools/bin/edg-rematerialize-test-run

@@ -73,8 +73,14 @@ def build_diff_files(diff_file: Path,
     curr_diff_file_descr = None
     diff_file_to_handle = {}
 
-    for line in diff_file_handle:
-      line_match = GIT_DIFF_REGEX.match(line)
+    # Lines are read in bounded pieces so that an enormous one is never held
+    # whole; only a piece that starts a line can be a header, and every piece
+    # is written through unchanged.
+    for line, at_line_start in edgutil.iter_line_pieces(
+      diff_file_handle,
+      edgutil.MAX_LINE_CHARS
+    ):
+      line_match = GIT_DIFF_REGEX.match(line) if at_line_start else None
       if line_match is not None:
         if curr_diff_file_descr is not None:
           # Close the file handle for the previous diff file (if any) to
@@ -180,22 +186,35 @@ def collect_affected_tests(test_home: Path,
       suite_name = file_path.stem
       # An expectations update.
       with open(diff_file.diff_file) as file_handle:
-        for line in file_handle:
-          if line.startswith('---') or line.startswith('+++'):
+        # edgutil.MAX_LINE_CHARS is far more than any genuine expectation
+        # entry needs, so a longer line is refused (as when the file is
+        # read), without being held whole.
+        try:
+          for line in edgutil.iter_limited_lines(
+            file_handle,
+            edgutil.MAX_LINE_CHARS
+          ):
+            if line.startswith('---') or line.startswith('+++'):
               # Do not take in lines like:
               #
               #   --- a/tests/expectations/edg_x86_64_cp/modules.log
               #   +++ b/tests/expectations/edg_x86_64_cp/modules.log
               #
               continue
 
-          if line.startswith('-') or line.startswith('+'):
-            # Do a naive parse to get the test file name.
-            test_file_rel_path = PurePosixPath(line[1:].split(':')[0])
-            result.tests.add(
-              tests_dir.joinpath(suite_name, test_file_rel_path)
-            )
-            result.configs.add(config_name)
+            if line.startswith('-') or line.startswith('+'):
+              # Do a naive parse to get the test file name.
+              test_file_rel_path = PurePosixPath(line[1:].split(':')[0])
+              result.tests.add(
+                tests_dir.joinpath(suite_name, test_file_rel_path)
+              )
+              result.configs.add(config_name)
+        except edgutil.LineTooLongError:
+          eprint(
+            f"{diff_file.file_name}: a changed line is longer than "
+            f"{edgutil.MAX_LINE_CHARS} characters"
+          )
+          sys.exit(1)
 
   return result
 
@@ -548,10 +567,20 @@ async def simulate_test(test_home: Path, config_run_dir: Path,
   return test_status_lines
 
 def read_suite_expectation_lines(suite_file: Path) -> List[str]:
+  '''The lines of ``suite_file`` (none if there is no such file); exits if
+  one is longer than ``edgutil.MAX_LINE_CHARS``, rather than holding it.
+  '''
   if not suite_file.exists():
     return []
-  with open(suite_file, 'r') as file_handle:
-    return file_handle.readlines()
+  try:
+    with open(suite_file, 'r') as file_handle:
+      return list(edgutil.iter_limited_lines(
+        file_handle,
+        edgutil.MAX_LINE_CHARS
+      ))
+  except edgutil.LineTooLongError as ex:
+    eprint(f"{suite_file}: {ex}")
+    sys.exit(1)
 
 def load_test_suite_expectations(all_expectations: Dict[str, List[str]],
                                  suite_name: str, expectations_dir: Path,

dev_tools/pylibs/edgacknowledg/__init__.py

@@ -121,12 +121,17 @@ def request_tags_with_prefix(prefix: str) -> FrozenSet[str]:
   ALLOWED_BENCH_EDIT_TAGS
 )
 
-PROTOCOL_REVISION = 20
+PROTOCOL_REVISION = 21
 
 # 10MiB
 MAX_MESSAGE_SIZE = 10485760
 # 8MiB
 MAX_DISPLAY_CONTENT_SIZE = 8388608
+# 16Ki characters: a longer line is cut for display (see lines_truncated).
+MAX_DISPLAY_LINE_SIZE = 16384
+# 1Ki characters: a longer label (e.g. a status line's variant, at most a few
+# hundred in practice) is cut for display, ending in an ellipsis.
+MAX_DISPLAY_LABEL_SIZE = 1024
 
 # Most lines shown on each side of the requested line by a ranged
 # ``test.src.show-src`` request.

dev_tools/pylibs/edgacknowledg/budget.py

@@ -0,0 +1,27 @@
+# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM
+# Exceptions.
+# See https://edgcpp.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+'''A size budget shared by readers that count bytes or characters.'''
+
+
+class BudgetExceeded(Exception):
+  '''Raised when an operation would spend more than its budget allows.'''
+
+
+class Budget:
+  def __init__(self, limit: int) -> None:
+    if limit < 0:
+      raise ValueError('budget limit must be nonnegative')
+    self.limit = limit
+    self.remaining = limit
+    self.exceeded = False
+
+  def spend(self, count: int) -> None:
+    if count < 0:
+      raise ValueError('budget count must be nonnegative')
+    if count > self.remaining:
+      self.exceeded = True
+      raise BudgetExceeded()
+    self.remaining -= count