Skip to content

Commit 5c5b793

Browse files
Add safety limits on line length []
1 parent 0e58df9 commit 5c5b793

19 files changed

Lines changed: 1120 additions & 170 deletions

File tree

‎dev_tools/bin/edg-rematerialize-test-run‎

Lines changed: 42 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,14 @@ def build_diff_files(diff_file: Path,
7373
curr_diff_file_descr = None
7474
diff_file_to_handle = {}
7575

76-
for line in diff_file_handle:
77-
line_match = GIT_DIFF_REGEX.match(line)
76+
# Lines are read in bounded pieces so that an enormous one is never held
77+
# whole; only a piece that starts a line can be a header, and every piece
78+
# is written through unchanged.
79+
for line, at_line_start in edgutil.iter_line_pieces(
80+
diff_file_handle,
81+
edgutil.MAX_LINE_CHARS
82+
):
83+
line_match = GIT_DIFF_REGEX.match(line) if at_line_start else None
7884
if line_match is not None:
7985
if curr_diff_file_descr is not None:
8086
# Close the file handle for the previous diff file (if any) to
@@ -180,22 +186,35 @@ def collect_affected_tests(test_home: Path,
180186
suite_name = file_path.stem
181187
# An expectations update.
182188
with open(diff_file.diff_file) as file_handle:
183-
for line in file_handle:
184-
if line.startswith('---') or line.startswith('+++'):
189+
# edgutil.MAX_LINE_CHARS is far more than any genuine expectation
190+
# entry needs, so a longer line is refused (as when the file is
191+
# read), without being held whole.
192+
try:
193+
for line in edgutil.iter_limited_lines(
194+
file_handle,
195+
edgutil.MAX_LINE_CHARS
196+
):
197+
if line.startswith('---') or line.startswith('+++'):
185198
# Do not take in lines like:
186199
#
187200
# --- a/tests/expectations/edg_x86_64_cp/modules.log
188201
# +++ b/tests/expectations/edg_x86_64_cp/modules.log
189202
#
190203
continue
191204

192-
if line.startswith('-') or line.startswith('+'):
193-
# Do a naive parse to get the test file name.
194-
test_file_rel_path = PurePosixPath(line[1:].split(':')[0])
195-
result.tests.add(
196-
tests_dir.joinpath(suite_name, test_file_rel_path)
197-
)
198-
result.configs.add(config_name)
205+
if line.startswith('-') or line.startswith('+'):
206+
# Do a naive parse to get the test file name.
207+
test_file_rel_path = PurePosixPath(line[1:].split(':')[0])
208+
result.tests.add(
209+
tests_dir.joinpath(suite_name, test_file_rel_path)
210+
)
211+
result.configs.add(config_name)
212+
except edgutil.LineTooLongError:
213+
eprint(
214+
f"{diff_file.file_name}: a changed line is longer than "
215+
f"{edgutil.MAX_LINE_CHARS} characters"
216+
)
217+
sys.exit(1)
199218

200219
return result
201220

@@ -548,10 +567,20 @@ async def simulate_test(test_home: Path, config_run_dir: Path,
548567
return test_status_lines
549568

550569
def read_suite_expectation_lines(suite_file: Path) -> List[str]:
570+
'''The lines of ``suite_file`` (none if there is no such file); exits if
571+
one is longer than ``edgutil.MAX_LINE_CHARS``, rather than holding it.
572+
'''
551573
if not suite_file.exists():
552574
return []
553-
with open(suite_file, 'r') as file_handle:
554-
return file_handle.readlines()
575+
try:
576+
with open(suite_file, 'r') as file_handle:
577+
return list(edgutil.iter_limited_lines(
578+
file_handle,
579+
edgutil.MAX_LINE_CHARS
580+
))
581+
except edgutil.LineTooLongError as ex:
582+
eprint(f"{suite_file}: {ex}")
583+
sys.exit(1)
555584

556585
def load_test_suite_expectations(all_expectations: Dict[str, List[str]],
557586
suite_name: str, expectations_dir: Path,

‎dev_tools/pylibs/edgacknowledg/__init__.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,12 +121,17 @@ def request_tags_with_prefix(prefix: str) -> FrozenSet[str]:
121121
ALLOWED_BENCH_EDIT_TAGS
122122
)
123123

124-
PROTOCOL_REVISION = 19
124+
PROTOCOL_REVISION = 20
125125

126126
# 10MiB
127127
MAX_MESSAGE_SIZE = 10485760
128128
# 8MiB
129129
MAX_DISPLAY_CONTENT_SIZE = 8388608
130+
# 16Ki characters: a longer line is cut for display (see lines_truncated).
131+
MAX_DISPLAY_LINE_SIZE = 16384
132+
# 1Ki characters: a longer label (e.g. a status line's variant, at most a few
133+
# hundred in practice) is cut for display, ending in an ellipsis.
134+
MAX_DISPLAY_LABEL_SIZE = 1024
130135

131136
WebSocketsMessageType = Union[str, bytes]
132137
ProtocolMessage = Dict[str, Any]

‎dev_tools/pylibs/edgacknowledg/client/shared.py‎

Lines changed: 111 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
import asyncio
99
import errno
10+
import io
1011
import os
1112
import signal
1213
import stat
@@ -19,6 +20,7 @@
1920
Dict,
2021
List,
2122
IO,
23+
Iterable,
2224
Iterator,
2325
NamedTuple,
2426
Optional,
@@ -37,6 +39,8 @@
3739

3840
from edgacknowledg import (
3941
MAX_DISPLAY_CONTENT_SIZE,
42+
MAX_DISPLAY_LABEL_SIZE,
43+
MAX_DISPLAY_LINE_SIZE,
4044
ProtocolMessage,
4145
RequestTags,
4246
RichName,
@@ -83,6 +87,8 @@ async def handle_request(self, channel: Channel) -> None:
8387
ReviewClientHandler = Any # type: ignore[misc,assignment]
8488

8589

90+
import edgutil
91+
8692
from edgutil import get_running_event_loop, is_path_relative_to
8793

8894
CoroFactory = Callable[[], Awaitable[Any]]
@@ -149,32 +155,111 @@ async def announce_section(channel: Channel,
149155
))
150156

151157

152-
def trunc_display_content(content: str) -> Dict[str, Union[str, bool]]:
153-
'''Truncate ``content`` for UI display; may set ``truncated``.'''
154-
max_content_size = MAX_DISPLAY_CONTENT_SIZE
155-
if len(content) > max_content_size:
156-
# Attempt to truncate at line boundaries.
157-
#
158-
# To save time, find the newline closest to the boundary before walking
159-
# the file size back.
160-
last_newline_permitted = content.find('\n', max_content_size)
161-
if last_newline_permitted != -1:
162-
content = content[:last_newline_permitted]
163-
# Now walk the file size back.
164-
while len(content) > max_content_size:
165-
last_new_line_idx = content.rfind('\n')
166-
if last_new_line_idx != -1:
167-
content = content[:(last_new_line_idx - 1)]
168-
else:
169-
# If there are no more newlines, resort to truncating within
170-
# the line.
171-
content = content[:(max_content_size - 4)]
172-
content += '...'
173-
return {
174-
'truncated': True,
175-
'content': content
176-
}
177-
return { 'content': content }
158+
def trunc_display_label(text: str) -> str:
159+
'''``text`` (one line, e.g. a status) cut for UI display to
160+
``MAX_DISPLAY_LABEL_SIZE`` characters, a cut ending in an ellipsis.
161+
'''
162+
if len(text) > MAX_DISPLAY_LABEL_SIZE:
163+
text = text[:MAX_DISPLAY_LABEL_SIZE - 1] + '\N{HORIZONTAL ELLIPSIS}'
164+
return text
165+
166+
167+
def _display_lines(
168+
pieces: Iterable[Tuple[str, bool]]) -> Iterator[Tuple[str, str, bool]]:
169+
'''Join ``(piece, at_line_start)`` pieces (as ``edgutil.iter_line_pieces``
170+
yields them) into lines, yielding ``(text, line_end, cut)`` for each: at
171+
most ``MAX_DISPLAY_LINE_SIZE`` characters of its text (``cut`` if it had
172+
more, which are dropped as they come), and its end ('\\n', or '' for a
173+
last line without one).
174+
175+
A piece that starts a line ends the one before it, so a text that doesn't
176+
end in a line end (e.g. a command's output) never runs into the next.
177+
'''
178+
# The line being joined (as much of it as is kept), and whether some was
179+
# dropped; ``None`` between lines.
180+
line: Optional[str] = None
181+
cut = False
182+
for piece, at_line_start in pieces:
183+
if line is not None and at_line_start:
184+
yield line, '\n', cut
185+
line = None
186+
if line is None:
187+
line = ''
188+
cut = False
189+
text = piece[:-1] if piece.endswith('\n') else piece
190+
room = MAX_DISPLAY_LINE_SIZE - len(line)
191+
if len(text) > room:
192+
cut = True
193+
line += text[:room]
194+
if piece.endswith('\n'):
195+
yield line, '\n', cut
196+
line = None
197+
if line is not None:
198+
yield line, '', cut
199+
200+
201+
def display_content(pieces: Iterable[Tuple[str, bool]]) -> Dict[str, Any]:
202+
'''The ``content`` of a display payload, from ``(piece, at_line_start)``
203+
pieces (as ``edgutil.iter_line_pieces`` yields them), and as needed
204+
``truncated`` and ``lines_truncated``:
205+
206+
- Each line is cut to ``MAX_DISPLAY_LINE_SIZE`` characters, its 1-based
207+
number listed in ``lines_truncated``.
208+
- The whole is cut to ``MAX_DISPLAY_CONTENT_SIZE``, after the last line
209+
that fits (setting ``truncated``); no more pieces are taken then, so
210+
what they come from is read no further. (Closing ``pieces``, should it
211+
be a generator with files open, is left to the caller.)
212+
213+
Neither key is set when nothing was cut, and nothing marks a cut in the
214+
content itself: the UI does.
215+
'''
216+
kept: List[str] = []
217+
size = 0
218+
lines_truncated: List[int] = []
219+
truncated = False
220+
for line_number, (text, line_end, cut) in enumerate(
221+
_display_lines(pieces),
222+
start = 1
223+
):
224+
num_chars_for_line = len(text) + len(line_end)
225+
if size + num_chars_for_line > MAX_DISPLAY_CONTENT_SIZE:
226+
truncated = True
227+
if not kept:
228+
# The first line alone doesn't fit (MAX_DISPLAY_LINE_SIZE is
229+
# larger than MAX_DISPLAY_CONTENT_SIZE): keep what does of it, as
230+
# a cut line.
231+
kept.append(text[:MAX_DISPLAY_CONTENT_SIZE])
232+
lines_truncated.append(line_number)
233+
break
234+
kept.append(text + line_end)
235+
size += num_chars_for_line
236+
if cut:
237+
lines_truncated.append(line_number)
238+
239+
content = ''.join(kept)
240+
if truncated and content.endswith('\n'):
241+
# Without the end of the last line kept.
242+
content = content[:-1]
243+
result: Dict[str, Any] = { 'content': content }
244+
if truncated:
245+
result['truncated'] = True
246+
if lines_truncated:
247+
result['lines_truncated'] = lines_truncated
248+
return result
249+
250+
251+
def display_text(handle: IO[str]) -> Dict[str, Any]:
252+
'''``display_content`` of the text ``handle`` reads: only as much as is
253+
shown, and of a line, at most a character more.
254+
'''
255+
return display_content(
256+
edgutil.iter_line_pieces(handle, MAX_DISPLAY_LINE_SIZE + 1)
257+
)
258+
259+
260+
def trunc_display_content(content: str) -> Dict[str, Any]:
261+
'''``display_content`` of ``content``, already in memory.'''
262+
return display_text(io.StringIO(content))
178263

179264

180265
def reviewer_of(request: Any) -> Optional[str]:

0 commit comments

Comments
 (0)