Skip to content

Commit 5331ced

Browse files
authored
security: bound and validate NumPy headers (#5)
* security: bound and validate numpy headers * fix: strip NUL padding in NPY headers * test: cover malformed and oversized NPY headers * test: run NPY header hardening regression * fix: encode NPY regression fixtures correctly
1 parent 189c365 commit 5331ced

2 files changed

Lines changed: 76 additions & 11 deletions

File tree

‎modelhawk.py‎

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -283,20 +283,50 @@ def scan_yaml_bytes(data: bytes, label: str = "<yaml>") -> "StreamReport":
283283
# Numpy .npy scanning
284284
# --------------------------------------------------------------------------- #
285285

286+
# Reject hostile or corrupt headers before decoding/parsing them. NumPy's
287+
# format uses a small Python-literal dictionary; a 1 MiB ceiling is generous
288+
# for real headers while bounding work on untrusted artifacts.
289+
_MAX_NPY_HEADER_BYTES = 1 << 20
290+
291+
286292
def _parse_npy_header(data: bytes) -> tuple[str, int]:
287-
"""Return (dtype_str, data_start_offset). Raises ValueError on bad files."""
288-
if not data.startswith(_NPY_MAGIC):
289-
raise ValueError("missing numpy .npy magic bytes")
290-
major = data[6]
293+
"""Return (dtype_str, data_start_offset); reject malformed/oversized headers."""
294+
if len(data) < 8 or not data.startswith(_NPY_MAGIC):
295+
raise ValueError("missing or truncated numpy .npy magic/version bytes")
296+
297+
major, minor = data[6], data[7]
291298
if major == 1:
292-
header_len = int.from_bytes(data[8:10], "little")
293-
hdr_start, data_start = 10, 10 + header_len
294-
else: # v2+: 4-byte header length field
295-
header_len = int.from_bytes(data[8:12], "little")
296-
hdr_start, data_start = 12, 12 + header_len
299+
length_start, length_size, hdr_start = 8, 2, 10
300+
encoding = "latin-1"
301+
elif major in (2, 3):
302+
length_start, length_size, hdr_start = 8, 4, 12
303+
encoding = "utf-8" if major == 3 else "latin-1"
304+
else:
305+
raise ValueError(f"unsupported numpy .npy format version {major}.{minor}")
306+
307+
if len(data) < hdr_start:
308+
raise ValueError("truncated numpy .npy header-length field")
309+
310+
header_len = int.from_bytes(data[length_start:length_start + length_size], "little")
311+
if header_len <= 0:
312+
raise ValueError("empty numpy .npy header")
313+
if header_len > _MAX_NPY_HEADER_BYTES:
314+
raise ValueError(
315+
f"numpy .npy header exceeds {_MAX_NPY_HEADER_BYTES} byte safety limit"
316+
)
317+
318+
data_start = hdr_start + header_len
319+
if data_start > len(data):
320+
raise ValueError("truncated numpy .npy header body")
321+
297322
raw_hdr = data[hdr_start:data_start]
298-
header = ast.literal_eval(raw_hdr.decode("latin-1").strip().rstrip("\x00").rstrip())
299-
return header.get("descr", ""), data_start
323+
try:
324+
header = ast.literal_eval(raw_hdr.decode(encoding).strip().rstrip("\x00").rstrip())
325+
except (UnicodeDecodeError, SyntaxError, ValueError) as exc:
326+
raise ValueError(f"invalid numpy .npy header: {exc}") from exc
327+
if not isinstance(header, dict) or not isinstance(header.get("descr"), str):
328+
raise ValueError("numpy .npy header must contain a string 'descr' field")
329+
return header["descr"], data_start
300330

301331

302332
def _npy_extract_pickle(data: bytes):

‎tests/test_modelhawk.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,40 @@ def test_npy_float_array_safe():
168168
assert stream.severity == "SAFE", f"expected SAFE, got {stream.severity}"
169169

170170

171+
172+
def test_npy_header_bounds_and_versions():
173+
"""Reject truncated/oversized headers and parse supported v3 UTF-8 headers."""
174+
import struct
175+
176+
# Truncated version and length fields must fail closed.
177+
for data in (b"\x93NUMPY", b"\x93NUMPY\x01\x00\x05"):
178+
try:
179+
modelhawk._parse_npy_header(data)
180+
except ValueError:
181+
pass
182+
else:
183+
raise AssertionError(f"truncated header accepted: {data!r}")
184+
185+
# An attacker-controlled length must be rejected before header parsing.
186+
oversized = (
187+
b"\x93NUMPY\x02\x00"
188+
+ struct.pack("<I", modelhawk._MAX_NPY_HEADER_BYTES + 1)
189+
)
190+
try:
191+
modelhawk._parse_npy_header(oversized)
192+
except ValueError as exc:
193+
assert "safety limit" in str(exc), str(exc)
194+
else:
195+
raise AssertionError("oversized NPY header accepted")
196+
197+
# Version 3 headers use UTF-8; a valid small header must still parse.
198+
header = b"{'descr': '<f4', 'fortran_order': False, 'shape': (1,), }\n"
199+
v3 = b"\x93NUMPY\x03\x00" + struct.pack("<I", len(header)) + header + b"\x00\x00\x80?"
200+
dtype, offset = modelhawk._parse_npy_header(v3)
201+
assert dtype == "<f4"
202+
assert v3[offset:] == b"\x00\x00\x80?"
203+
204+
171205
def test_npy_real_numpy_object_array():
172206
"""Use real numpy (if installed) to write a malicious object array and detect it."""
173207
try:
@@ -201,6 +235,7 @@ def __reduce__(self):
201235
test_yaml_detection,
202236
test_npy_object_array_handcrafted,
203237
test_npy_float_array_safe,
238+
test_npy_header_bounds_and_versions,
204239
test_npy_real_numpy_object_array,
205240
]
206241
failed = 0

0 commit comments

Comments
 (0)