Skip to content

Commit d2d0c25

Browse files
[executorch][native] Align tensor data in PTN packages (pytorch#23334)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: pytorch#23229 by @digantdesai ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/109/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/109/head Merge bot PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/108/orig Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/109/orig Differential Revision: [D121704318](https://our.internmc.facebook.com/intern/diff/D121704318/) @diff-train-skip-merge Co-authored-by: Digant Desai <digantdesai@meta.com>
1 parent dc2cff4 commit d2d0c25

2 files changed

Lines changed: 108 additions & 19 deletions

File tree

‎exir/_serialize/_ptn.py‎

Lines changed: 68 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@
2222
is always a real safetensors key and never appears in the alias map, so resolution
2323
is ``owner = aliases.get(key, key)``.
2424
25+
Tensor data is laid out so a reader can map the package and use constants in
26+
place: the data section of program.safetensors starts 64-byte aligned in the
27+
file, padded through a zip extra field, and each tensor starts at a multiple of
28+
its element size.
29+
2530
An alias means only "these keys had identical bytes at save time, so one copy was
2631
stored". It never asserts that two keys share runtime state, which is why mutable
2732
keys are excluded from dedup entirely -- see _dedup. Do not overload aliases to
@@ -46,8 +51,8 @@
4651
import hashlib
4752
import json
4853
import os
54+
import time
4955
import zipfile
50-
from typing import IO
5156

5257
import torch
5358

@@ -56,6 +61,17 @@
5661
ALIASES_ENTRY = "aliases.json"
5762
_SAFETENSORS_METADATA_KEY = "__metadata__"
5863

64+
# Tensor data starts on this boundary in the file, and each tensor is naturally
65+
# aligned within it, so a reader that maps the package can use constants in place.
66+
_DATA_ALIGNMENT = 64
67+
# Zip extra-field id for padding, as used by Android's zipalign: a u16 alignment
68+
# followed by zeros.
69+
_ALIGNMENT_EXTRA_ID = 0xD935
70+
# Fixed part of a zip local file header, and the zip64 extra field Python's
71+
# zipfile appends to it when force_zip64 is set.
72+
_LOCAL_HEADER_SIZE = 30
73+
_ZIP64_LOCAL_EXTRA_SIZE = 20
74+
5975
# safetensors dtype codes; mirrors safetensors.torch._TYPES inverted.
6076
_DTYPE_CODES: dict[torch.dtype, str] = {
6177
torch.float64: "F64",
@@ -196,30 +212,64 @@ def _dedup(
196212
return owners, aliases
197213

198214

199-
def _write_safetensors(stream: IO[bytes], owners: dict[str, torch.Tensor]) -> None:
200-
"""Emit the safetensors format: header length, JSON header, then tensor data.
215+
def _safetensors_header(owners: dict[str, torch.Tensor]) -> tuple[bytes, list[str]]:
216+
"""Encode the safetensors length prefix and JSON header.
201217
202-
Offsets in the header are relative to the start of the data section, and the
203-
header is space-padded so that section begins 8-byte aligned.
218+
Returns the encoded bytes and the order to write tensor data in. Data is laid
219+
out widest element first, so with the data section aligned every tensor starts
220+
at a multiple of its element size without leaving holes, which safetensors
221+
forbids. The header is space-padded so the data section begins
222+
_DATA_ALIGNMENT-aligned relative to the start of the member.
204223
"""
205-
header: dict[str, object] = {}
224+
order = sorted(owners, key=lambda key: -owners[key].element_size())
225+
offsets: dict[str, tuple[int, int]] = {}
206226
offset = 0
207-
for key, tensor in owners.items():
227+
for key in order:
228+
tensor = owners[key]
208229
size = tensor.numel() * tensor.element_size()
209-
header[key] = {
230+
offsets[key] = (offset, offset + size)
231+
offset += size
232+
233+
header = {
234+
key: {
210235
"dtype": _DTYPE_CODES[tensor.dtype],
211236
"shape": list(tensor.shape),
212-
"data_offsets": [offset, offset + size],
237+
"data_offsets": list(offsets[key]),
213238
}
214-
offset += size
215-
239+
for key, tensor in owners.items()
240+
}
216241
encoded = json.dumps(header, separators=(",", ":")).encode("utf-8")
217-
encoded += b" " * ((8 - len(encoded) % 8) % 8)
218-
219-
stream.write(len(encoded).to_bytes(8, "little"))
220-
stream.write(encoded)
221-
for tensor in owners.values():
222-
stream.write(_raw(tensor))
242+
encoded += b" " * ((-8 - len(encoded)) % _DATA_ALIGNMENT)
243+
return len(encoded).to_bytes(8, "little") + encoded, order
244+
245+
246+
def _write_safetensors(pkg: zipfile.ZipFile, owners: dict[str, torch.Tensor]) -> None:
247+
"""Stream the safetensors member into `pkg` with its data section aligned."""
248+
header, order = _safetensors_header(owners)
249+
size = len(header) + sum(t.numel() * t.element_size() for t in owners.values())
250+
zip64 = size > zipfile.ZIP64_LIMIT
251+
252+
zinfo = zipfile.ZipInfo(SAFETENSORS_ENTRY, time.localtime(time.time())[:6])
253+
local_header_size = (
254+
_LOCAL_HEADER_SIZE
255+
+ len(SAFETENSORS_ENTRY.encode("utf-8"))
256+
+ (_ZIP64_LOCAL_EXTRA_SIZE if zip64 else 0)
257+
)
258+
assert pkg.fp is not None
259+
# The padding field needs 6 bytes of its own: id, length and the u16.
260+
data_start = pkg.fp.tell() + local_header_size + 6
261+
padding = (-data_start) % _DATA_ALIGNMENT
262+
zinfo.extra = (
263+
_ALIGNMENT_EXTRA_ID.to_bytes(2, "little")
264+
+ (2 + padding).to_bytes(2, "little")
265+
+ _DATA_ALIGNMENT.to_bytes(2, "little")
266+
+ bytes(padding)
267+
)
268+
269+
with pkg.open(zinfo, "w", force_zip64=zip64) as stream:
270+
stream.write(header)
271+
for key in order:
272+
stream.write(_raw(owners[key]))
223273

224274

225275
def write_ptn(
@@ -258,8 +308,7 @@ def write_ptn(
258308
with zipfile.ZipFile(path, "w", zipfile.ZIP_STORED, allowZip64=True) as pkg:
259309
pkg.writestr(PTG_ENTRY, ptg)
260310
if owners:
261-
with pkg.open(SAFETENSORS_ENTRY, "w") as stream:
262-
_write_safetensors(stream, owners)
311+
_write_safetensors(pkg, owners)
263312
if aliases:
264313
pkg.writestr(
265314
ALIASES_ENTRY, json.dumps(aliases, indent=2, sort_keys=True)

‎exir/_serialize/test/test_ptn.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
import json
88
import os
9+
import struct
910
import tempfile
1011
import unittest
1112
import zipfile
@@ -35,6 +36,17 @@ def _read_entry(path: str, entry: str) -> bytes:
3536
return pkg.read(entry)
3637

3738

39+
def _member_data_offset(path: str, entry: str) -> int:
40+
"""File offset of a stored member's data, read from its local header."""
41+
with zipfile.ZipFile(path) as pkg:
42+
header_offset = pkg.getinfo(entry).header_offset
43+
with open(path, "rb") as f:
44+
f.seek(header_offset)
45+
local = f.read(30)
46+
name_size, extra_size = struct.unpack("<HH", local[26:30])
47+
return header_offset + 30 + name_size + extra_size
48+
49+
3850
def _rewrite(path: str, overrides: dict[str, bytes]) -> None:
3951
"""Rebuild a package in place, replacing or adding the named entries."""
4052
with zipfile.ZipFile(path) as src:
@@ -252,6 +264,34 @@ def test_multiple_tensors_read_back_at_correct_offsets(self):
252264
for key, tensor in constants.items():
253265
self.assertTrue(torch.equal(out[key], tensor), f"{key} mismatched")
254266

267+
def test_tensor_data_is_aligned_in_the_file(self):
268+
# Mixed element sizes with odd counts, so packing in key order would leave
269+
# the wider tensors misaligned.
270+
constants = {
271+
"a": torch.arange(3, dtype=torch.int8),
272+
"b": torch.arange(5, dtype=torch.float16),
273+
"c": torch.arange(7, dtype=torch.float32),
274+
"d": torch.arange(3, dtype=torch.int64),
275+
}
276+
# Vary the graph size so the safetensors member starts at every offset
277+
# modulo the alignment.
278+
for ptg_size in range(64):
279+
with self.subTest(ptg_size=ptg_size):
280+
write_ptn(self.path, b"x" * ptg_size, constants)
281+
member = _member_data_offset(self.path, SAFETENSORS_ENTRY)
282+
safetensors = _read_entry(self.path, SAFETENSORS_ENTRY)
283+
header_size = int.from_bytes(safetensors[:8], "little")
284+
data = member + 8 + header_size
285+
self.assertEqual(data % 64, 0)
286+
287+
header = json.loads(safetensors[8 : 8 + header_size])
288+
for key, tensor in constants.items():
289+
begin = header[key]["data_offsets"][0]
290+
self.assertEqual((data + begin) % tensor.element_size(), 0, key)
291+
_, out = read_ptn(self.path)
292+
for key, tensor in constants.items():
293+
self.assertTrue(torch.equal(out[key], tensor), key)
294+
255295
def test_member_names_are_fixed_so_renaming_is_safe(self):
256296
write_ptn(self.path, _PTG, {"a": torch.zeros(2)})
257297
self.assertEqual(_entries(self.path), {PTG_ENTRY, SAFETENSORS_ENTRY})

0 commit comments

Comments
 (0)