Skip to content

Commit 8ac4894

Browse files
authored
[executorch][native] Add the .ptn package format
Differential Revision: D116231152 Pull Request resolved: pytorch#22007
1 parent 2aac1f5 commit 8ac4894

4 files changed

Lines changed: 647 additions & 0 deletions

File tree

‎exir/_serialize/BUCK‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,21 @@ fbcode_target(_kind = runtime.cxx_python_extension,
2323
)
2424

2525
# Use runtime.python_library instead of the one defined in python_library.bzl,
26+
# The .ptn format. Its own target so safetensors stays out of :lib, and therefore
27+
# out of everything that depends on exir.
28+
fbcode_target(_kind = runtime.python_library,
29+
name = "ptn",
30+
typing = True,
31+
srcs = [
32+
"_ptn.py",
33+
],
34+
visibility = ["PUBLIC"],
35+
deps = [
36+
"fbsource//third-party/pypi/safetensors:safetensors",
37+
"//caffe2:torch",
38+
],
39+
)
40+
2641
# so we can have access to EXECUTORCH_CLIENTS list.
2742
fbcode_target(_kind = runtime.python_library,
2843
name = "lib",

‎exir/_serialize/_ptn.py‎

Lines changed: 321 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,321 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
# pyre-strict
8+
9+
"""
10+
The .ptn package format.
11+
12+
A .ptn is an uncompressed zip holding a native Program flatbuffer and, when the
13+
program references constants, those tensors as safetensors. Member names are fixed,
14+
so renaming the package does not break it:
15+
16+
program.ptg native Program flatbuffer (file identifier "NPTG")
17+
program.safetensors referenced constants, content-deduped (owners only)
18+
aliases.json duplicate data_key -> owner key, only when duplicates exist
19+
20+
Byte-identical *immutable* tensors (same dtype, shape, and content) are stored
21+
once: the first key owns the safetensors entry and the rest alias to it. An owner
22+
is always a real safetensors key and never appears in the alias map, so resolution
23+
is ``owner = aliases.get(key, key)``.
24+
25+
An alias means only "these keys had identical bytes at save time, so one copy was
26+
stored". It never asserts that two keys share runtime state, which is why mutable
27+
keys are excluded from dedup entirely -- see _dedup. Do not overload aliases to
28+
express mutable alias topology; that would need explicit storage groups in the
29+
format.
30+
31+
Sits alongside the PTE and PTD serializers here because the format is independent
32+
of any backend: write_ptn treats the graph blob as opaque bytes. Its producer is
33+
``exir.native.to_native``, whose NativeProgramManager.save calls it.
34+
35+
The safetensors payload is written a tensor at a time, straight into the zip entry,
36+
so the writer never materializes the whole model as Python bytes. Owner tensors are
37+
still retained until the write completes, and normalizing non-CPU or non-contiguous
38+
inputs can allocate tensor copies. The header is emitted here by hand because
39+
safetensors.torch.save_file goes through _flatten, which converts every tensor to
40+
bytes before writing anything. Reading goes through the real library, so the tests
41+
validate this writer against it.
42+
"""
43+
44+
from __future__ import annotations
45+
46+
import hashlib
47+
import json
48+
import os
49+
import zipfile
50+
from typing import IO
51+
52+
import torch
53+
54+
PTG_ENTRY = "program.ptg"
55+
SAFETENSORS_ENTRY = "program.safetensors"
56+
ALIASES_ENTRY = "aliases.json"
57+
_SAFETENSORS_METADATA_KEY = "__metadata__"
58+
59+
# safetensors dtype codes; mirrors safetensors.torch._TYPES inverted.
60+
_DTYPE_CODES: dict[torch.dtype, str] = {
61+
torch.float64: "F64",
62+
torch.float32: "F32",
63+
torch.float16: "F16",
64+
torch.bfloat16: "BF16",
65+
torch.int64: "I64",
66+
torch.int32: "I32",
67+
torch.int16: "I16",
68+
torch.int8: "I8",
69+
torch.uint8: "U8",
70+
torch.bool: "BOOL",
71+
torch.complex64: "C64",
72+
}
73+
for _name, _code in (("uint16", "U16"), ("uint32", "U32"), ("uint64", "U64")):
74+
_dtype = getattr(torch, _name, None)
75+
if _dtype is not None:
76+
_DTYPE_CODES[_dtype] = _code
77+
78+
79+
def _raw(tensor: torch.Tensor) -> memoryview:
80+
"""A zero-copy byte view of a contiguous CPU tensor.
81+
82+
Reinterpreting as uint8 keeps dtypes numpy cannot express, such as bfloat16,
83+
working; neither the view nor .numpy() copies.
84+
"""
85+
return tensor.flatten().view(torch.uint8).numpy().data
86+
87+
88+
def _content_signature(tensor: torch.Tensor) -> tuple[str, tuple[int, ...], str]:
89+
digest = hashlib.sha256()
90+
digest.update(_raw(tensor))
91+
return (str(tensor.dtype), tuple(tensor.shape), digest.hexdigest())
92+
93+
94+
def _check_mutable_storage_aliases(
95+
constants: dict[str, torch.Tensor], mutable_keys: frozenset[str]
96+
) -> None:
97+
"""Reject alias topology that the current PTN data model cannot express.
98+
99+
Immutable views may be materialized independently because their storage
100+
identity is not observable. If either of two distinct keys is mutable,
101+
separating shared source storage would change program semantics. The current
102+
format has no storage/view manifest, so fail rather than silently sever that
103+
alias.
104+
"""
105+
keys_by_storage: dict[tuple[str, int, int], list[str]] = {}
106+
for key, tensor in constants.items():
107+
# All empty tensors have an unobservable zero-byte range and commonly
108+
# report data_ptr() == 0 even when their storages are unrelated.
109+
if tensor.numel() == 0:
110+
continue
111+
device_index = tensor.device.index if tensor.device.index is not None else -1
112+
storage = (
113+
tensor.device.type,
114+
device_index,
115+
tensor.untyped_storage().data_ptr(),
116+
)
117+
keys_by_storage.setdefault(storage, []).append(key)
118+
119+
for keys in keys_by_storage.values():
120+
if len(keys) > 1 and any(key in mutable_keys for key in keys):
121+
names = sorted(keys)
122+
raise ValueError(
123+
"write_ptn: distinct data keys share source storage and at least "
124+
f"one is mutable: {names}. The current PTN format cannot preserve "
125+
"mutable alias topology; use one data key for one logical state "
126+
"object."
127+
)
128+
129+
130+
def _dedup(
131+
constants: dict[str, torch.Tensor],
132+
mutable_keys: frozenset[str],
133+
) -> tuple[dict[str, torch.Tensor], dict[str, str]]:
134+
"""Split constants into safetensors owners and a duplicate -> owner alias map.
135+
136+
Only immutable keys are deduplicated. Equal bytes are not evidence that two
137+
mutable keys are the same runtime state: two independently mutable buffers can
138+
be zero-initialized to identical bytes and still need separate storage, so
139+
aliasing them would make a write to one visible through the other. A mutable
140+
key is therefore always its own owner, and never becomes an owner some other
141+
key aliases to.
142+
143+
Immutable tensors sharing storage need no special handling: each owner's bytes
144+
are written independently. Distinct keys sharing storage when either is
145+
mutable are rejected, because writing them independently would silently break
146+
runtime aliasing and the current PTN format cannot encode their storage
147+
topology.
148+
"""
149+
unknown_mutable = sorted(mutable_keys.difference(constants))
150+
if unknown_mutable:
151+
raise ValueError(
152+
"write_ptn: mutable_keys contains keys with no tensor data: "
153+
f"{unknown_mutable}"
154+
)
155+
156+
for key, tensor in constants.items():
157+
if not isinstance(key, str):
158+
raise TypeError(
159+
f"write_ptn: tensor data key must be str, got {type(key).__name__}"
160+
)
161+
if key == _SAFETENSORS_METADATA_KEY:
162+
raise ValueError(
163+
f"write_ptn: {_SAFETENSORS_METADATA_KEY!r} is reserved by "
164+
"safetensors and cannot be a tensor data key."
165+
)
166+
if not isinstance(tensor, torch.Tensor):
167+
raise TypeError(
168+
f"write_ptn: value for {key!r} must be a Tensor, got "
169+
f"{type(tensor).__name__}"
170+
)
171+
if tensor.dtype not in _DTYPE_CODES:
172+
# Preflight before opening the destination. This does not make the
173+
# complete save atomic, but predictable schema errors must not destroy
174+
# an existing valid artifact.
175+
raise KeyError(
176+
f"write_ptn: safetensors cannot represent dtype {tensor.dtype} "
177+
f"for data key {key!r}"
178+
)
179+
180+
_check_mutable_storage_aliases(constants, mutable_keys)
181+
owners: dict[str, torch.Tensor] = {}
182+
aliases: dict[str, str] = {}
183+
owner_by_content: dict[tuple[str, tuple[int, ...], str], str] = {}
184+
for key in sorted(constants):
185+
tensor = constants[key].detach().cpu().contiguous()
186+
if key in mutable_keys:
187+
owners[key] = tensor
188+
continue
189+
signature = _content_signature(tensor)
190+
owner = owner_by_content.get(signature)
191+
if owner is None:
192+
owner_by_content[signature] = key
193+
owners[key] = tensor
194+
else:
195+
aliases[key] = owner
196+
return owners, aliases
197+
198+
199+
def _write_safetensors(stream: IO[bytes], owners: dict[str, torch.Tensor]) -> None:
200+
"""Emit the safetensors format: header length, JSON header, then tensor data.
201+
202+
Offsets in the header are relative to the start of the data section, and the
203+
header is space-padded so that section begins 8-byte aligned.
204+
"""
205+
header: dict[str, object] = {}
206+
offset = 0
207+
for key, tensor in owners.items():
208+
size = tensor.numel() * tensor.element_size()
209+
header[key] = {
210+
"dtype": _DTYPE_CODES[tensor.dtype],
211+
"shape": list(tensor.shape),
212+
"data_offsets": [offset, offset + size],
213+
}
214+
offset += size
215+
216+
encoded = json.dumps(header, separators=(",", ":")).encode("utf-8")
217+
encoded += b" " * ((8 - len(encoded) % 8) % 8)
218+
219+
stream.write(len(encoded).to_bytes(8, "little"))
220+
stream.write(encoded)
221+
for tensor in owners.values():
222+
stream.write(_raw(tensor))
223+
224+
225+
def write_ptn(
226+
path: str,
227+
ptg: bytes,
228+
constants: dict[str, torch.Tensor],
229+
mutable_keys: frozenset[str] = frozenset(),
230+
) -> None:
231+
"""Write a .ptn package, streaming the constants a tensor at a time.
232+
233+
Args:
234+
path: Destination .ptn path. Member names inside are fixed, so the
235+
filename carries no meaning and the package survives a rename.
236+
ptg: Serialized native Program flatbuffer.
237+
constants: Every data key the program references, mapped to its tensor.
238+
mutable_keys: Keys whose runtime state is mutable. Excluded from content
239+
dedup, since equal bytes do not make two mutable buffers the same
240+
state. Passed in rather than derived here, so this stays independent of
241+
the graph format. Every mutable key must occur in ``constants``.
242+
243+
Raises:
244+
KeyError: If a constant has a dtype safetensors cannot represent.
245+
TypeError: If a data key is not a string or a value is not a tensor.
246+
ValueError: If mutable-key policy is inconsistent or a data key is reserved.
247+
"""
248+
# TODO: save atomically. This opens the destination directly, so a failure
249+
# part way through truncates an existing valid package. Write to a temporary
250+
# file beside the destination and os.replace it once complete.
251+
owners, aliases = _dedup(constants, mutable_keys)
252+
parent = os.path.dirname(path)
253+
if parent:
254+
os.makedirs(parent, exist_ok=True)
255+
256+
# ZIP_STORED keeps entries byte-addressable; allowZip64 lifts the format's
257+
# 4GiB per-entry and total-size limits.
258+
with zipfile.ZipFile(path, "w", zipfile.ZIP_STORED, allowZip64=True) as pkg:
259+
pkg.writestr(PTG_ENTRY, ptg)
260+
if owners:
261+
with pkg.open(SAFETENSORS_ENTRY, "w") as stream:
262+
_write_safetensors(stream, owners)
263+
if aliases:
264+
pkg.writestr(
265+
ALIASES_ENTRY, json.dumps(aliases, indent=2, sort_keys=True)
266+
)
267+
268+
269+
def read_ptn(path: str) -> tuple[bytes, dict[str, torch.Tensor]]:
270+
"""Inverse of write_ptn, with aliases resolved back to their owner's tensor.
271+
272+
Returns every data key the program references, owners and aliases alike. Used
273+
by the tests to verify the round-trip; the lowering path only ever writes.
274+
275+
Raises:
276+
ValueError: If an alias names a key with no safetensors entry, or a key is
277+
both an owner and an alias.
278+
"""
279+
from safetensors.torch import load as safetensors_load
280+
281+
with zipfile.ZipFile(path) as pkg:
282+
entries = set(pkg.namelist())
283+
ptg = pkg.read(PTG_ENTRY)
284+
owners: dict[str, torch.Tensor] = (
285+
safetensors_load(pkg.read(SAFETENSORS_ENTRY))
286+
if SAFETENSORS_ENTRY in entries
287+
else {}
288+
)
289+
aliases: dict[str, str] = {}
290+
if ALIASES_ENTRY in entries:
291+
raw_aliases = json.loads(pkg.read(ALIASES_ENTRY))
292+
if not isinstance(raw_aliases, dict):
293+
raise ValueError("read_ptn: aliases.json must contain a JSON object.")
294+
for key, owner in raw_aliases.items():
295+
if not isinstance(key, str) or not isinstance(owner, str):
296+
raise ValueError(
297+
"read_ptn: every alias and owner name must be a string."
298+
)
299+
if key == _SAFETENSORS_METADATA_KEY:
300+
raise ValueError(
301+
f"read_ptn: alias key {_SAFETENSORS_METADATA_KEY!r} is "
302+
"reserved by safetensors."
303+
)
304+
aliases[key] = owner
305+
306+
constants = dict(owners)
307+
for key, owner in aliases.items():
308+
if owner not in owners:
309+
raise ValueError(
310+
f"read_ptn: alias {key!r} names owner {owner!r}, which has no "
311+
f"safetensors entry."
312+
)
313+
if key in owners:
314+
raise ValueError(
315+
f"read_ptn: {key!r} is both a safetensors owner and an alias."
316+
)
317+
constants[key] = owners[owner]
318+
return ptg, constants
319+
320+
321+
__all__ = ["ALIASES_ENTRY", "PTG_ENTRY", "SAFETENSORS_ENTRY", "read_ptn", "write_ptn"]

‎exir/_serialize/test/BUCK‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,3 +69,15 @@ fbcode_target(_kind = runtime.python_test,
6969
"//executorch/exir/tests:lib",
7070
],
7171
)
72+
73+
fbcode_target(_kind = runtime.python_test,
74+
name = "test_ptn",
75+
srcs = [
76+
"test_ptn.py",
77+
],
78+
deps = [
79+
"fbsource//third-party/pypi/safetensors:safetensors",
80+
"//caffe2:torch",
81+
"//executorch/exir/_serialize:ptn",
82+
],
83+
)

0 commit comments

Comments
 (0)