|
22 | 22 | is always a real safetensors key and never appears in the alias map, so resolution |
23 | 23 | is ``owner = aliases.get(key, key)``. |
24 | 24 |
|
| 25 | +Tensor data is laid out so a reader can map the package and use constants in |
| 26 | +place: the data section of program.safetensors starts 64-byte aligned in the |
| 27 | +file, padded through a zip extra field, and each tensor starts at a multiple of |
| 28 | +its element size. |
| 29 | +
|
25 | 30 | An alias means only "these keys had identical bytes at save time, so one copy was |
26 | 31 | stored". It never asserts that two keys share runtime state, which is why mutable |
27 | 32 | keys are excluded from dedup entirely -- see _dedup. Do not overload aliases to |
|
46 | 51 | import hashlib |
47 | 52 | import json |
48 | 53 | import os |
| 54 | +import time |
49 | 55 | import zipfile |
50 | | -from typing import IO |
51 | 56 |
|
52 | 57 | import torch |
53 | 58 |
|
|
56 | 61 | ALIASES_ENTRY = "aliases.json" |
57 | 62 | _SAFETENSORS_METADATA_KEY = "__metadata__" |
58 | 63 |
|
| 64 | +# Tensor data starts on this boundary in the file, and each tensor is naturally |
| 65 | +# aligned within it, so a reader that maps the package can use constants in place. |
| 66 | +_DATA_ALIGNMENT = 64 |
| 67 | +# Zip extra-field id for padding, as used by Android's zipalign: a u16 alignment |
| 68 | +# followed by zeros. |
| 69 | +_ALIGNMENT_EXTRA_ID = 0xD935 |
| 70 | +# Fixed part of a zip local file header, and the zip64 extra field Python's |
| 71 | +# zipfile appends to it when force_zip64 is set. |
| 72 | +_LOCAL_HEADER_SIZE = 30 |
| 73 | +_ZIP64_LOCAL_EXTRA_SIZE = 20 |
| 74 | + |
59 | 75 | # safetensors dtype codes; mirrors safetensors.torch._TYPES inverted. |
60 | 76 | _DTYPE_CODES: dict[torch.dtype, str] = { |
61 | 77 | torch.float64: "F64", |
@@ -196,30 +212,64 @@ def _dedup( |
196 | 212 | return owners, aliases |
197 | 213 |
|
198 | 214 |
|
199 | | -def _write_safetensors(stream: IO[bytes], owners: dict[str, torch.Tensor]) -> None: |
200 | | - """Emit the safetensors format: header length, JSON header, then tensor data. |
| 215 | +def _safetensors_header(owners: dict[str, torch.Tensor]) -> tuple[bytes, list[str]]: |
| 216 | + """Encode the safetensors length prefix and JSON header. |
201 | 217 |
|
202 | | - Offsets in the header are relative to the start of the data section, and the |
203 | | - header is space-padded so that section begins 8-byte aligned. |
| 218 | + Returns the encoded bytes and the order to write tensor data in. Data is laid |
| 219 | + out widest element first, so with the data section aligned every tensor starts |
| 220 | + at a multiple of its element size without leaving holes, which safetensors |
| 221 | + forbids. The header is space-padded so the data section begins |
| 222 | + _DATA_ALIGNMENT-aligned relative to the start of the member. |
204 | 223 | """ |
205 | | - header: dict[str, object] = {} |
| 224 | + order = sorted(owners, key=lambda key: -owners[key].element_size()) |
| 225 | + offsets: dict[str, tuple[int, int]] = {} |
206 | 226 | offset = 0 |
207 | | - for key, tensor in owners.items(): |
| 227 | + for key in order: |
| 228 | + tensor = owners[key] |
208 | 229 | size = tensor.numel() * tensor.element_size() |
209 | | - header[key] = { |
| 230 | + offsets[key] = (offset, offset + size) |
| 231 | + offset += size |
| 232 | + |
| 233 | + header = { |
| 234 | + key: { |
210 | 235 | "dtype": _DTYPE_CODES[tensor.dtype], |
211 | 236 | "shape": list(tensor.shape), |
212 | | - "data_offsets": [offset, offset + size], |
| 237 | + "data_offsets": list(offsets[key]), |
213 | 238 | } |
214 | | - offset += size |
215 | | - |
| 239 | + for key, tensor in owners.items() |
| 240 | + } |
216 | 241 | encoded = json.dumps(header, separators=(",", ":")).encode("utf-8") |
217 | | - encoded += b" " * ((8 - len(encoded) % 8) % 8) |
218 | | - |
219 | | - stream.write(len(encoded).to_bytes(8, "little")) |
220 | | - stream.write(encoded) |
221 | | - for tensor in owners.values(): |
222 | | - stream.write(_raw(tensor)) |
| 242 | + encoded += b" " * ((-8 - len(encoded)) % _DATA_ALIGNMENT) |
| 243 | + return len(encoded).to_bytes(8, "little") + encoded, order |
| 244 | + |
| 245 | + |
| 246 | +def _write_safetensors(pkg: zipfile.ZipFile, owners: dict[str, torch.Tensor]) -> None: |
| 247 | + """Stream the safetensors member into `pkg` with its data section aligned.""" |
| 248 | + header, order = _safetensors_header(owners) |
| 249 | + size = len(header) + sum(t.numel() * t.element_size() for t in owners.values()) |
| 250 | + zip64 = size > zipfile.ZIP64_LIMIT |
| 251 | + |
| 252 | + zinfo = zipfile.ZipInfo(SAFETENSORS_ENTRY, time.localtime(time.time())[:6]) |
| 253 | + local_header_size = ( |
| 254 | + _LOCAL_HEADER_SIZE |
| 255 | + + len(SAFETENSORS_ENTRY.encode("utf-8")) |
| 256 | + + (_ZIP64_LOCAL_EXTRA_SIZE if zip64 else 0) |
| 257 | + ) |
| 258 | + assert pkg.fp is not None |
| 259 | + # The padding field needs 6 bytes of its own: id, length and the u16. |
| 260 | + data_start = pkg.fp.tell() + local_header_size + 6 |
| 261 | + padding = (-data_start) % _DATA_ALIGNMENT |
| 262 | + zinfo.extra = ( |
| 263 | + _ALIGNMENT_EXTRA_ID.to_bytes(2, "little") |
| 264 | + + (2 + padding).to_bytes(2, "little") |
| 265 | + + _DATA_ALIGNMENT.to_bytes(2, "little") |
| 266 | + + bytes(padding) |
| 267 | + ) |
| 268 | + |
| 269 | + with pkg.open(zinfo, "w", force_zip64=zip64) as stream: |
| 270 | + stream.write(header) |
| 271 | + for key in order: |
| 272 | + stream.write(_raw(owners[key])) |
223 | 273 |
|
224 | 274 |
|
225 | 275 | def write_ptn( |
@@ -258,8 +308,7 @@ def write_ptn( |
258 | 308 | with zipfile.ZipFile(path, "w", zipfile.ZIP_STORED, allowZip64=True) as pkg: |
259 | 309 | pkg.writestr(PTG_ENTRY, ptg) |
260 | 310 | if owners: |
261 | | - with pkg.open(SAFETENSORS_ENTRY, "w") as stream: |
262 | | - _write_safetensors(stream, owners) |
| 311 | + _write_safetensors(pkg, owners) |
263 | 312 | if aliases: |
264 | 313 | pkg.writestr( |
265 | 314 | ALIASES_ENTRY, json.dumps(aliases, indent=2, sort_keys=True) |
|
0 commit comments