Repository navigation
Expand file tree
/
Copy pathmissingGigabytes.ts
More file actions
1157 lines (1087 loc) · 54.6 KB
/
Copy pathmissingGigabytes.ts
File metadata and controls
1157 lines (1087 loc) · 54.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import { promises as fsp } from 'fs';
import { platform } from '../platform';
import { capabilityState } from '../platform/capabilities';
import { wholeDiskOf } from '../platform/macos/diskutil';
import { ELEVATE_HOW, getSnapshotAccounting } from './snapshotAccounting';
import { groupZombies, RESTART_REFUSES } from './zombieHandles';
import { storeOf } from './scanStore';
import { boundaryReport } from './boundaryReport';
import { formatBytes } from '../utils/formatBytes';
import { formatCount } from '../utils/formatCount';
import { separatorOf } from '../utils/pathSeparator';
import { scanAccessAdvice, granteeFor } from '../utils/accessAdvice';
import { lastFdaState, type FdaState } from '../utils/fdaState';
import type { ScanResult } from '../models/types';
import type { LogicalVolumeInfo, VolumeTopology } from '../platform/types';
import type { SnapshotAccounting } from './snapshotAccounting';
import type { ZombieReport } from './zombieHandles';
/**
* The Missing Gigabytes (Phase 5) — one accounting statement for a volume.
*
* The most common real complaint on macOS is that the numbers do not add up:
* the Finder says 188 GB used, a disk tool says 176 GB of files, and nothing
* explains the difference. Every piece needed to explain it is already
* computed somewhere in this app. This service puts them on one page and makes
* them balance — or names the gap.
*
* ── The shape of the answer ──
*
* A receipt. Every line is bytes, the lines sum to the volume's used space, and
* the residual has its own line called `unaccounted` rather than being quietly
* spread across the others to make the arithmetic look clean. §5.2 is explicit
* about this and it is the whole point of the feature: a statement that always
* balances because it was forced to is not evidence of anything.
*
* ── The invariant, stated once ──
*
* **sum(lines whose bytes are known) + unaccounted === volume used**
*
* exactly, in integer bytes, always. `assertBalances` proves it on every build
* and `tests/missingGigabytes.test.ts` asserts it directly. A line whose bytes
* cannot be measured contributes **nothing** to the sum and its real value is
* therefore sitting inside `unaccounted` — so `unaccounted` names those lines
* (`includes`) instead of letting the user assume the residual is a mystery.
*
* ── Why the reference number is the container, not the volume ──
*
* Measured on this Mac (macOS 15, APFS), not assumed:
*
* statfs('/') → 494.38 GB total, 305.61 GB free
* statfs('/System/Volumes/Data') → 494.38 GB total, 305.61 GB free (identical)
* diskutil info -plist / → APFSContainerSize 494384795648
* APFSContainerFree 305612984320
*
* Both mount points report the *container's* shared pool, and diskutil agrees
* with statfs to the byte. So "used" here means the container's used space —
* which is correct, because that is the pool everything on this disk draws
* from, and it is the number the Finder shows.
*
* It also means the sibling volumes are a real, large line. On this Mac the
* container holds Preboot (9.04 GB), VM (2.15 GB), Update, and an unmounted
* volume (1.30 GB) that no scan of `/` ever walks: ~12.5 GB that a
* files-only tool simply loses. Naming it is most of this feature's value.
*
* ── What cannot be measured here, and is therefore stated ──
*
* **Purgeable space.** macOS exposes it only through
* `NSURLVolumeAvailableCapacityForImportantUsageKey`, a native API. Checked,
* not assumed: `diskutil info -plist`, `diskutil apfs list -plist` and
* `system_profiler -json SPStorageDataType` were each read on this machine and
* none carries a purgeable figure. The native scan core does not read it (it
* lists folders and governs the scan, nothing else), so this line is
* unavailable with that reason, its bytes live inside `unaccounted`, and
* `unaccounted` says so. It never reads 0.
*/
export type StatementLineId =
| 'scanned'
| 'cloudPlaceholders'
| 'sparseFiles'
| 'snapshots'
| 'purgeable'
| 'openHandles'
| 'otherVolumes'
| 'mountedInside'
| 'unscannable'
| 'unaccounted';
/** A remedy the user can act on, always routed at an endpoint that already exists. */
export interface StatementRemedy {
/** Which existing gated action this is. The frontend maps it to its own flow. */
action: 'purge-snapshots' | 'review-open-handles' | 'scan-volume';
label: string;
/** Stated up front so the gate is never a surprise mid-flow. */
caveat: string;
}
/** One line of the receipt. */
export interface StatementLine {
id: StatementLineId;
label: string;
/**
* Bytes, signed — negative for a correction. **`null` means unknown**, and is
* never rendered as a zero. A line that is genuinely zero (no snapshots
* exist) carries `0`, which is a measurement, not an absence.
*/
bytes: number | null;
available: boolean;
/** Present only when `available` is false. Shown verbatim, never rewritten. */
reason?: string;
/** What lands on this line, in plain English. */
detail: string;
/** A count where one is meaningful — snapshots, refused folders, processes. */
count: number | null;
/**
* Facts that belong to this line but are not part of the arithmetic, because
* the scan already applied them. Hard links are the case §5.2 names: the
* walker zeroes every name after the first, so the tree total already counts
* each inode once and correcting again here would subtract them twice.
*/
notes: string[];
remedy: StatementRemedy | null;
}
export interface StatementVolume {
mountPoint: string;
/** Container total and used, in bytes. */
totalBytes: number;
usedBytes: number;
freeBytes: number;
/** How the figure was obtained, for the footnote. */
mechanism: string;
/**
* The same fact in the user's words — "what macOS reports for Macintosh HD"
* — with no call name and no mount path. `mechanism` stays for the API and
* the tests; this is what the page prints.
*/
mechanismLabel: string;
/**
* Blocks only root may use (ext4's 5% reserve): total − used − free. Zero
* on APFS and Windows. Reported so used + free + reserved is the disk, and
* the dashboard's "used" and this statement's "used" can be the same figure.
*/
reservedBytes: number;
}
export interface AccountingStatement {
scanId: string;
rootPath: string;
volume: StatementVolume;
lines: StatementLine[];
/**
* The residual, signed. Positive: space the volume holds that no line
* explained. Negative: TreeMap counted more than the volume says exists,
* which shared storage (clones, reflinks) produces and which is stated as
* such rather than clamped to zero.
*/
unaccountedBytes: number;
/**
* Did the scan cover this whole volume? When false the residual necessarily
* contains everything outside the scanned folder, and the UI leads with that
* rather than presenting a mystery.
*/
coversWholeVolume: boolean;
/** Limitations that apply to the statement as a whole. Rendered verbatim. */
caveats: string[];
generatedAt: number;
}
/**
* Everything the statement reads from outside itself.
*
* Injected rather than imported directly so the arithmetic can be proven
* against fixtures for machines this one is not — a volume with snapshots, a
* machine whose backup tool is missing, a statement that does not balance.
* On this Mac only one of the interesting cases is reachable live, and a
* receipt whose only proof is "it added up on the author's laptop" is not
* proof of the invariant.
*/
export interface StatementSources {
platform: NodeJS.Platform;
topology(): Promise<VolumeTopology>;
statfs(mountPoint: string): Promise<{ blocks: number; bfree: number; bavail: number; bsize: number }>;
/** `stat().dev`, or null when the path cannot be read. */
devOf(target: string): Promise<number | null>;
/** `mountPoint` lets Windows scope its restore points to the volume in question. */
snapshots(mountPoint?: string): Promise<SnapshotAccounting>;
zombies(): Promise<ZombieReport>;
}
/** The real machine. */
export function liveSources(): StatementSources {
return {
platform: process.platform,
topology: () => platform().getVolumeTopology(),
async statfs(mountPoint: string) {
const st = await fsp.statfs(mountPoint);
return { blocks: Number(st.blocks), bfree: Number(st.bfree), bavail: Number(st.bavail), bsize: Number(st.bsize) };
},
devOf,
snapshots: (mountPoint?: string) => getSnapshotAccounting(mountPoint),
zombies: leanZombieReport,
};
}
/**
* The held-space figures, without the part this page does not show.
*
* `zombieReport()` finishes by resolving each holder's `.app` bundle. That
* cost **one `ps` per process** when this was written — 305 processes were
* holding a deleted file on this machine, so 305 subprocesses for a field the
* receipt never prints (measured: 568 ms with the enrichment, 226 ms without,
* and the three numbers this line actually uses — total bytes, process count,
* unknown-size count — came back identical both times). Track U's U-5 made it
* one `ps` for up to 500 processes; it is still a subprocess for nothing here.
*
* The Held-space view still calls the full `zombieReport()`, because it names
* the applications and genuinely needs them.
*
* Gated as GET /api/zombie-handles is (BQL item 11): where no pass can run —
* Windows, a Mac without lsof — the provider answers an empty list, and the
* line read that as a measured "nothing held". The probe's reason makes the
* line unknown instead.
*/
async function leanZombieReport(): Promise<ZombieReport> {
const state = await capabilityState('zombieHandles');
if (!state.available) throw new Error(state.reason ?? 'Held-space detection is not available on this system.');
const scan = await platform().getZombieHandleScan();
return { ...groupZombies(scan.handles, scan.undecided, scan.unreadablePrograms), scannedAt: Date.now() };
}
/**
* Which volumes the scan's own filesystem covers, and which one to name.
*
* **Matched by device id, never by path prefix.** This is the macOS firmlink
* trap, and it is not subtle: `/Users` is a firmlink onto the writable volume,
* so a path like `/Users/me/Desktop` is under the mount point `/` by string
* comparison while physically living on `/System/Volumes/Data`. Prefix matching
* therefore attributes a home-folder scan to the sealed 12 GB system volume,
* and the 163 GB volume the files are actually on is then booked as somebody
* else's — which is exactly the wrong answer, off by the size of the disk.
*
* Measured on this Mac:
*
* stat('/').dev = 16777233
* stat('/System/Volumes/Data').dev = 16777233 (same)
* stat('/Users/me/…').dev = 16777233 (same)
* stat('/System/Volumes/VM').dev = 16777232 (different)
*
* The kernel presents the firmlinked system+data pair as **one** device, which
* is precisely the unit a scan of `/` walks. So grouping mount points by device
* id gets firmlinks right without naming a single one of them — and it stays
* right on Linux and Windows, where each filesystem simply has its own device
* and the group is a single volume.
*/
export interface ScanVolume {
/** The volume to name in the statement — the outermost mount point on this device. */
primary: LogicalVolumeInfo;
/** Every mounted volume on the same device, including `primary`. */
onSameDevice: LogicalVolumeInfo[];
}
/** `stat().dev`, or null when the path cannot be read. */
async function devOf(target: string): Promise<number | null> {
try {
return (await fsp.stat(target)).dev;
} catch {
return null;
}
}
export async function resolveScanVolume(
volumes: readonly LogicalVolumeInfo[],
rootPath: string,
readDev: (target: string) => Promise<number | null> = devOf,
): Promise<ScanVolume | null> {
const mounted = volumes.filter((v) => v.mountPoint !== null);
const rootDev = await readDev(rootPath);
if (rootDev !== null) {
const devs = await Promise.all(mounted.map((v) => readDev(v.mountPoint as string)));
const onSameDevice = mounted.filter((_, i) => devs[i] === rootDev);
if (onSameDevice.length > 0) {
// The outermost mount point is the one a user recognises as "the disk":
// `/` rather than `/System/Volumes/Data`, both of which are this device.
const primary = onSameDevice.reduce((a, b) =>
(a.mountPoint as string).length <= (b.mountPoint as string).length ? a : b,
);
return { primary, onSameDevice };
}
}
// Fallback: the device could not be read (a path that vanished, a permission
// wall on the mount point itself). Longest matching mount point is the best
// remaining guess, and it is only ever reached when the exact answer is gone.
const byPrefix = volumeForPath(mounted, rootPath);
return byPrefix ? { primary: byPrefix, onSameDevice: [byPrefix] } : null;
}
/** Which mounted volume does this path live on, by path alone? Longest mount point wins. */
export function volumeForPath(volumes: readonly LogicalVolumeInfo[], target: string): LogicalVolumeInfo | null {
let best: LogicalVolumeInfo | null = null;
for (const v of volumes) {
if (!v.mountPoint) continue;
if (!isUnder(target, v.mountPoint)) continue;
if (!best || v.mountPoint.length > (best.mountPoint ?? '').length) best = v;
}
return best;
}
/** Is `target` at or beneath `dir`? Segment-wise, so `/Volumes/Disk2` is not under `/Volumes/Disk`. */
function isUnder(target: string, dir: string): boolean {
if (target === dir) return true;
const sep = separatorOf(dir);
const base = dir.endsWith(sep) ? dir : dir + sep;
return target.startsWith(base);
}
/**
* The volumes sharing one storage pool with `volume`.
*
* **APFS only, and that is a correctness rule rather than a missing feature.**
* This line exists because `statfs` on an APFS volume reports the *container's*
* pool, so sibling volumes really are inside the total being reconciled. On
* ext4, NTFS and friends `statfs` reports that filesystem alone: adding a
* sibling's usage there would add bytes the total never contained, and the
* statement would fail to balance by exactly that amount. So elsewhere the
* answer is "nothing shares this pool", which is true of how the number was
* obtained.
*
* Container membership is read off the device identifier: `disk3s5` and
* `disk3s1s1` both live in container `disk3`, while `disk1s1` does not.
* `wholeDiskOf` is the mapping the topology reader already uses and tests, and
* it is the only field in `diskutil list`'s answer that distinguishes
* containers at all — every volume there reports the same *physical* disk,
* because one SSD backs all three containers on this machine.
*/
export function containerSiblings(
volumes: readonly LogicalVolumeInfo[],
volume: LogicalVolumeInfo,
plat: NodeJS.Platform = process.platform,
): LogicalVolumeInfo[] {
if (plat !== 'darwin') return [];
const container = wholeDiskOf(volume.id);
return volumes.filter((v) => v.id !== volume.id && wholeDiskOf(v.id) === container);
}
/**
* Build the statement.
*
* Every source is best-effort and independently degradable: a snapshot tool
* that is missing must not cost the user the other seven lines. Each `catch`
* therefore produces an unavailable line carrying the real error text — §10's
* "unavailable is a first-class state", applied per line rather than per page.
*/
export async function buildStatement(
scan: ScanResult,
sources: StatementSources = liveSources(),
): Promise<AccountingStatement> {
const rootPath = scan.rootPath;
const plat = sources.platform;
// The held-space probe (lsof over every open file: 200-370 ms measured) and
// the disk layout (170-270 ms) read nothing of each other's, so the probe
// starts first and runs while the layout and the snapshots are read: the
// statement waits for the slower, not for their sum (design pass G06).
// Nothing is remembered between statements — each line says what the
// machine says now. Its failure is held here until its own line reads it,
// so one that lands while the layout is being read is never unhandled.
const held = (async () => sources.zombies())();
held.catch(() => { /* reported as the openHandles line's reason */ });
const topology = await sources.topology();
const resolved = await resolveScanVolume(topology.logicalVolumes, rootPath, sources.devOf);
if (!resolved) {
throw new Error(
`no mounted volume contains ${rootPath} — the disk layout does not describe where this scan lives, so nothing can be reconciled against it`,
);
}
const volume = resolved.primary;
const mountPoint = volume.mountPoint as string;
// The pool everything on this container draws from. statfs is the reference
// because it is the same number the Finder shows and it agrees with diskutil
// to the byte (measured; see the header).
const st = await sources.statfs(mountPoint);
const blockSize = st.bsize;
const totalBytes = st.blocks * blockSize;
const freeBytes = st.bavail * blockSize;
const usedBytes = (st.blocks - st.bfree) * blockSize;
const reservedBytes = Math.max(0, (st.bfree - st.bavail) * blockSize);
// A scan covers the whole volume only when it started AT a mount point of the
// scan's own device. Anything deeper is a folder, however large.
const coversWholeVolume = resolved.onSameDevice.some((v) => v.mountPoint === rootPath);
const lines: StatementLine[] = [];
const caveats: string[] = [];
/* ── Scanned files ─────────────────────────────────────────────────────── */
const store = storeOf(scan);
const scannedBytes = store.size(store.rootId);
const hardlinked = scan.hardlinkedBytes ?? 0;
const scannedNotes: string[] = [];
if (hardlinked > 0) {
scannedNotes.push(
`Files here have more than one name, and ${formatBytes(hardlinked)} would be counted twice by a tool that adds every name up. The scan counts each file once, so that is already left out here — it is not a separate line.`,
);
}
// Plain words on a page a person reads (B19, as D41 did for Purgeable): the
// obstacle — no clone identity from the file system calls the scan makes —
// is recorded in src/platform/macos/allocation.ts, not on the page.
scannedNotes.push(
'Some copies share their storage with the original — a clone, which is what duplicating a file on a modern Mac makes. TreeMap cannot yet tell a clone from a real copy, so it counts each at its full size. Where clones exist, this line counts more than they take on the disk, and the difference shows up in Unaccounted below.',
);
lines.push({
id: 'scanned',
label: 'Files TreeMap scanned',
bytes: scannedBytes,
available: true,
detail: coversWholeVolume
? 'Everything the scan walked, which is this whole volume.'
: `Everything the scan walked. The scan started at ${rootPath}, which is only part of this volume.`,
count: scan.fileCount,
notes: scannedNotes,
remedy: coversWholeVolume
? null
: {
action: 'scan-volume',
label: `Scan all of ${mountPoint}`,
caveat: 'A full-volume scan takes longer, and is the only way this statement can account for every file.',
},
});
/* ── Cloud placeholders ────────────────────────────────────────────────── */
const cloudBytes = scan.cloudBytes ?? 0;
lines.push({
id: 'cloudPlaceholders',
label: 'Online-only files counted above but not on the disk',
bytes: -cloudBytes,
available: true,
detail:
cloudBytes > 0
? 'Cloud placeholders report their full size but occupy almost no blocks. The line above counted them; this takes them back off.'
: 'No cloud placeholders were found in this scan, so nothing is taken back off.',
count: scan.cloudFiles ?? 0,
notes: [],
remedy: null,
});
/* ── Space claimed but not occupied ────────────────────────────────────── */
lines.push(sparseLine(scan, plat));
/* ── Filesystem snapshots ──────────────────────────────────────────────── */
lines.push(await snapshotLine(sources, mountPoint));
/* ── Purgeable ─────────────────────────────────────────────────────────── */
lines.push(purgeableLine(plat));
/* ── Space held by open handles ────────────────────────────────────────── */
lines.push(await openHandleLine(held));
/* ── Other volumes in the same container ──────────────────────────────── */
lines.push(otherVolumesLine(topology.logicalVolumes, volume, resolved.onSameDevice, plat));
/* ── Drives mounted inside the scanned folder (Track U, U-28) ──────────── */
lines.push(mountedInsideLine(scan));
/* ── What the scan was refused ─────────────────────────────────────────── */
lines.push(unscannableLine(scan, plat));
/* ── The residual ──────────────────────────────────────────────────────── */
const known = lines.reduce((sum, l) => sum + (l.bytes ?? 0), 0);
const unaccountedBytes = usedBytes - known;
const unknownLines = lines.filter((l) => l.bytes === null);
const includes = unknownLines.map((l) => l.label);
if (!coversWholeVolume) includes.unshift(`everything on ${mountPoint} outside ${rootPath}`);
includes.push('space shared by copy-on-write clones, whose identity the scan does not read');
lines.push({
id: 'unaccounted',
label: 'Unaccounted',
bytes: unaccountedBytes,
available: true,
detail:
`The volume holds this much that no line above explained. It is shown rather than absorbed, because a statement ` +
`that always balances is not evidence of anything. What is sitting in here: ${includes.join('; ')}.`,
count: null,
notes: [],
remedy: null,
});
if (!coversWholeVolume) {
caveats.push(
`This scan started at ${rootPath}, not at ${mountPoint}, so everything else on the volume is inside Unaccounted. Scan the whole volume for a statement that accounts for every file.`,
);
}
if (plat !== 'darwin' && plat !== 'win32' && plat !== 'linux') {
caveats.push(`TreeMap has no per-volume accounting for ${plat}; only the filesystem's own totals are shown.`);
}
const statement: AccountingStatement = {
scanId: scan.scanId,
rootPath,
volume: {
mountPoint,
totalBytes,
usedBytes,
freeBytes,
mechanism: `statfs(${mountPoint}) — the shared pool this volume draws from`,
mechanismLabel: describeMechanism(volume, plat),
reservedBytes,
},
lines,
unaccountedBytes,
coversWholeVolume,
caveats,
generatedAt: Date.now(),
};
assertBalances(statement);
return statement;
}
/* ─────────────────────────── the individual lines ─────────────────────────── */
async function snapshotLine(sources: StatementSources, mountPoint: string): Promise<StatementLine> {
// Windows calls them restore points and the receipt says so; elsewhere they
// are the filesystem's snapshots.
const win = sources.platform === 'win32';
const label = win ? 'Restore points' : 'Filesystem snapshots';
const some = (n: number) => (win ? `${String(n)} restore point${n === 1 ? '' : 's'}` : `${String(n)} snapshot${n === 1 ? '' : 's'}`);
const hidden = win
? 'Restore points hold space no folder scan can see, so any space they hold is inside Unaccounted.'
: 'Snapshots pin blocks that no directory walk can see, so any space they hold is inside Unaccounted.';
const line = (rest: Omit<StatementLine, 'id' | 'label' | 'notes'> & { notes?: string[] }): StatementLine => ({ id: 'snapshots', label, notes: [], ...rest });
const remedy: StatementRemedy = {
action: 'purge-snapshots',
label: 'Delete local snapshots',
// A purge is for good (hunt C15; review F1.8/F1.15): Time Machine makes new snapshots of the disk as it is
// from then on, never these, so what only they held goes with them. Said as the Dashboard's row says it.
caveat:
'Needs an explicit confirmation. Deletes local snapshots for good, and with them any older or deleted files only they still hold; Time Machine makes new ones from now on. This cannot be undone.',
};
try {
const acc = await sources.snapshots(mountPoint);
if (!acc.available) {
return line({ bytes: null, available: false, reason: acc.reason ?? 'snapshot tooling is not available on this system', detail: hidden, count: null, remedy: null });
}
const count = acc.snapshots.length;
// Windows could not say which drive each restore point belongs to, so the
// figure is the whole PC's; the receipt says so rather than pretend.
const notes = acc.scope === 'machine' ? ['This covers restore points on every drive of this PC: Windows did not say which drive each one belongs to.'] : [];
if (acc.totalBytes !== null) {
// A measured figure is used whatever the list says. Windows can report
// space in shadow storage with no restore point listed at that moment,
// and a measurement must never be replaced by a zero.
if (count === 0 && acc.totalBytes === 0) {
return line({
bytes: 0,
available: true,
detail: win ? 'Windows reports no space used by restore points, so they hold nothing.' : 'The operating system reports no space held by snapshots, so they hold nothing.',
count: 0,
remedy: null,
});
}
const detail =
count > 0
? win
? `${some(count)} hold${count === 1 ? 's' : ''} space no folder scan can see.`
: `${some(count)} pin${count === 1 ? 's' : ''} blocks that a directory walk never sees.`
: win
? 'Windows holds this much in storage for restore points.'
: 'The operating system holds this much in snapshot storage.';
return line({ bytes: acc.totalBytes, available: true, detail, count, notes, remedy: acc.canPurge && count > 0 ? remedy : null });
}
// Zero snapshots listed, by a listing that answered and has no doubt about
// itself, is a measurement — the tool ran and found none — so this line
// legitimately reads 0. A source that lists none but says the list may not
// be the whole picture (a standard Windows user) is an unknown, below. One
// or more is the other case: `tmutil listlocalsnapshots` names them and
// does not size them, so the bytes are genuinely unknown and must not be
// shown as anything.
if (count === 0 && !acc.sizeReason) {
return line({
bytes: 0,
available: true,
detail: win ? 'This PC has no restore points, so they hold nothing.' : 'There are no local snapshots on this volume, so they hold nothing.',
count: 0,
remedy: null,
});
}
return line({
bytes: null,
available: false,
// The source knows why the size is missing (Windows: administrator
// rights) and says so in its own words; the generic sentence is tmutil's.
reason:
acc.sizeReason ??
`${some(count)} exist${count === 1 ? 's' : ''}, but the operating system names them without sizing them, so how much they hold cannot be read here.`,
detail: hidden,
count,
notes,
remedy: acc.canPurge ? remedy : null,
});
} catch (err) {
return line({ bytes: null, available: false, reason: msg(err), detail: hidden, count: null, remedy: null });
}
}
/**
* Purgeable space — unavailable everywhere, for a different reason per OS.
*
* This line exists precisely so that the thing macOS users are most often
* confused by is named rather than absent. An absent line reads as "TreeMap
* did not think of this"; an unavailable one reads as "TreeMap knows, and here
* is why it cannot tell you".
*
* The macOS reason is written for the person reading the page (QA D41). The
* checkable detail: the figure is exposed only as the Foundation resource key
* NSURLVolumeAvailableCapacityForImportantUsageKey; `diskutil`, `diskutil apfs`
* and `system_profiler` were each checked and none carries it, and TreeMap's
* native scan core does not read that key.
*/
export function purgeableLine(plat: NodeJS.Platform): StatementLine {
const reason =
plat === 'darwin'
? 'macOS tells only apps that ask through Apple’s own programming interface how much space is purgeable. TreeMap does not ask that way, and none of the disk tools that come with macOS report the figure, so it cannot be measured here.'
: plat === 'win32'
? 'Windows has no purgeable-space concept; space reclaimable by Storage Sense is not reported as used, so there is nothing to account for on this line.'
: 'This filesystem has no purgeable-space concept, so there is nothing to account for on this line.';
return {
id: 'purgeable',
label: 'Purgeable',
bytes: null,
available: false,
reason,
detail:
plat === 'darwin'
// Not "on this machine": its size is unknown here, so the line can
// claim nothing about it on this Mac (fifth browser pass F07).
? 'Space macOS counts as used but will free automatically when the disk fills — caches, downloaded content and old snapshots. It cannot be measured here, so whatever it holds is counted inside Unaccounted. On many Macs it is one of the largest things there.'
: 'Space the operating system counts as used but would free automatically under pressure.',
count: null,
notes: [],
remedy: null,
};
}
/**
* Whose programs the held line speaks for: lsof and /proc show another
* account's open files only to root. Said by the zero and, beside the holders,
* by a line with holders too (BQR-1 item 15: the Dashboard card said it only
* when it found none).
*/
const OTHER_ACCOUNTS_UNSEEN = 'TreeMap sees programs running under another account only when it runs as root; any space they hold is otherwise in Unaccounted.';
/** `held`: the held-space probe, started by buildStatement before the layout was read. */
async function openHandleLine(held: Promise<ZombieReport>): Promise<StatementLine> {
const remedy: StatementRemedy = {
action: 'review-open-handles',
label: 'See what is holding it',
caveat:
// The restart's own sentence (QA D40: this used to claim refusals the code did not make).
`TreeMap only asks a program to quit. It refuses ${RESTART_REFUSES}, and a program that declines is reported as still running rather than force-killed.`,
};
try {
const report = await held;
// Open files the pass could not decide (BQL item 11). The line keeps what
// it measured, which is then a floor — as an unsized held file makes it —
// and says how many, never a measured "nothing": AGENTS.md's 0 is a
// measurement, and these files were not measured. Any space they hold is
// in the residual, so the statement still balances.
const undecided = report.undecidedCount;
const undecidedNote = undecided === 1
? '1 file a program has open could not be checked, so this line is a floor, not a total: if it was deleted, the space it holds is in Unaccounted.'
: `${String(undecided)} files a program has open could not be checked, so this line is a floor, not a total: if they were deleted, the space they hold is in Unaccounted.`;
// And programs whose open files the pass could not read at all (the BQL-a
// review, F1.4) — on Linux, one of your account that is not dumpable: the
// same floor, the same residual.
const unreadable = report.unreadableProgramCount;
const unreadableNote = unreadable === 1
? 'The open files of 1 program could not be read, so this line is a floor, not a total: any deleted file it holds open is in Unaccounted.'
: `The open files of ${String(unreadable)} programs could not be read, so this line is a floor, not a total: any deleted file they hold open is in Unaccounted.`;
const unchecked = [...(undecided > 0 ? [undecidedNote] : []), ...(unreadable > 0 ? [unreadableNote] : [])];
if (report.processes.length === 0) {
return {
id: 'openHandles',
label: 'Held by programs still running',
bytes: 0,
available: true,
// Said of the programs TreeMap can see: lsof and /proc show another
// account's open files only to root. And said as what it found, never
// "none of your programs is holding one": under a /proc mounted with
// hidepid=2 a program of yours that is not dumpable is not even listed.
detail: unchecked.length > 0
? `TreeMap found no deleted file held open by your programs, but could not check every file they have open. ${OTHER_ACCOUNTS_UNSEEN}`
: `TreeMap found no deleted file held open by your programs, so it counts none of their space as held back. ${OTHER_ACCOUNTS_UNSEEN}`,
count: 0,
notes: unchecked,
remedy: null,
};
}
const notes: string[] = [];
// Each deleted file is counted once however many programs hold it (QA D17,
// D41), so the count is of files and the programs are named separately.
const programs = report.processes.length;
notes.push(
`${String(report.fileCount)} deleted file${report.fileCount === 1 ? ' is' : 's are'} held open by ${String(programs)} program${programs === 1 ? '' : 's'}.`,
);
if (report.sharedFileCount > 0) {
notes.push(
`${String(report.sharedFileCount)} of them ${report.sharedFileCount === 1 ? 'is' : 'are'} open in more than one program. Each is counted once here, and its space returns only when every program holding it lets go.`,
);
}
if (report.unknownSizeCount > 0) {
notes.push(
`${String(report.unknownSizeCount)} held file${report.unknownSizeCount === 1 ? '' : 's'} could not be sized, so this line is a floor, not a total. The remainder is in Unaccounted.`,
);
}
notes.push(...unchecked);
notes.push(OTHER_ACCOUNTS_UNSEEN);
return {
id: 'openHandles',
label: 'Held by programs still running',
bytes: report.totalBytes,
available: true,
detail:
'Files that were deleted while a program still had them open. The space is not returned until that program lets go, and no directory walk can see them.',
count: report.fileCount,
notes,
remedy,
};
} catch (err) {
return {
id: 'openHandles',
label: 'Held by programs still running',
bytes: null,
available: false,
reason: msg(err),
detail: 'Deleted files a program still holds open. Their space is not returned until the program lets go.',
count: null,
notes: [],
remedy: null,
};
}
}
function otherVolumesLine(
volumes: readonly LogicalVolumeInfo[],
volume: LogicalVolumeInfo,
onSameDevice: readonly LogicalVolumeInfo[],
plat: NodeJS.Platform,
): StatementLine {
// "Other" means a different filesystem in the same pool. The firmlinked twin
// is the same filesystem seen twice, and booking it here would double-count
// the largest volume on the machine.
const ours = new Set(onSameDevice.map((v) => v.id));
const siblings = containerSiblings(volumes, volume, plat).filter((v) => !ours.has(v.id));
const sized = siblings.filter((v) => v.usedBytes !== null);
const unsized = siblings.filter((v) => v.usedBytes === null);
if (siblings.length === 0) {
return {
id: 'otherVolumes',
label: 'Other volumes sharing this disk space',
bytes: 0,
available: true,
detail:
plat === 'darwin'
? 'Nothing else shares this storage pool, so no space is going anywhere else.'
: "This filesystem's total covers only itself — other filesystems on the same drive are not inside the number above, so there is nothing to subtract here.",
count: 0,
notes: [],
remedy: null,
};
}
if (sized.length === 0) {
return {
id: 'otherVolumes',
label: 'Other volumes sharing this disk space',
bytes: null,
available: false,
reason: `${String(siblings.length)} other volume${siblings.length === 1 ? '' : 's'} share this pool, but the system did not report how much any of them uses.`,
detail: 'Volumes in the same container draw on one pool of free space, and a scan of one never walks the others.',
count: siblings.length,
notes: [],
remedy: null,
};
}
const bytes = sized.reduce((sum, v) => sum + (v.usedBytes ?? 0), 0);
const notes = sized
.slice()
.sort((a, b) => (b.usedBytes ?? 0) - (a.usedBytes ?? 0))
.map((v) => `${v.name ?? v.id} (${v.mountPoint ?? 'not mounted'}) — ${formatBytes(v.usedBytes ?? 0)}`);
if (unsized.length > 0) {
notes.push(
`${String(unsized.length)} more volume${unsized.length === 1 ? '' : 's'} in this pool reported no usage figure, so anything they hold is in Unaccounted.`,
);
}
return {
id: 'otherVolumes',
label: 'Other volumes sharing this disk space',
bytes,
available: true,
detail:
'Volumes in the same container draw on one shared pool of free space. A scan of one volume never walks the others, but their bytes still come out of the same total.',
count: sized.length,
notes,
remedy: null,
};
}
/**
* Drives mounted inside the scanned folder (Track U, U-28).
*
* A drive mounted inside the folder — a disk image, the iOS Simulator's
* volumes, a separate partition — is not on this volume, so a scan that went
* into it counted files this volume does not hold. Under "Stay on this drive"
* (the default) the scan went into none of them: a measured 0, with the count.
*
* Under "Include drives mounted inside it" the over-count is real but UNKNOWN,
* and the line is null with the reason — never a subtraction of the logical
* bytes counted there. The tree's sizes are logical, and `sparseLine` has
* already netted every file's sparse and slack share, those inside entered
* drives included; taking their logical total off again would subtract a
* sparse disk image's unallocated part twice. Allocation is not tracked per
* drive, so the logical total is stated in the detail and the rest is in
* Unaccounted, which names this line.
*
* Unknown too, with the reason: a scan with no boundary on its record, a mount
* table that could not be read, and a crossing the table did not list (the
* walker's device detector), which may be a drive counted as this disk.
*/
function mountedInsideLine(scan: ScanResult): StatementLine {
const label = 'Drives mounted inside the scanned folder';
const detail =
'Drives mounted inside the scanned folder — disk images, the iOS Simulator\u2019s volumes, separate partitions — hold files that are not on this volume.';
const unknown = (reason: string, count: number | null, more?: string): StatementLine => ({
id: 'mountedInside', label, bytes: null, available: false, reason, detail: more ?? detail, count, notes: [], remedy: null,
});
const report = boundaryReport(scan);
const root = scan.rootPath;
const plural = (n: number, one: string, many: string): string => (n === 1 ? one : many);
if (report.rule === null) {
return unknown('This scan was made without reading the mount table, so whether it counted a drive mounted inside the folder is not known.', null);
}
if (!report.known) {
return unknown(`TreeMap could not read the list of mounted drives (${report.reason ?? 'no reason given'}), so a drive mounted inside ${root} may have been counted above as part of this disk.`, null);
}
const crossed = report.crossedUnlisted;
if (crossed.count > 0) {
return unknown(
`The scan went into ${String(crossed.count)} ${plural(crossed.count, 'folder', 'folders')} on another device that the mount table does not list (${crossed.examples.join(', ')}); ` +
`if ${plural(crossed.count, 'it is a separate drive, its', 'they are separate drives, their')} files are counted above although they are not on this disk.`,
crossed.count,
);
}
// Windows keeps no table to read (U-23): its own sentence says why, and the
// engines do not follow a volume mounted in a folder there, so nothing on
// another disk is counted — but how many such drives exist is not known, and
// "no drive is mounted" would claim a reading nobody made (TS review R11).
if (report.source === 'none' && report.reason) {
return {
id: 'mountedInside', label, bytes: 0, available: true,
detail: `${report.reason} So nothing on another disk is counted above.`,
count: null, notes: [], remedy: null,
};
}
if (report.rule === 'stay') {
const n = report.notEntered.count;
return {
id: 'mountedInside', label, bytes: 0, available: true,
detail: n > 0
? `${String(n)} separate ${plural(n, 'drive is', 'drives are')} mounted inside ${root}; by the mount table the scan did not enter ${plural(n, 'it', 'them')}, so none of ${plural(n, 'its', 'their')} files are counted above.`
: `No drive is mounted inside ${root}, so nothing on another disk is counted above.`,
count: n, notes: [], remedy: null,
};
}
const n = report.entered.count;
if (n === 0) {
return {
id: 'mountedInside', label, bytes: 0, available: true,
detail: `No drive is mounted inside ${root}, so nothing on another disk is counted above.`,
count: 0, notes: [], remedy: null,
};
}
const total = report.entered.bytes;
const counted = total === null ? 'an amount it cannot look up' : formatBytes(total);
return unknown(
`The scan counted ${counted} inside drives mounted in this folder; their allocated share is already inside the sparse-file line and is not tracked per drive, so TreeMap cannot say how much to take back off.`,
n,
`${String(n)} ${plural(n, 'drive', 'drives')} mounted inside ${root} ${plural(n, 'was', 'were')} entered (\u201cInclude drives mounted inside it\u201d): the scan counted ${counted} of files there, by their logical size, and those files are not on this disk.`,
);
}
/**
* Engines that count what they were refused.
*
* The walker does, because it sees every `readdir` rejection itself. **gdu does
* not**, and that was measured rather than assumed: pointed at a directory with
* mode 000, `gdu -o-` exits 0 and emits it as an ordinary *empty* directory —
* no error key, no annotation, byte-identical in shape to a directory that
* genuinely has nothing in it. `gdu --help` offers no flag that changes this.
*
* So on a gdu scan the count is not zero, it is unknown, and the line says so.
* Printing "0 refused" from an engine that cannot tell would be precisely the
* confidently-wrong answer this whole view exists to eliminate.
*
* The native engine counts them as the walker does — the equivalence gate
* holds its counters equal to the walker's — and the NTFS turbo mode fills the
* same counters from its own table (both through `ingestColumns`).
*/
const ENGINES_THAT_COUNT_REFUSALS: ReadonlySet<string> = new Set(['walker', 'turbo-walker', 'native', 'ntfs-mft']);
/**
* Engines that can measure how much room a file really takes.
*
* The walker reads `stat.blocks`, the native engine each file's allocation,
* and gdu reports `dsize`, so all three can say what a file occupies as well
* as what it claims. A cloud listing has no such figure — it knows only the
* size a provider reports — so on a cloud scan the answer is unknown, not
* zero. (On Windows no engine is asked: `sparseLine` answers first.)
*/
const ENGINES_THAT_MEASURE_ALLOCATION: ReadonlySet<string> = new Set(['walker', 'turbo-walker', 'native', 'gdu-turbo']);
/**
* Space the scanned files claim but do not occupy.
*
* The single most common reason the numbers do not add up on a Mac. A virtual
* machine's disk — Docker.raw is the usual one — reserves the whole size it was
* created with and fills only part of it, and macOS also stores many files
* compressed. The line above counted what they claim, so this takes back the
* part that is not on the disk. Cloud placeholders and hard-linked duplicates
* are deliberately not in this figure: the first is taken off in full by the
* line above, and the second was never added.
*
* Exported for its own test: the arithmetic here decides whether the receipt
* balances, and driving it through a whole live scan would prove far less.
*/
/** The usual reserving file and who compresses files behind the scan's back are per-OS facts; the rest of the sentence is not. */
function sparseDetail(plat: NodeJS.Platform): string {
const reserver =
plat === 'darwin'
? 'a virtual machine disk such as Docker.raw'
: plat === 'win32'
? 'a virtual machine disk such as WSL\u2019s ext4.vhdx'
: 'a virtual machine disk image';
const compressor =
plat === 'darwin'
? 'macOS stores many files compressed'
: plat === 'win32'
? 'Windows can store files compressed'
: 'some filesystems store files compressed';
return `Some files reserve room they have not filled — ${reserver} is the usual one — and ${compressor}, so they claim more than they hold. A very small file is the other way round: the disk hands out room in fixed-size pieces, so it holds a little more than it claims. The line above counted what every file claims; this is the difference, whichever way it falls.`;
}
/**
* The line's two parts, in words (QA third pass D09). Its count is the files
* that claim more than they take, and its bytes the net of those against
* every other file's rounding the other way — so "4 items · −7.3 GB" sat
* beside the Dashboard's "4 files claim 7.6 GB more than they take" and
* read as another set of files. The net stays (the receipt balances on it);
* this says the parts it is made of. Said only when there is rounding to
* make the difference: without it the count and the bytes are the same files.
*/
function sparseParts(count: number, sparse: number, slack: number): string {
const rest = `take ${formatBytes(slack)} more than they claim, rounded up to the disk’s fixed-size pieces`;
if (count === 0) return `No file here claims more than it takes; files ${rest} — this line adds that back.`;
const one = count === 1;
return `${formatCount(count)} file${one ? ' claims' : 's claim'} ${formatBytes(sparse)} more than ${one ? 'it takes' : 'they take'}, ` +
`and the rest ${rest} — this line is the difference.`;
}
export function sparseLine(scan: ScanResult, plat: NodeJS.Platform): StatementLine {
const label = 'What these files claim, against what they hold';
const detail = sparseDetail(plat);
const engine = scan.engine ?? 'walker';
if (plat === 'win32') {
return {
id: 'sparseFiles', label, bytes: null, available: false,
reason:
'Windows does not tell a folder walk how much room each file really takes, so how much of what these files claim is not on the disk cannot be known from this scan.',
detail, count: null, notes: [], remedy: null,
};
}
if (!ENGINES_THAT_MEASURE_ALLOCATION.has(engine)) {
return {
id: 'sparseFiles', label, bytes: null, available: false,