Skip to content

Commit 51e9124

Browse files
yyodsclaude
yyods
and
claude
committed
[red] pin V4 fixed SLOs, source-era routing, NVS reopening
Witnessed RED for the ADR-0037/ADR-0038 replacement contract and the amended C3 post-OI NVS receipt. Audited and completed the predecessor's unwitnessed draft: - test_oi1_profile_bench.py: keep the V3/V4 registries, fixed 3000/7000 ms and 6600 B/s SLOs, the 9,929,696,969/9,929,696,970 ns goodput boundary pair, the a8be631 baseline immutability pin, and the v0.5.1 historical-derivation pin; fix the boundary test to assert the derived fixed SLOs instead of overwriting them and to match the *_min_* threshold failure keys; pin the active policy identity to exactly 5,015 bytes, sha256 b9b19ddc. - test_oi1_rp2_profile_bench.py: same two fixes for the exact Pico 7000/6600 boundary test. - test_release_v060_contract.py: new V060ReplacementSourceEraContract tests; release tooling must expose QUALIFICATION_DERIVATION_V4 and the frozen 5620f2f boundary / 719b211 abandoned-candidate identities, select the derivation from the bound source commit's git ancestry (strict descendant => V4; boundary/ancestor => historical; unknown, unrelated, or unprovable => fail closed), and keep the superseded policy identity (4,986 bytes, sha256 09e208ab) auditable with the active policy as its exact 11-numeric-field + derivation-id delta. - test_target_runtime_contract.py: the committed C3 policy row must carry the fixed 6600 B/s floors with byte-identical static/heap gates. - test_v060_qualification_workflow.py + test_release_v060_private_gates.py: amend the bypassable 5620f2f receipt RED to the raw-slice design; the runner captures the raw NVS slice after the workload and before teardown/threshold evaluation; the only passing slice is 24,576 bytes all 0xff (sha256 1df8949b); writer, completion, and finalization reopen and re-hash the exclusive mode-0600 receipt/slice/acquisition log and reject copied summaries; a diagnostic capture never mints a passing result. Witnessed failing suites (all for missing-V4 / missing-receipt behavior; collateral suites stay green): test_oi1_profile_bench.py FAILED (failures=8) test_oi1_rp2_profile_bench.py FAILED (failures=2) test_release_v060_contract.py FAILED (failures=4) test_v060_qualification_workflow.py FAILED (failures=5, errors=6) test_release_v060_private_gates.py FAILED (failures=2, errors=4) test_target_runtime_contract.py C3 policy pin FAILED (failures=1) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CAVghCmvKjF4BZFBUjAMwc Signed-off-by: Viwat Vchirawongkwin <viwat.v@chula.ac.th>
1 parent 482c57b commit 51e9124

6 files changed

Lines changed: 897 additions & 90 deletions

‎tests/firmware_tests/host/test_oi1_profile_bench.py‎

Lines changed: 152 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,32 @@
4242
"esp32-c3-4mb",
4343
"rpi-pico2-w",
4444
)
45+
HISTORICAL_V3_DERIVATION = {
46+
"application_image": "exact-byte-identical-two-root-v1",
47+
"application_headroom": "factory-minus-application-v1",
48+
"heap_floor": "floor-min-1024-v1",
49+
"boot_ceiling": "fixed-product-slo-3000-v3",
50+
"goodput_floor": "floor-95pct-min-100-v2",
51+
}
52+
REPLACEMENT_V4_DERIVATION = {
53+
"application_image": "exact-byte-identical-two-root-v1",
54+
"application_headroom": "factory-minus-application-v1",
55+
"heap_floor": "floor-min-1024-v1",
56+
"boot_ceiling": "fixed-profile-product-slo-esp3000-pico7000-v4",
57+
"goodput_floor": "fixed-product-slo-64k-under-10s-6600-v3",
58+
}
59+
FIXED_PERFORMANCE_THRESHOLDS = {
60+
profile_id: {
61+
"reset_to_service_advertisement_max_ms": (
62+
7000 if profile_id == "rpi-pico2-w" else 3000
63+
),
64+
"put_committed_goodput_min_bytes_per_second": 6600,
65+
"get_verified_goodput_min_bytes_per_second": 6600,
66+
}
67+
for profile_id in CURRENT_PROFILE_ORDER
68+
}
69+
PASSING_6600_DURATION_NS = 9_929_696_969
70+
FAILING_6599_DURATION_NS = PASSING_6600_DURATION_NS + 1
4571
HISTORICAL_V042_PROFILE_ORDER = (
4672
"esp32-4mb",
4773
"esp32-s3-n16r8",
@@ -378,26 +404,16 @@ def test_exact_v060_candidate_profiles_and_common_workload_are_frozen(self):
378404
)
379405
self.assertEqual(
380406
bench.DERIVATION,
381-
{
382-
"application_image": "exact-byte-identical-two-root-v1",
383-
"application_headroom": "factory-minus-application-v1",
384-
"heap_floor": "floor-min-1024-v1",
385-
"boot_ceiling": "fixed-product-slo-3000-v3",
386-
"goodput_floor": "floor-95pct-min-100-v2",
387-
},
407+
REPLACEMENT_V4_DERIVATION,
388408
)
389409

390-
def test_committed_v060_policy_is_immutable_five_profile_evidence(self):
410+
def test_committed_v060_policy_uses_fixed_slos_and_retains_a8_baseline(self):
391411
policy_path = (
392412
REPO_ROOT / "firmware" / "qualification" / "oi1-gates.json"
393413
)
394414
payload = policy_path.read_bytes()
395415
policy = json.loads(payload.decode("utf-8"))
396416

397-
self.assertEqual(
398-
hashlib.sha256(payload).hexdigest(),
399-
"09e208ab5069b2229d2ae0983df33bb9310de85a8aab83ee816db4456cf21bdd",
400-
)
401417
self.assertEqual(policy["schema_version"], 3)
402418
self.assertEqual(
403419
policy["profile_order"],
@@ -428,6 +444,53 @@ def test_committed_v060_policy_is_immutable_five_profile_evidence(self):
428444
),
429445
},
430446
)
447+
self.assertEqual(policy["derivation"], REPLACEMENT_V4_DERIVATION)
448+
self.assertEqual(
449+
{
450+
entry["profile_id"]: {
451+
key: entry["thresholds"][key]
452+
for key in (
453+
"reset_to_service_advertisement_max_ms",
454+
"put_committed_goodput_min_bytes_per_second",
455+
"get_verified_goodput_min_bytes_per_second",
456+
)
457+
}
458+
for entry in policy["profiles"]
459+
},
460+
FIXED_PERFORMANCE_THRESHOLDS,
461+
)
462+
self.assertEqual(len(payload), 5015)
463+
self.assertEqual(
464+
hashlib.sha256(payload).hexdigest(),
465+
"b9b19ddc217598835185429d6d1f1da60bb3a504a447e32dcdeae15bf18bd0c5",
466+
)
467+
468+
def test_retained_a8_baseline_bytes_and_samples_remain_immutable(self):
469+
baseline_path = (
470+
REPO_ROOT
471+
/ "docs"
472+
/ "validation"
473+
/ "firmware"
474+
/ "oi1"
475+
/ "a8be631df46590166307aa41afaea30b39e29230.json"
476+
)
477+
payload = baseline_path.read_bytes()
478+
baseline = json.loads(payload.decode("utf-8"))
479+
480+
self.assertEqual(
481+
hashlib.sha256(payload).hexdigest(),
482+
"8a7dbf328ba8d70f5161582b56d1566821f20b7a259ff29dc7d6fe6bb75a6044",
483+
)
484+
self.assertEqual(baseline["schema_version"], 2)
485+
self.assertEqual(baseline["source_commit"], "a8be631df46590166307aa41afaea30b39e29230")
486+
self.assertEqual(baseline["profile_order"], list(CURRENT_PROFILE_ORDER))
487+
self.assertEqual(
488+
[
489+
max(entry["oi1_observation"]["reset_to_service_advertisement_ms"])
490+
for entry in baseline["profiles"]
491+
],
492+
[1828, 1187, 1594, 1167, 3378],
493+
)
431494

432495
def test_v051_three_profile_policy_contract_uses_synthetic_evidence(self):
433496
synthetic_thresholds = {
@@ -443,7 +506,7 @@ def test_v051_three_profile_policy_contract_uses_synthetic_evidence(self):
443506
"profile_order": list(PROFILE_ORDER),
444507
"deferred_profiles": ["esp32-c3-4mb"],
445508
"workload": bench.V051_WORKLOAD,
446-
"derivation": bench.DERIVATION,
509+
"derivation": HISTORICAL_V3_DERIVATION,
447510
"baseline_evidence": {
448511
"path": "synthetic-v0.5.1-evidence.json",
449512
"sha256": "0" * 64,
@@ -874,7 +937,7 @@ def test_partition_table_requires_one_md5_verified_factory_that_fits(self):
874937
with self.assertRaises(bench.BenchError):
875938
bench.oi1_build_from_bytes(b"A" * 101, _partition_table(100))
876939

877-
def test_frozen_metric_specific_threshold_derivation(self):
940+
def test_fixed_esp_product_slos_ignore_baseline_extrema(self):
878941
post_hello = [
879942
_heap(9001 + i, 1000, 17001 + i, 9001 + i, 5001 + i)
880943
for i in range(10)
@@ -908,8 +971,8 @@ def test_frozen_metric_specific_threshold_derivation(self):
908971
"idf_internal_largest_block_min_bytes": 8192,
909972
"idf_internal_minimum_free_min_bytes": 4096,
910973
"reset_to_service_advertisement_max_ms": 3000,
911-
"put_committed_goodput_min_bytes_per_second": 31100,
912-
"get_verified_goodput_min_bytes_per_second": 15500,
974+
"put_committed_goodput_min_bytes_per_second": 6600,
975+
"get_verified_goodput_min_bytes_per_second": 6600,
913976
},
914977
)
915978

@@ -941,8 +1004,8 @@ def test_repeatability_allowances_do_not_change_static_or_heap_gates(self):
9411004
"idf_internal_largest_block_min_bytes": 81920,
9421005
"idf_internal_minimum_free_min_bytes": 86016,
9431006
"reset_to_service_advertisement_max_ms": 3000,
944-
"put_committed_goodput_min_bytes_per_second": 6900,
945-
"get_verified_goodput_min_bytes_per_second": 13200,
1007+
"put_committed_goodput_min_bytes_per_second": 6600,
1008+
"get_verified_goodput_min_bytes_per_second": 6600,
9461009
},
9471010
)
9481011

@@ -971,7 +1034,7 @@ def test_repeatability_allowances_do_not_change_static_or_heap_gates(self):
9711034
repeat_derived[
9721035
"put_committed_goodput_min_bytes_per_second"
9731036
],
974-
6800,
1037+
6600,
9751038
)
9761039

9771040
def test_fixed_reset_product_slo_accepts_boundary_and_rejects_above_it(self):
@@ -1024,6 +1087,76 @@ def test_fixed_reset_product_slo_accepts_boundary_and_rejects_above_it(self):
10241087
thresholds,
10251088
)
10261089

1090+
def test_all_four_esp_profiles_enforce_every_fixed_boundary_sample(self):
1091+
build = {
1092+
"application_image_bytes": 100,
1093+
"factory_partition_bytes": 1000,
1094+
"application_headroom_bytes": 900,
1095+
}
1096+
for profile_id in CURRENT_PROFILE_ORDER[:-1]:
1097+
with self.subTest(profile_id=profile_id):
1098+
observation = _valid_observation(profile_id)
1099+
observation["reset_to_service_advertisement_ms"] = [3000] * 10
1100+
for prefix in ("put", "get"):
1101+
observation["%s_duration_ns" % prefix] = [
1102+
PASSING_6600_DURATION_NS
1103+
] * 5
1104+
observation["put_committed_goodput_bytes_per_second"] = [6600] * 5
1105+
observation["get_verified_goodput_bytes_per_second"] = [6600] * 5
1106+
bench.validate_observation(observation, profile_id=profile_id)
1107+
1108+
thresholds = bench.derive_thresholds(build, observation)
1109+
self.assertEqual(
1110+
{
1111+
key: thresholds[key]
1112+
for key in FIXED_PERFORMANCE_THRESHOLDS[profile_id]
1113+
},
1114+
FIXED_PERFORMANCE_THRESHOLDS[profile_id],
1115+
"derivation must yield the fixed product SLOs for %s"
1116+
% profile_id,
1117+
)
1118+
bench.evaluate_thresholds(build, observation, thresholds)
1119+
1120+
reset_failure = copy.deepcopy(observation)
1121+
reset_failure["reset_to_service_advertisement_ms"][-1] = 3001
1122+
with self.assertRaisesRegex(
1123+
bench.BenchError,
1124+
"reset_to_service_advertisement_max_ms=3001 exceeds 3000",
1125+
):
1126+
bench.evaluate_thresholds(build, reset_failure, thresholds)
1127+
1128+
for direction, duration_key, goodput_key, floor_key in (
1129+
(
1130+
"PUT",
1131+
"put_duration_ns",
1132+
"put_committed_goodput_bytes_per_second",
1133+
"put_committed_goodput_min_bytes_per_second",
1134+
),
1135+
(
1136+
"GET",
1137+
"get_duration_ns",
1138+
"get_verified_goodput_bytes_per_second",
1139+
"get_verified_goodput_min_bytes_per_second",
1140+
),
1141+
):
1142+
transfer_failure = copy.deepcopy(observation)
1143+
transfer_failure[duration_key][-1] = FAILING_6599_DURATION_NS
1144+
transfer_failure[goodput_key][-1] = 6599
1145+
bench.validate_observation(
1146+
transfer_failure,
1147+
profile_id=profile_id,
1148+
)
1149+
with self.assertRaisesRegex(
1150+
bench.BenchError,
1151+
"%s=6599 is below 6600" % floor_key,
1152+
msg="%s boundary must fail for %s" % (direction, profile_id),
1153+
):
1154+
bench.evaluate_thresholds(
1155+
build,
1156+
transfer_failure,
1157+
thresholds,
1158+
)
1159+
10271160

10281161
class CapsAndMtuTest(unittest.TestCase):
10291162
def test_caps_are_strict_and_transport_values_are_exact(self):

‎tests/firmware_tests/host/test_oi1_rp2_profile_bench.py‎

Lines changed: 66 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -369,9 +369,9 @@ def test_rp2_thresholds_and_observation_are_target_discriminated(self):
369369
RP2_IMAGE_LIMIT_BYTES - 256
370370
),
371371
"gc_free_min_bytes": 8192,
372-
"reset_to_service_advertisement_max_ms": 3000,
373-
"put_committed_goodput_min_bytes_per_second": 62200,
374-
"get_verified_goodput_min_bytes_per_second": 62200,
372+
"reset_to_service_advertisement_max_ms": 7000,
373+
"put_committed_goodput_min_bytes_per_second": 6600,
374+
"get_verified_goodput_min_bytes_per_second": 6600,
375375
},
376376
)
377377
self.assertEqual(
@@ -387,6 +387,69 @@ def test_rp2_thresholds_and_observation_are_target_discriminated(self):
387387
profile_id=PICO_PROFILE,
388388
)
389389

390+
def test_pico_fixed_reset_and_transfer_boundaries_are_exact(self):
391+
build = {
392+
"firmware_bin_bytes": 256,
393+
"firmware_image_limit_bytes": RP2_IMAGE_LIMIT_BYTES,
394+
"firmware_image_headroom_bytes": RP2_IMAGE_LIMIT_BYTES - 256,
395+
}
396+
observation = _rp2_observation()
397+
observation["reset_to_service_advertisement_ms"] = [7000] * 10
398+
for prefix in ("put", "get"):
399+
observation["%s_duration_ns" % prefix] = [9_929_696_969] * 5
400+
observation["put_committed_goodput_bytes_per_second"] = [6600] * 5
401+
observation["get_verified_goodput_bytes_per_second"] = [6600] * 5
402+
bench.validate_observation(observation, profile_id=PICO_PROFILE)
403+
404+
thresholds = bench.derive_thresholds(build, observation)
405+
self.assertEqual(
406+
{
407+
key: thresholds[key]
408+
for key in (
409+
"reset_to_service_advertisement_max_ms",
410+
"put_committed_goodput_min_bytes_per_second",
411+
"get_verified_goodput_min_bytes_per_second",
412+
)
413+
},
414+
{
415+
"reset_to_service_advertisement_max_ms": 7000,
416+
"put_committed_goodput_min_bytes_per_second": 6600,
417+
"get_verified_goodput_min_bytes_per_second": 6600,
418+
},
419+
"derivation must yield the fixed Pico product SLOs",
420+
)
421+
bench.evaluate_thresholds(build, observation, thresholds)
422+
423+
reset_failure = dict(observation)
424+
reset_failure["reset_to_service_advertisement_ms"] = [7000] * 9 + [7001]
425+
with self.assertRaisesRegex(
426+
bench.BenchError,
427+
"reset_to_service_advertisement_max_ms=7001 exceeds 7000",
428+
):
429+
bench.evaluate_thresholds(build, reset_failure, thresholds)
430+
431+
for duration_key, goodput_key, floor_key in (
432+
(
433+
"put_duration_ns",
434+
"put_committed_goodput_bytes_per_second",
435+
"put_committed_goodput_min_bytes_per_second",
436+
),
437+
(
438+
"get_duration_ns",
439+
"get_verified_goodput_bytes_per_second",
440+
"get_verified_goodput_min_bytes_per_second",
441+
),
442+
):
443+
transfer_failure = dict(observation)
444+
transfer_failure[duration_key] = [9_929_696_969] * 4 + [9_929_696_970]
445+
transfer_failure[goodput_key] = [6600] * 4 + [6599]
446+
bench.validate_observation(transfer_failure, profile_id=PICO_PROFILE)
447+
with self.assertRaisesRegex(
448+
bench.BenchError,
449+
"%s=6599 is below 6600" % floor_key,
450+
):
451+
bench.evaluate_thresholds(build, transfer_failure, thresholds)
452+
390453
def test_rp2_baseline_fragment_binds_install_and_resource_hashes(self):
391454
profile = profile_bench.build_baseline_profile(
392455
profile_id=PICO_PROFILE,

0 commit comments

Comments
 (0)