Skip to content

Commit 4369354

Browse files
committed
[green] validate refreshed OI policy while preserving historical deltas
Signed-off-by: Viwat Vchirawongkwin <viwat.v@chula.ac.th>
1 parent 59a5fe3 commit 4369354

2 files changed

Lines changed: 64 additions & 46 deletions

File tree

‎tests/firmware_tests/host/test_oi1_profile_bench.py‎

Lines changed: 7 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,7 @@ def load_release_module():
5151

5252

5353
def load_current_policy():
54-
payload = CURRENT_POLICY_PATH.read_bytes()
55-
return json.loads(payload.decode("utf-8")), hashlib.sha256(payload).hexdigest()
54+
return RELEASE._load_qualification_policy(REPO_ROOT)
5655

5756

5857
PROFILE_ORDER = (
@@ -87,12 +86,6 @@ def load_current_policy():
8786
}
8887
WAVESHARE_PROFILE_ID = "waveshare-esp32-s3-lcd-147b"
8988
V5_FIXED_WAVESHARE_LARGEST_BLOCK_MIN_BYTES = 98304
90-
EXPECTED_LARGEST_BLOCK_FLOORS = {
91-
"esp32-4mb": 55296,
92-
"esp32-s3-n16r8": 102400,
93-
WAVESHARE_PROFILE_ID: V5_FIXED_WAVESHARE_LARGEST_BLOCK_MIN_BYTES,
94-
"esp32-c3-4mb": 57344,
95-
}
9689
FIXED_PERFORMANCE_THRESHOLDS = {
9790
profile_id: {
9891
"reset_to_service_advertisement_max_ms": (
@@ -526,7 +519,7 @@ def test_exact_v060_candidate_profiles_and_common_workload_are_frozen(self):
526519
SECOND_REPLACEMENT_V5_DERIVATION,
527520
)
528521

529-
def test_committed_v060_policy_uses_fixed_slos_and_retains_a8_baseline(self):
522+
def test_current_policy_uses_fixed_slos_and_validated_baseline_resources(self):
530523
policy_path = (
531524
REPO_ROOT / "firmware" / "qualification" / "oi1-gates.json"
532525
)
@@ -551,19 +544,6 @@ def test_committed_v060_policy_uses_fixed_slos_and_retains_a8_baseline(self):
551544
PROFILE_TARGETS,
552545
)
553546
self.assertNotIn("deferred_profiles", policy)
554-
self.assertEqual(
555-
policy["baseline_evidence"],
556-
{
557-
"path": (
558-
"docs/validation/firmware/oi1/"
559-
"a8be631df46590166307aa41afaea30b39e29230.json"
560-
),
561-
"sha256": (
562-
"8a7dbf328ba8d70f5161582b56d1566821f20b7a259ff29d"
563-
"c7d6fe6bb75a6044"
564-
),
565-
},
566-
)
567547
self.assertEqual(policy["derivation"], SECOND_REPLACEMENT_V5_DERIVATION)
568548
self.assertEqual(
569549
{
@@ -580,19 +560,14 @@ def test_committed_v060_policy_uses_fixed_slos_and_retains_a8_baseline(self):
580560
FIXED_PERFORMANCE_THRESHOLDS,
581561
)
582562
self.assertEqual(
583-
{
584-
entry["profile_id"]: entry["thresholds"][
563+
next(
564+
entry["thresholds"][
585565
"idf_internal_largest_block_min_bytes"
586566
]
587567
for entry in policy["profiles"]
588-
if "idf_internal_largest_block_min_bytes" in entry["thresholds"]
589-
},
590-
EXPECTED_LARGEST_BLOCK_FLOORS,
591-
)
592-
self.assertEqual(len(payload), 5036)
593-
self.assertEqual(
594-
hashlib.sha256(payload).hexdigest(),
595-
"c3167853df6c31d7364b58616701d45ea62f2374d18cacc89e51292624dca8db",
568+
if entry["profile_id"] == WAVESHARE_PROFILE_ID
569+
),
570+
V5_FIXED_WAVESHARE_LARGEST_BLOCK_MIN_BYTES,
596571
)
597572

598573
def test_retained_a8_baseline_bytes_and_samples_remain_immutable(self):

‎tests/firmware_tests/host/test_release_v060_contract.py‎

Lines changed: 57 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@
7171
V060_FIRST_REPLACEMENT_SOURCE_BOUNDARY = (
7272
"7d853289815751c7381c9fd0b9a9a4409bdb6879"
7373
)
74+
V060_INITIAL_SECOND_REPLACEMENT_POLICY_SOURCE = (
75+
"0c7230d6708797c241160ba71fbd37e6b22f180a"
76+
)
7477
QUALIFICATION_DERIVATION_V5 = {
7578
"application_image": "exact-byte-identical-two-root-v1",
7679
"application_headroom": "factory-minus-application-v1",
@@ -540,7 +543,7 @@ def test_unknown_or_unrelated_source_ancestry_fails_closed(self):
540543
firmware_version="0.6.0",
541544
)
542545

543-
def test_active_policy_is_exact_fixed_slo_delta_of_superseded_policy(self):
546+
def test_retained_replacement_policies_are_exact_fixed_slo_deltas(self):
544547
historical_raw = subprocess.run(
545548
[
546549
"git",
@@ -562,26 +565,37 @@ def test_active_policy_is_exact_fixed_slo_delta_of_superseded_policy(self):
562565
historical = json.loads(historical_raw.decode("utf-8"))
563566
self.assertEqual(historical["derivation"], QUALIFICATION_DERIVATION_V3)
564567

565-
active_path = (
566-
REPO_ROOT / "firmware" / "qualification" / "oi1-gates.json"
568+
second_replacement_raw = subprocess.run(
569+
[
570+
"git", "-C", str(REPO_ROOT), "show",
571+
"%s:firmware/qualification/oi1-gates.json"
572+
% V060_INITIAL_SECOND_REPLACEMENT_POLICY_SOURCE,
573+
],
574+
check=True,
575+
capture_output=True,
576+
).stdout
577+
self.assertEqual(len(second_replacement_raw), 5036)
578+
self.assertEqual(
579+
hashlib.sha256(second_replacement_raw).hexdigest(),
580+
"c3167853df6c31d7364b58616701d45ea62f2374d18cacc89e51292624dca8db",
581+
"the initial second-replacement policy identity must stay auditable",
567582
)
568-
active_raw = active_path.read_bytes()
569-
active = json.loads(active_raw.decode("utf-8"))
583+
second_replacement = json.loads(second_replacement_raw.decode("utf-8"))
570584
self.assertEqual(
571585
(
572586
json.dumps(
573-
active,
587+
second_replacement,
574588
indent=2,
575589
sort_keys=True,
576590
ensure_ascii=False,
577591
allow_nan=False,
578592
)
579593
+ "\n"
580594
).encode("utf-8"),
581-
active_raw,
582-
"the active policy must stay canonical sorted-key JSON",
595+
second_replacement_raw,
596+
"the retained second-replacement policy must stay canonical sorted-key JSON",
583597
)
584-
self.assertEqual(active["derivation"], QUALIFICATION_DERIVATION_V5)
598+
self.assertEqual(second_replacement["derivation"], QUALIFICATION_DERIVATION_V5)
585599

586600
first_replacement_raw = subprocess.run(
587601
[
@@ -629,12 +643,12 @@ def test_active_policy_is_exact_fixed_slo_delta_of_superseded_policy(self):
629643
"binding must stay byte-identical to the superseded policy",
630644
)
631645

632-
expected_active = copy.deepcopy(first_replacement)
633-
expected_active["derivation"] = copy.deepcopy(
646+
expected_second_replacement = copy.deepcopy(first_replacement)
647+
expected_second_replacement["derivation"] = copy.deepcopy(
634648
QUALIFICATION_DERIVATION_V5
635649
)
636650
second_changed_numeric_fields = 0
637-
for entry in expected_active["profiles"]:
651+
for entry in expected_second_replacement["profiles"]:
638652
if entry["profile_id"] == "waveshare-esp32-s3-lcd-147b":
639653
self.assertEqual(
640654
entry["thresholds"][
@@ -653,12 +667,41 @@ def test_active_policy_is_exact_fixed_slo_delta_of_superseded_policy(self):
653667
"the heap-floor identifier is a separate string field",
654668
)
655669
self.assertEqual(
656-
active,
657-
expected_active,
670+
second_replacement,
671+
expected_second_replacement,
658672
"every other threshold, schema field, and baseline binding "
659673
"must stay byte-identical to the first-replacement policy",
660674
)
661675

676+
def test_current_policy_keeps_fixed_slos_with_validated_measured_resources(self):
677+
policy, digest = RELEASE._load_qualification_policy(REPO_ROOT)
678+
payload = (REPO_ROOT / "firmware/qualification/oi1-gates.json").read_bytes()
679+
self.assertEqual(digest, hashlib.sha256(payload).hexdigest())
680+
self.assertEqual(
681+
payload,
682+
(json.dumps(policy, indent=2, sort_keys=True) + "\n").encode("utf-8"),
683+
)
684+
self.assertEqual(policy["derivation"], QUALIFICATION_DERIVATION_V5)
685+
self.assertEqual(policy["profile_order"], list(V060_PROFILE_ORDER))
686+
self.assertEqual(
687+
{
688+
entry["profile_id"]: {
689+
key: entry["thresholds"][key]
690+
for key in FIXED_PERFORMANCE_THRESHOLDS[entry["profile_id"]]
691+
}
692+
for entry in policy["profiles"]
693+
},
694+
FIXED_PERFORMANCE_THRESHOLDS,
695+
)
696+
self.assertEqual(
697+
next(
698+
entry["thresholds"]["idf_internal_largest_block_min_bytes"]
699+
for entry in policy["profiles"]
700+
if entry["profile_id"] == "waveshare-esp32-s3-lcd-147b"
701+
),
702+
V5_FIXED_WAVESHARE_LARGEST_BLOCK_MIN_BYTES,
703+
)
704+
662705

663706
class V060ReproducibilityRoutingContractTests(unittest.TestCase):
664707
@staticmethod

0 commit comments

Comments
 (0)