Skip to content

Commit fa4b207

Browse files
committed
[build] Integrate reproducible v0.4.2 source
Signed-off-by: Viwat Vchirawongkwin <viwat.v@chula.ac.th>
2 parents 454a270 + ce02b68 commit fa4b207

7 files changed

Lines changed: 112 additions & 10 deletions

File tree

‎docs/specifications/firmware/TDD.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1155,7 +1155,10 @@ PBLE/1 **conformance** tests run against an **in-memory fake transport** shared
11551155
toolchain-distribution license bytes from the trusted ESP-IDF download cache,
11561156
exact metadata/cache/install binding with distinct archive and version roots,
11571157
absence of host-absolute paths in receipts, and profile-specific zero-input
1158-
not-shipped proof.
1158+
not-shipped proof. Supplemental source-tree digests exclude Python bytecode
1159+
cache artifacts while the audit rejects any such artifacts in the retained
1160+
checkout; release builds force bytecode generation off so checkout-local
1161+
absolute paths cannot contaminate otherwise identical source evidence.
11591162
- **size:** enforce the total application-image ceiling and derived
11601163
factory-partition headroom floor during build/candidate validation. Continue
11611164
structural application-fit checks on all three source targets, including

‎firmware/licenses/license-policy.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17094,7 +17094,7 @@
1709417094
"source": {
1709517095
"ref": "3.6.4@b5d87eaa6748b7a6fa70593178c08b4480e9b71e",
1709617096
"tree_path": "firmware/.esp-idf/components/mbedtls",
17097-
"tree_sha256": "0338a54ab75fa598cf77260998deb48879c326d8a785305e4d31f3e9b232a3cc",
17097+
"tree_sha256": "4366d742b5721fab1705ce58962558b25d08aa0176913e9466a3f02284d729b3",
1709817098
"url": "https://github.com/Mbed-TLS/mbedtls/tree/b5d87eaa6748b7a6fa70593178c08b4480e9b71e"
1709917099
},
1710017100
"source_spdx_expression": "(Apache-2.0 OR GPL-2.0-or-later)"

‎firmware/release-tools.lock‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ sha256 = "a1444a7f23740c44cacbce4845efb5cbcb08927878b6a3852c33a52d8b2b5da9"
1414
excluded_cves_path = "firmware/licenses/excluded-cves.yaml"
1515
excluded_cves_sha256 = "77c02542775c018c5f32c0dcb6e88050adb0e69dbcd95e84ad7be128e0f536e5"
1616
license_policy_path = "firmware/licenses/license-policy.json"
17-
license_policy_sha256 = "ea1077f5ea9d8986fa2d1522d1552508d64712e4e0e930f8837c2a19a60938a8"
17+
license_policy_sha256 = "f31f6c1b823e079ac15e810d9b982a30916f9c4670c3a6a2c2071fa166219a8f"
1818

1919
[[artifacts]]
2020
name = "esp-idf-sbom"

‎firmware/scripts/build.sh‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@
2222

2323
set -u
2424

25+
# Python helpers invoked by ESP-IDF must never write checkout-local bytecode.
26+
# Besides mutating the retained source tree, .pyc payloads embed host paths and
27+
# make otherwise identical license evidence depend on the checkout location.
28+
PYTHONDONTWRITEBYTECODE=1
29+
export PYTHONDONTWRITEBYTECODE
30+
2531
HERE="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd -P)"
2632
FW="$(cd "$HERE/.." && pwd -P)"
2733
REPO_ROOT="$(cd "$FW/.." && pwd -P)"
@@ -161,6 +167,14 @@ if [ ! -f "$IDF_DIR/export.sh" ]; then
161167
exit 1
162168
fi
163169

170+
# Use the same physical checkout prefix that ESP-IDF exports to CMake. This is
171+
# also the prefix covered by the final, more-specific deterministic path map.
172+
if ! IDF_DIR="$(cd "$IDF_DIR" 2>/dev/null && pwd -P)"; then
173+
echo "build.sh: cannot resolve the pinned ESP-IDF checkout" >&2
174+
exit 1
175+
fi
176+
export PYBLE_IDF_DIR="$IDF_DIR"
177+
164178
# Release provenance may say clean=true only after this check. Generated build
165179
# outputs and overlay copies are ignored/submodule-untracked and do not weaken
166180
# tracked source cleanliness.
@@ -255,11 +269,15 @@ export MICROPY_MPYCROSS
255269
# and PyBLE USER_C_MODULES sit outside those prefixes. -ffile-prefix-map is the
256270
# pinned GCC spelling that covers debug paths and macro/file expansions. GCC
257271
# gives the final matching map precedence, so order broad prefixes before the
258-
# more-specific target build and retained MicroPython checkout.
272+
# more-specific target build, retained MicroPython checkout, and ESP-IDF
273+
# checkout. The IDF mapping must remain last because a repo-local pinned
274+
# checkout is nested below REPO_ROOT and must keep ESP-IDF's stable /IDF
275+
# spelling rather than the broader /PYBLE spelling.
259276
DETERMINISTIC_CFLAGS="\
260277
-ffile-prefix-map=$REPO_ROOT=/PYBLE \
261278
-ffile-prefix-map=$OUT=/IDF_BUILD \
262-
-ffile-prefix-map=$UPSTREAM_DIR=/MICROPYTHON"
279+
-ffile-prefix-map=$UPSTREAM_DIR=/MICROPYTHON \
280+
-ffile-prefix-map=$IDF_DIR=/IDF"
263281

264282
make -C "$PORT" \
265283
submodules \

‎firmware/scripts/release_bundle.py‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2409,6 +2409,43 @@ def _audit_sha256_tree(path: Path) -> str:
24092409
return digest.hexdigest()
24102410

24112411

2412+
def _audit_python_cache_artifact(relative: Path) -> bool:
2413+
return "__pycache__" in relative.parts or relative.suffix.lower() in {
2414+
".pyc",
2415+
".pyo",
2416+
}
2417+
2418+
2419+
def _audit_sha256_source_tree(
2420+
path: Path,
2421+
*,
2422+
reject_python_cache: bool = False,
2423+
) -> str:
2424+
"""Hash source bytes canonically without host-generated Python caches."""
2425+
2426+
_require(path.is_dir(), "reviewed source tree is missing: %s" % path)
2427+
digest = hashlib.sha256()
2428+
for item in sorted(path.rglob("*")):
2429+
_require(not item.is_symlink(), "reviewed source tree contains a symlink")
2430+
if not item.is_file():
2431+
continue
2432+
relative_path = item.relative_to(path)
2433+
if _audit_python_cache_artifact(relative_path):
2434+
_require(
2435+
not reject_python_cache,
2436+
"reviewed source tree contains Python cache artifact: %s"
2437+
% relative_path.as_posix(),
2438+
)
2439+
continue
2440+
relative = relative_path.as_posix().encode("utf-8")
2441+
value = item.read_bytes()
2442+
digest.update(len(relative).to_bytes(8, "big"))
2443+
digest.update(relative)
2444+
digest.update(len(value).to_bytes(8, "big"))
2445+
digest.update(value)
2446+
return digest.hexdigest()
2447+
2448+
24122449
def _audit_artifact_requirements(artifact: dict[str, Any]) -> list[str]:
24132450
value = artifact.get("requires")
24142451
_require(isinstance(value, list), "locked artifact requires must be an array")
@@ -4145,7 +4182,7 @@ def _audit_v2_source(
41454182
path = repo_root / relative
41464183
_audit_no_symlink_components(repo_root, path, "%s source tree" % label)
41474184
_require(
4148-
_audit_sha256_tree(path) == digest,
4185+
_audit_sha256_source_tree(path, reject_python_cache=True) == digest,
41494186
"%s source tree changed" % label,
41504187
)
41514188
return copy.deepcopy(record)

‎tests/firmware_tests/host/test_release_hardening.py‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -746,7 +746,7 @@ def __init__(self):
746746
done
747747
printf '\n'
748748
} >> "$PYBLE_MAKE_LOG"
749-
printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
749+
printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
750750
"${CFLAGS_EXTRA-}" \
751751
"${EXTRA_CPPFLAGS-}" \
752752
"${CFLAGS-}" \
@@ -757,7 +757,8 @@ def __init__(self):
757757
"${MAKEFLAGS-}" \
758758
"${MFLAGS-}" \
759759
"${GNUMAKEFLAGS-}" \
760-
"${MAKEOVERRIDES-}" >> "$PYBLE_MAKE_ENV_LOG"
760+
"${MAKEOVERRIDES-}" \
761+
"${PYTHONDONTWRITEBYTECODE-}" >> "$PYBLE_MAKE_ENV_LOG"
761762
printf '%s\n' "${MICROPY_MPYCROSS-}" \
762763
>> "$PYBLE_MPY_CROSS_ENV_LOG"
763764
output=""
@@ -968,11 +969,12 @@ def test_port_build_uses_only_stable_source_prefix_maps(self):
968969
fixture.output.resolve() / "esp32"
969970
),
970971
"-ffile-prefix-map={}=/MICROPYTHON".format(upstream),
972+
"-ffile-prefix-map={}=/IDF".format(fixture.idf.resolve()),
971973
)
972974
)
973975
environments = fixture.make_environments()
974976
self.assertEqual(
975-
environments[0],
977+
environments[0][:-1],
976978
("", "", "", "", "", "", "", "", "", "", ""),
977979
"ambient compiler flags must not influence the rebuilt compiler",
978980
)
@@ -999,7 +1001,7 @@ def test_port_build_uses_only_stable_source_prefix_maps(self):
9991001
1,
10001002
)
10011003
self.assertEqual(
1002-
environments,
1004+
[environment[:-1] for environment in environments],
10031005
[("", "", "", "", "", "", "", "", "", "", "")] * 3,
10041006
"submodule configuration and the final application build must "
10051007
"share exact runner-owned path maps; ambient flags may not be "
@@ -1008,6 +1010,21 @@ def test_port_build_uses_only_stable_source_prefix_maps(self):
10081010
finally:
10091011
fixture.cleanup()
10101012

1013+
def test_build_disables_python_bytecode_cache_generation(self):
1014+
fixture = BuildScriptFixture()
1015+
try:
1016+
completed = fixture.build()
1017+
self.assertEqual(completed.returncode, 0, completed.stdout)
1018+
environments = fixture.make_environments()
1019+
self.assertTrue(environments)
1020+
self.assertEqual(
1021+
[environment[-1] for environment in environments],
1022+
["1"] * len(environments),
1023+
"every build phase must prevent checkout-local Python caches",
1024+
)
1025+
finally:
1026+
fixture.cleanup()
1027+
10111028
def test_port_build_reuses_the_explicitly_built_host_mpy_cross(self):
10121029
fixture = BuildScriptFixture()
10131030
try:

‎tests/firmware_tests/host/test_release_license_policy_v2_hardening.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,33 @@ def test_supplemental_relationship_is_exactly_depends_on(self):
461461
)
462462
self.assert_rejected()
463463

464+
def test_supplemental_source_hash_excludes_but_audit_rejects_python_caches(self):
465+
source_hash = RELEASE._audit_sha256_source_tree
466+
source_tree = self.fixture.mbedtls_tree
467+
expected = source_hash(source_tree)
468+
469+
cache = source_tree / "mbedcrypto" / "__pycache__" / "config.cpython-313.pyc"
470+
legacy_cache = source_tree / "mbedcrypto" / "legacy.pyc"
471+
cache.parent.mkdir(parents=True)
472+
cache.write_bytes(b"cache containing /first/checkout/path\n")
473+
legacy_cache.write_bytes(b"legacy cache containing /first/checkout/path\n")
474+
self.assertEqual(source_hash(source_tree), expected)
475+
cache.write_bytes(b"cache containing /second/checkout/path\n")
476+
legacy_cache.write_bytes(b"legacy cache containing /second/checkout/path\n")
477+
self.assertEqual(source_hash(source_tree), expected)
478+
self.assert_rejected()
479+
480+
cache.unlink()
481+
legacy_cache.unlink()
482+
cache.parent.rmdir()
483+
real_source = source_tree / "mbedcrypto" / "mbedcrypto.c"
484+
real_source.write_text(
485+
real_source.read_text(encoding="utf-8") + "// mutation\n",
486+
encoding="utf-8",
487+
)
488+
self.assertNotEqual(source_hash(source_tree), expected)
489+
self.assert_rejected()
490+
464491
def test_observed_input_with_symlink_ancestor_is_rejected(self):
465492
identifier = "core--esp32-4mb--application"
466493
observed = copy.deepcopy(self.fixture.observed_inputs)

0 commit comments

Comments
 (0)