From e5bd9a3eb10d8e11a6c416bfd310c97c84efed96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Prpi=C4=8D?= Date: Thu, 10 Sep 2026 08:52:32 -0400 Subject: [PATCH 1/6] feat(sbom): merge Maturin components into wheel SBOM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Import non-excluded CycloneDX components from Maturin-generated wheel SBOMs into Fromager's canonical SPDX document. Relate bundled components to the wheel with CONTAINS, normalize local PURLs, and preserve the original CycloneDX files. Co-Authored-By: OpenAI Codex Closes: #965 Signed-off-by: Martin Prpič --- docs/reference/files.md | 32 ++++ src/fromager/sbom.py | 400 +++++++++++++++++++++++++++++++++++++++- src/fromager/wheels.py | 4 + tests/test_sbom.py | 378 +++++++++++++++++++++++++++++++++++++ tests/test_wheels.py | 65 +++++-- 5 files changed, 864 insertions(+), 15 deletions(-) diff --git a/docs/reference/files.md b/docs/reference/files.md index 0485d2d3..0bc6379c 100644 --- a/docs/reference/files.md +++ b/docs/reference/files.md @@ -149,6 +149,38 @@ The following example shows `graph.json` file for the top-level dependency `whee } ``` +## Wheel SBOMs + +When SBOM generation is enabled, Fromager writes the canonical SPDX 2.3 +document to `.dist-info/sboms/fromager.spdx.json`. During source builds, +CycloneDX SBOMs generated by Maturin are read from the same directory and +their components are merged into the canonical SPDX document. Each imported +component is related to the wheel with `CONTAINS`; the CycloneDX dependency +graph is not copied. Nested target components are included, while components +with CycloneDX scope `excluded` are omitted because they are not shipped +runtime dependencies. Local `file://` download qualifiers are removed from +imported PURLs because those paths are only meaningful in the build +environment. + +A CycloneDX root with a PyPI PURL matching the wheel's normalized name and +version is associated with the wheel package, so auditwheel components are not +attached to the upstream source. The original CycloneDX files are preserved. + +Maturin must be version 1.12.0 or newer and must be built with its `sbom` +feature enabled. Fromager does not enable that Maturin feature automatically. +For example, a packaging environment can pass Maturin's +`MATURIN_SETUP_ARGS` with a feature set that includes `sbom`. The exact +feature set depends on the platform and packaging environment. + +This merge is applied to wheels processed by Fromager's source-build path. +Downloaded prebuilt wheels retain any native SBOM files, but are not guaranteed +to receive a merged Fromager SPDX document until they go through a separate +post-download processing path. + +```{versionchanged} 0.96.0 +Maturin CycloneDX SBOMs are merged into the canonical Fromager SPDX SBOM. +``` + ## Output Directories During the wheel building process, fromager generates multiple output directories namely `sdists-repo`, `wheels-repo` and `work-dir`. These directories contain important information related to the wheel build. diff --git a/src/fromager/sbom.py b/src/fromager/sbom.py index 52c3d5b6..a5d83103 100644 --- a/src/fromager/sbom.py +++ b/src/fromager/sbom.py @@ -22,7 +22,7 @@ from packaging.metadata import RawMetadata from packaging.requirements import Requirement from packaging.utils import NormalizedName, canonicalize_name -from packaging.version import Version +from packaging.version import InvalidVersion, Version from .pkgmetadata import DeclaredLicense, declared_license_from_raw_metadata @@ -338,6 +338,404 @@ def generate_sbom( return doc +_CYCLONEDX_HASH_TO_SPDX = { + "SHA-1": "SHA1", + "SHA-224": "SHA224", + "SHA-256": "SHA256", + "SHA-384": "SHA384", + "SHA-512": "SHA512", + "SHA3-224": "SHA3-224", + "SHA3-256": "SHA3-256", + "SHA3-384": "SHA3-384", + "SHA3-512": "SHA3-512", + "BLAKE2B-256": "BLAKE2b-256", + "BLAKE2B-384": "BLAKE2b-384", + "BLAKE2B-512": "BLAKE2b-512", +} + + +def _cyclonedx_string(value: typing.Any) -> str | None: + if isinstance(value, str) and value: + return value + return None + + +def _cyclonedx_component_purl(component: dict[str, typing.Any]) -> str | None: + return _cyclonedx_string(component.get("purl")) + + +def _cyclonedx_spdx_purl(component: dict[str, typing.Any]) -> str | None: + purl = _cyclonedx_component_purl(component) + if purl is None: + return None + try: + parsed_purl = PackageURL.from_string(purl) + except ValueError: + return purl + + qualifiers = dict(parsed_purl.qualifiers or {}) + download_url = qualifiers.get("download_url") + if download_url is None or not download_url.startswith("file://"): + return purl + del qualifiers["download_url"] + return PackageURL( + type=parsed_purl.type, + namespace=parsed_purl.namespace, + name=parsed_purl.name, + version=parsed_purl.version, + qualifiers=qualifiers or None, + subpath=parsed_purl.subpath, + ).to_string() + + +def _iter_cyclonedx_components( + component: dict[str, typing.Any], +) -> typing.Iterator[dict[str, typing.Any]]: + yield component + nested_components = component.get("components") + if not isinstance(nested_components, list): + return + for nested_component in nested_components: + if isinstance(nested_component, dict): + yield from _iter_cyclonedx_components(nested_component) + + +def _versions_match(left: str, right: str) -> bool: + try: + return Version(left) == Version(right) + except InvalidVersion: + return left == right + + +def _is_python_wheel_root( + component: dict[str, typing.Any], + sbom: dict[str, typing.Any], +) -> bool: + purl = _cyclonedx_component_purl(component) + if purl is None: + return False + try: + parsed_purl = PackageURL.from_string(purl) + except ValueError: + return False + if ( + parsed_purl.type != "pypi" + or parsed_purl.name is None + or parsed_purl.version is None + ): + return False + + component_name = _cyclonedx_string(component.get("name")) + component_version = _cyclonedx_string(component.get("version")) + if component_name is None or component_version is None: + return False + + packages = sbom.get("packages") + if not isinstance(packages, list): + return False + wheel = next( + ( + package + for package in packages + if isinstance(package, dict) and package.get("SPDXID") == "SPDXRef-wheel" + ), + None, + ) + if wheel is None: + return False + + wheel_name = _cyclonedx_string(wheel.get("name")) + wheel_version = _cyclonedx_string(wheel.get("versionInfo")) + if wheel_name is None or wheel_version is None: + return False + return ( + canonicalize_name(parsed_purl.name) == canonicalize_name(wheel_name) + and _versions_match(parsed_purl.version, wheel_version) + and canonicalize_name(component_name) == canonicalize_name(wheel_name) + and _versions_match(component_version, wheel_version) + ) + + +def _cyclonedx_component_identity(component: dict[str, typing.Any]) -> str: + purl = _cyclonedx_spdx_purl(component) + if purl: + return f"purl:{purl}" + + return "component:" + "\x00".join( + [ + _cyclonedx_string(component.get("type")) or "", + _cyclonedx_string(component.get("group")) or "", + _cyclonedx_string(component.get("name")) or "", + _cyclonedx_string(component.get("version")) or "", + ] + ) + + +def _cyclonedx_spdx_id(component: dict[str, typing.Any]) -> str: + identity = _cyclonedx_component_identity(component) + digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:20] + return f"SPDXRef-cyclonedx-{digest}" + + +def _cyclonedx_license_expression( + component: dict[str, typing.Any], +) -> str | None: + licenses = component.get("licenses") + if not isinstance(licenses, list): + return None + + expressions: list[str] = [] + for license_choice in licenses: + if not isinstance(license_choice, dict): + continue + expression = _cyclonedx_string(license_choice.get("expression")) + if expression: + expressions.append(expression) + continue + license_info = license_choice.get("license") + if isinstance(license_info, dict): + identifier = _cyclonedx_string(license_info.get("id")) + if identifier: + expressions.append(identifier) + + if not expressions: + return None + return " AND ".join(expressions) + + +def _cyclonedx_checksums( + component: dict[str, typing.Any], +) -> list[dict[str, str]]: + hashes = component.get("hashes") + if not isinstance(hashes, list): + return [] + + checksums: list[dict[str, str]] = [] + for hash_info in hashes: + if not isinstance(hash_info, dict): + continue + algorithm = _cyclonedx_string(hash_info.get("alg")) + content = _cyclonedx_string(hash_info.get("content")) + if not algorithm or not content: + continue + normalized_algorithm = _CYCLONEDX_HASH_TO_SPDX.get(algorithm.upper()) + if normalized_algorithm is None: + continue + checksums.append({"algorithm": normalized_algorithm, "checksumValue": content}) + return checksums + + +def _cyclonedx_package( + component: dict[str, typing.Any], + spdx_id: str, +) -> dict[str, typing.Any]: + purl = _cyclonedx_spdx_purl(component) + name = ( + _cyclonedx_string(component.get("name")) + or purl + or _cyclonedx_string(component.get("bom-ref")) + or "unknown" + ) + version = _cyclonedx_string(component.get("version")) or "NOASSERTION" + package: dict[str, typing.Any] = { + "SPDXID": spdx_id, + "name": name, + "versionInfo": version, + "downloadLocation": "NOASSERTION", + "supplier": "NOASSERTION", + } + + if purl: + package["externalRefs"] = [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": purl, + } + ] + + checksums = _cyclonedx_checksums(component) + if checksums: + package["checksums"] = checksums + + license_expression = _cyclonedx_license_expression(component) + if license_expression: + package["licenseDeclared"] = license_expression + + scope = _cyclonedx_string(component.get("scope")) + if scope and scope != "required": + package["comment"] = f"CycloneDX scope: {scope}" + + return package + + +def _merge_cyclonedx_package( + package: dict[str, typing.Any], + component: dict[str, typing.Any], +) -> None: + """Add non-conflicting metadata from a duplicate CycloneDX component.""" + component_checksums = _cyclonedx_checksums(component) + if component_checksums: + checksums = package.setdefault("checksums", []) + if isinstance(checksums, list): + existing_checksums = { + (item.get("algorithm"), item.get("checksumValue")) + for item in checksums + if isinstance(item, dict) + } + for checksum in component_checksums: + key = (checksum["algorithm"], checksum["checksumValue"]) + if key not in existing_checksums: + checksums.append(checksum) + existing_checksums.add(key) + + license_expression = _cyclonedx_license_expression(component) + if license_expression and "licenseDeclared" not in package: + package["licenseDeclared"] = license_expression + + scope = _cyclonedx_string(component.get("scope")) + if scope and scope != "required" and "comment" not in package: + package["comment"] = f"CycloneDX scope: {scope}" + + +def _purl_to_spdx_id(sbom: dict[str, typing.Any]) -> dict[str, str]: + purls: dict[str, str] = {} + packages = sbom.get("packages") + if not isinstance(packages, list): + return purls + + for package in packages: + if not isinstance(package, dict): + continue + spdx_id = _cyclonedx_string(package.get("SPDXID")) + external_refs = package.get("externalRefs") + if not spdx_id or not isinstance(external_refs, list): + continue + for external_ref in external_refs: + if not isinstance(external_ref, dict): + continue + if external_ref.get("referenceType") != "purl": + continue + purl = _cyclonedx_string(external_ref.get("referenceLocator")) + if purl: + purls[purl] = spdx_id + return purls + + +def _add_spdx_relationship( + sbom: dict[str, typing.Any], + relationship: tuple[str, str, str], +) -> None: + relationships = sbom.setdefault("relationships", []) + if not isinstance(relationships, list): + return + existing = { + ( + item.get("spdxElementId"), + item.get("relationshipType"), + item.get("relatedSpdxElement"), + ) + for item in relationships + if isinstance(item, dict) + } + if relationship not in existing: + relationships.append( + { + "spdxElementId": relationship[0], + "relationshipType": relationship[1], + "relatedSpdxElement": relationship[2], + } + ) + + +def merge_cyclonedx_sboms( + *, + sbom: dict[str, typing.Any], + sboms_dir: pathlib.Path, +) -> None: + """Merge CycloneDX components into a Fromager SPDX document. + + Imported components are related to the wheel with ``CONTAINS``. Local file + download qualifiers are removed from their PURLs. The original CycloneDX + files are only read and remain alongside ``fromager.spdx.json``. + """ + if not sboms_dir.is_dir(): + return + + purl_to_spdx_id = _purl_to_spdx_id(sbom) + identity_to_spdx_id: dict[str, str] = {} + packages = sbom.setdefault("packages", []) + if not isinstance(packages, list): + return + + for sbom_path in sorted(sboms_dir.iterdir()): + if not sbom_path.is_file() or sbom_path.name == SBOM_FILENAME: + continue + try: + with sbom_path.open(encoding="utf-8") as sbom_file: + cyclonedx = json.load(sbom_file) + except (OSError, json.JSONDecodeError) as err: + logger.warning("could not read SBOM file %s: %s", sbom_path, err) + continue + + if not isinstance(cyclonedx, dict) or cyclonedx.get("bomFormat") != "CycloneDX": + continue + + metadata = cyclonedx.get("metadata") + root = metadata.get("component") if isinstance(metadata, dict) else None + components: list[tuple[dict[str, typing.Any], bool]] = [] + if isinstance(root, dict): + components.extend( + (component, component is root) + for component in _iter_cyclonedx_components(root) + ) + raw_components = cyclonedx.get("components") + if isinstance(raw_components, list): + for component in raw_components: + if isinstance(component, dict): + components.extend( + (nested_component, False) + for nested_component in _iter_cyclonedx_components(component) + ) + + for component, is_root in components: + if _cyclonedx_string(component.get("scope")) == "excluded": + continue + + identity = _cyclonedx_component_identity(component) + purl = _cyclonedx_spdx_purl(component) + spdx_id: str | None + if is_root and _is_python_wheel_root(component, sbom): + spdx_id = "SPDXRef-wheel" + if purl: + purl_to_spdx_id[purl] = spdx_id + else: + spdx_id = identity_to_spdx_id.get(identity) + if purl: + mapped_spdx_id = purl_to_spdx_id.get(purl) + if mapped_spdx_id is not None: + spdx_id = mapped_spdx_id + if spdx_id is None: + spdx_id = _cyclonedx_spdx_id(component) + packages.append(_cyclonedx_package(component, spdx_id)) + if purl: + purl_to_spdx_id[purl] = spdx_id + else: + package = next( + package + for package in packages + if package.get("SPDXID") == spdx_id + ) + _merge_cyclonedx_package(package, component) + + identity_to_spdx_id[identity] = spdx_id + if spdx_id != "SPDXRef-wheel": + _add_spdx_relationship( + sbom, + ("SPDXRef-wheel", "CONTAINS", spdx_id), + ) + + def write_sbom( *, sbom: dict[str, typing.Any], diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index e442ffed..dc0a0984 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -357,6 +357,10 @@ def add_extra_metadata_to_wheels( # identifies the wheel that actually contains it wheel_version=dist_version, ) + sbom.merge_cyclonedx_sboms( + sbom=sbom_doc, + sboms_dir=dist_info_dir / "sboms", + ) sbom.write_sbom(sbom=sbom_doc, dist_info_dir=dist_info_dir) build_tag = get_build_tag( diff --git a/tests/test_sbom.py b/tests/test_sbom.py index ddfa1184..60ddb6c6 100644 --- a/tests/test_sbom.py +++ b/tests/test_sbom.py @@ -10,6 +10,7 @@ from packaging.metadata import RawMetadata from packaging.requirements import Requirement from packaging.version import Version +from pydantic import AnyUrl from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import ( JsonLikeDictParser, ) @@ -33,6 +34,58 @@ def _metadata(**fields: typing.Any) -> RawMetadata: return raw +def _write_cyclonedx_sbom( + sboms_dir: pathlib.Path, + filename: str, + *, + root_name: str = "example", + root_version: str = "1.0.0", +) -> None: + """Write a representative Maturin CycloneDX 1.5 document.""" + root_purl = f"pkg:cargo/{root_name}@{root_version}" + serde_purl = "pkg:cargo/serde@1.0.0" + pyo3_purl = "pkg:cargo/pyo3@0.21.0" + document = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "serialNumber": f"urn:uuid:{root_name}", + "metadata": { + "tools": [{"vendor": "PyO3", "name": "maturin", "version": "1.15.0"}], + "component": { + "type": "library", + "bom-ref": root_purl, + "name": root_name, + "version": root_version, + "purl": root_purl, + }, + }, + "components": [ + { + "type": "library", + "bom-ref": serde_purl, + "name": "serde", + "version": "1.0.0", + "scope": "required", + "purl": serde_purl, + "hashes": [{"alg": "SHA-256", "content": "a" * 64}], + "licenses": [{"license": {"id": "MIT"}}], + }, + { + "type": "library", + "bom-ref": pyo3_purl, + "name": "pyo3", + "version": "0.21.0", + "scope": "excluded", + "purl": pyo3_purl, + "licenses": [{"expression": "Apache-2.0"}], + }, + ], + "dependencies": [{"ref": root_purl, "dependsOn": [serde_purl, pyo3_purl]}], + } + sboms_dir.mkdir(parents=True, exist_ok=True) + (sboms_dir / filename).write_text(json.dumps(document)) + + def test_generate_sbom_structure(tmp_path: pathlib.Path) -> None: """Verify the generated SBOM has the required SPDX 2.3 fields.""" ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) @@ -799,3 +852,328 @@ def test_generate_sbom_license_with_unparsed_field(tmp_path: pathlib.Path) -> No + " Unparseable fields: Classifier." ) _validate_spdx(doc) + + +def test_merge_cyclonedx_sbom_imports_packages_and_relationships( + tmp_path: pathlib.Path, +) -> None: + """Verify Maturin CycloneDX data is represented in SPDX.""" + ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + document = sbom.generate_sbom( + ctx=ctx, + req=Requirement("example-pkg==1.2.3"), + version=Version("1.2.3"), + metadata=_metadata(), + ) + sboms_dir = tmp_path / "sboms" + _write_cyclonedx_sbom(sboms_dir, "example.cyclonedx.json") + + sbom.merge_cyclonedx_sboms(sbom=document, sboms_dir=sboms_dir) + + packages = {package["name"]: package for package in document["packages"]} + assert {"example-pkg", "example", "serde"} <= packages.keys() + assert "pyo3" not in packages + serde = packages["serde"] + assert serde["externalRefs"][0]["referenceLocator"] == "pkg:cargo/serde@1.0.0" + assert serde["checksums"] == [{"algorithm": "SHA256", "checksumValue": "a" * 64}] + assert serde["licenseDeclared"] == "MIT" + assert "comment" not in serde + spdx_ids = {package["name"]: package["SPDXID"] for package in document["packages"]} + relationships = { + ( + relationship["spdxElementId"], + relationship["relationshipType"], + relationship["relatedSpdxElement"], + ) + for relationship in document["relationships"] + } + assert ("SPDXRef-wheel", "CONTAINS", spdx_ids["example"]) in relationships + assert ("SPDXRef-wheel", "CONTAINS", spdx_ids["serde"]) in relationships + assert not any( + relationship_type == "DEPENDS_ON" for _, relationship_type, _ in relationships + ) + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_maps_python_root_to_wheel( + tmp_path: pathlib.Path, +) -> None: + """Verify an auditwheel Python root is attached to the wheel package.""" + settings = SbomSettings(repository_url=AnyUrl("https://packages.redhat.com")) + ctx = make_sbom_ctx(tmp_path, sbom_settings=settings) + document = sbom.generate_sbom( + ctx=ctx, + req=Requirement("pillow==12.3.0"), + version=Version("12.3.0"), + metadata=_metadata(), + ) + sboms_dir = tmp_path / "sboms" + auditwheel = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": { + "component": { + "type": "library", + "bom-ref": "pkg:pypi/pillow@12.3.0?file_name=pillow.whl", + "name": "Pillow", + "version": "12.3.0", + "purl": "pkg:pypi/pillow@12.3.0?file_name=pillow.whl", + } + }, + "components": [ + { + "type": "library", + "bom-ref": "pkg:rpm/almalinux/libXau@1.0.9-3.el8", + "name": "libXau", + "version": "1.0.9-3.el8", + "purl": "pkg:rpm/almalinux/libXau@1.0.9-3.el8", + } + ], + "dependencies": [ + { + "ref": "pkg:pypi/pillow@12.3.0?file_name=pillow.whl", + "dependsOn": ["pkg:rpm/almalinux/libXau@1.0.9-3.el8"], + } + ], + } + sboms_dir.mkdir(parents=True) + (sboms_dir / "auditwheel.cdx.json").write_text(json.dumps(auditwheel)) + + sbom.merge_cyclonedx_sboms(sbom=document, sboms_dir=sboms_dir) + + packages = document["packages"] + assert len(packages) == 3 + rpm = next(package for package in packages if package["name"] == "libXau") + relationships = { + ( + relationship["spdxElementId"], + relationship["relationshipType"], + relationship["relatedSpdxElement"], + ) + for relationship in document["relationships"] + } + assert ("SPDXRef-wheel", "CONTAINS", rpm["SPDXID"]) in relationships + assert not any( + relationship_type == "DEPENDS_ON" for _, relationship_type, _ in relationships + ) + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_does_not_map_unrelated_python_root_to_wheel( + tmp_path: pathlib.Path, +) -> None: + """Verify an unrelated PyPI root remains its own SPDX package.""" + ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + document = sbom.generate_sbom( + ctx=ctx, + req=Requirement("pillow==12.3.0"), + version=Version("12.3.0"), + metadata=_metadata(), + ) + root_purl = "pkg:pypi/unrelated@2.0" + dependency_purl = "pkg:rpm/almalinux/libXau@1.0.9-3.el8" + cyclonedx = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": { + "component": { + "type": "library", + "bom-ref": root_purl, + "name": "unrelated", + "version": "2.0", + "purl": root_purl, + } + }, + "components": [ + { + "type": "library", + "bom-ref": dependency_purl, + "name": "libXau", + "version": "1.0.9-3.el8", + "purl": dependency_purl, + } + ], + "dependencies": [ + {"ref": root_purl, "dependsOn": [dependency_purl]}, + ], + } + sboms_dir = tmp_path / "sboms" + sboms_dir.mkdir() + (sboms_dir / "custom.cyclonedx.json").write_text(json.dumps(cyclonedx)) + + sbom.merge_cyclonedx_sboms(sbom=document, sboms_dir=sboms_dir) + + packages = {package["name"]: package for package in document["packages"]} + assert {"unrelated", "libXau"} <= packages.keys() + assert packages["unrelated"]["SPDXID"] != "SPDXRef-wheel" + relationships = { + ( + relationship["spdxElementId"], + relationship["relationshipType"], + relationship["relatedSpdxElement"], + ) + for relationship in document["relationships"] + } + assert ( + "SPDXRef-wheel", + "CONTAINS", + packages["unrelated"]["SPDXID"], + ) in relationships + assert ( + "SPDXRef-wheel", + "CONTAINS", + packages["libXau"]["SPDXID"], + ) in relationships + assert not any( + relationship_type == "DEPENDS_ON" for _, relationship_type, _ in relationships + ) + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_imports_nested_components( + tmp_path: pathlib.Path, +) -> None: + """Verify nested target components are imported and linked.""" + ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + document = sbom.generate_sbom( + ctx=ctx, + req=Requirement("example-pkg==1.2.3"), + version=Version("1.2.3"), + metadata=_metadata(), + ) + root_purl = "pkg:cargo/example@1.0.0?download_url=file://." + target_purl = "pkg:cargo/example-target@1.0.0?download_url=file://../target" + nested_purl = "pkg:cargo/example-nested@1.0.0" + cyclonedx = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": { + "component": { + "type": "application", + "bom-ref": root_purl, + "name": "example", + "version": "1.0.0", + "purl": root_purl, + } + }, + "components": [ + { + "type": "library", + "bom-ref": target_purl, + "name": "example-target", + "version": "1.0.0", + "purl": target_purl, + "components": [ + { + "type": "library", + "bom-ref": nested_purl, + "name": "example-nested", + "version": "1.0.0", + "purl": nested_purl, + } + ], + } + ], + "dependencies": [ + {"ref": root_purl, "dependsOn": [target_purl]}, + {"ref": target_purl, "dependsOn": [nested_purl]}, + ], + } + sboms_dir = tmp_path / "sboms" + sboms_dir.mkdir() + (sboms_dir / "nested.cyclonedx.json").write_text(json.dumps(cyclonedx)) + + sbom.merge_cyclonedx_sboms(sbom=document, sboms_dir=sboms_dir) + + packages = {package["name"]: package for package in document["packages"]} + assert {"example", "example-target", "example-nested"} <= packages.keys() + relationships = { + ( + relationship["spdxElementId"], + relationship["relationshipType"], + relationship["relatedSpdxElement"], + ) + for relationship in document["relationships"] + } + assert { + package["externalRefs"][0]["referenceLocator"] + for package in ( + packages["example"], + packages["example-target"], + packages["example-nested"], + ) + } == { + "pkg:cargo/example@1.0.0", + "pkg:cargo/example-target@1.0.0", + "pkg:cargo/example-nested@1.0.0", + } + for package_name in ("example", "example-target", "example-nested"): + assert ( + "SPDXRef-wheel", + "CONTAINS", + packages[package_name]["SPDXID"], + ) in relationships + assert not any( + relationship_type == "DEPENDS_ON" for _, relationship_type, _ in relationships + ) + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_deduplicates_packages_by_purl( + tmp_path: pathlib.Path, +) -> None: + """Verify repeated Cargo components produce one SPDX package.""" + ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + document = sbom.generate_sbom( + ctx=ctx, + req=Requirement("example-pkg==1.2.3"), + version=Version("1.2.3"), + metadata=_metadata(), + ) + sboms_dir = tmp_path / "sboms" + _write_cyclonedx_sbom(sboms_dir, "first.cyclonedx.json") + _write_cyclonedx_sbom( + sboms_dir, + "second.cyclonedx.json", + root_name="another", + ) + + sbom.merge_cyclonedx_sboms(sbom=document, sboms_dir=sboms_dir) + + serde_packages = [ + package for package in document["packages"] if package["name"] == "serde" + ] + assert len(serde_packages) == 1 + assert ( + sum( + relationship["relationshipType"] == "CONTAINS" + and relationship["relatedSpdxElement"] == serde_packages[0]["SPDXID"] + for relationship in document["relationships"] + ) + == 1 + ) + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_ignores_invalid_and_other_formats( + tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture +) -> None: + """Verify unrelated or unreadable SBOM files do not break the merge.""" + ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + document = sbom.generate_sbom( + ctx=ctx, + req=Requirement("example-pkg==1.2.3"), + version=Version("1.2.3"), + metadata=_metadata(), + ) + sboms_dir = tmp_path / "sboms" + sboms_dir.mkdir() + (sboms_dir / "not-an-sbom.json").write_text('{"format": "other"}') + (sboms_dir / "broken.json").write_text("{") + + with caplog.at_level("WARNING", logger="fromager.sbom"): + sbom.merge_cyclonedx_sboms(sbom=document, sboms_dir=sboms_dir) + + assert len(document["packages"]) == 2 + assert "could not read" in caplog.text + _validate_spdx(document) diff --git a/tests/test_wheels.py b/tests/test_wheels.py index f9fd22f0..fb14557b 100644 --- a/tests/test_wheels.py +++ b/tests/test_wheels.py @@ -418,6 +418,34 @@ def test_add_extra_metadata_generates_sbom_when_enabled( wheel_dir.mkdir() wheel_file = wheel_dir / "test_pkg-1.0.0+local.1-py3-none-any.whl" + native_sbom = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": { + "component": { + "type": "library", + "bom-ref": "pkg:cargo/test-pkg@1.0.0", + "name": "test-pkg", + "version": "1.0.0", + "purl": "pkg:cargo/test-pkg@1.0.0", + } + }, + "components": [ + { + "type": "library", + "bom-ref": "pkg:cargo/serde@1.0.0", + "name": "serde", + "version": "1.0.0", + "purl": "pkg:cargo/serde@1.0.0", + } + ], + "dependencies": [ + { + "ref": "pkg:cargo/test-pkg@1.0.0", + "dependsOn": ["pkg:cargo/serde@1.0.0"], + } + ], + } with zipfile.ZipFile(wheel_file, "w") as zf: zf.writestr("test_pkg/__init__.py", "") zf.writestr( @@ -428,6 +456,10 @@ def test_add_extra_metadata_generates_sbom_when_enabled( "test_pkg-1.0.0+local.1.dist-info/WHEEL", "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", ) + zf.writestr( + "test_pkg-1.0.0+local.1.dist-info/sboms/rust.cyclonedx.json", + json.dumps(native_sbom), + ) mock_run.return_value = "" @@ -438,22 +470,14 @@ def test_add_extra_metadata_generates_sbom_when_enabled( sdist_dir.mkdir() # Capture the wheel contents before repack by inspecting what wheel pack receives - captured_contents: list[str] = [] - captured_sbom: dict[str, typing.Any] = {} + captured_contents: dict[str, str] = {} def fake_run(cmd: list[str], **kwargs: object) -> str: # wheel pack is called with the unpacked dir as second arg unpacked_dir = pathlib.Path(cmd[2]) for f in unpacked_dir.rglob("*"): if f.is_file(): - captured_contents.append(str(f.relative_to(unpacked_dir))) - sbom_file = ( - unpacked_dir - / "test_pkg-1.0.0+local.1.dist-info" - / "sboms" - / ("fromager.spdx.json") - ) - captured_sbom.update(json.loads(sbom_file.read_text())) + captured_contents[str(f.relative_to(unpacked_dir))] = f.read_text() repacked.touch() return "" @@ -468,12 +492,25 @@ def fake_run(cmd: list[str], **kwargs: object) -> str: wheel_file=wheel_file, ) - # Verify the SBOM file was added to the unpacked wheel before repacking - assert any("sboms/fromager.spdx.json" in c for c in captured_contents) + # Verify the canonical SBOM was merged and the native file was preserved. + assert ( + "test_pkg-1.0.0+local.1.dist-info/sboms/fromager.spdx.json" in captured_contents + ) + assert ( + "test_pkg-1.0.0+local.1.dist-info/sboms/rust.cyclonedx.json" + in captured_contents + ) + merged_sbom = json.loads( + captured_contents["test_pkg-1.0.0+local.1.dist-info/sboms/fromager.spdx.json"] + ) + assert {package["name"] for package in merged_sbom["packages"]} >= { + "test-pkg", + "serde", + } # The SBOM must identify the wheel with its local version - assert captured_sbom["name"] == "test-pkg-1.0.0+local.1" - wheel_entry, upstream_entry = captured_sbom["packages"] + assert merged_sbom["name"] == "test-pkg-1.0.0+local.1" + wheel_entry, upstream_entry = merged_sbom["packages"][:2] assert wheel_entry["versionInfo"] == "1.0.0+local.1" assert wheel_entry["externalRefs"][0]["referenceLocator"] == ( "pkg:pypi/test-pkg@1.0.0%2Blocal.1" From 17cc0c025ee6f98e3ca94410999de042dac4aef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Prpi=C4=8D?= Date: Wed, 30 Sep 2026 13:34:09 -0400 Subject: [PATCH 2/6] fix(sbom): validate CycloneDX data before merging into SPDX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the SHA3-224 hash mapping (not an SPDX 2.3 algorithm), validate license expressions with the SPDX licensing parser, and skip non-hexadecimal checksum content so merged CycloneDX components cannot produce an invalid canonical SBOM. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Martin Prpič --- src/fromager/sbom.py | 46 +++++++++++++++++++++++++++++++++------- tests/test_sbom.py | 50 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 8 deletions(-) diff --git a/src/fromager/sbom.py b/src/fromager/sbom.py index a5d83103..2bffa406 100644 --- a/src/fromager/sbom.py +++ b/src/fromager/sbom.py @@ -18,6 +18,7 @@ from datetime import UTC, datetime from urllib.parse import quote +from license_expression import ExpressionError, get_spdx_licensing from packageurl import PackageURL from packaging.metadata import RawMetadata from packaging.requirements import Requirement @@ -344,7 +345,7 @@ def generate_sbom( "SHA-256": "SHA256", "SHA-384": "SHA384", "SHA-512": "SHA512", - "SHA3-224": "SHA3-224", + # SPDX 2.3 has no SHA3-224 algorithm, so it is intentionally omitted. "SHA3-256": "SHA3-256", "SHA3-384": "SHA3-384", "SHA3-512": "SHA3-512", @@ -353,6 +354,12 @@ def generate_sbom( "BLAKE2B-512": "BLAKE2b-512", } +# Reused across components; parsing SPDX license expressions is relatively costly. +_SPDX_LICENSING = get_spdx_licensing() + +# SPDX checksum values must be lowercase or uppercase hexadecimal digits. +_HEX_RE = re.compile(r"[0-9a-fA-F]+") + def _cyclonedx_string(value: typing.Any) -> str | None: if isinstance(value, str) and value: @@ -489,20 +496,35 @@ def _cyclonedx_license_expression( if not isinstance(license_choice, dict): continue expression = _cyclonedx_string(license_choice.get("expression")) - if expression: - expressions.append(expression) + if not expression: + license_info = license_choice.get("license") + if isinstance(license_info, dict): + expression = _cyclonedx_string(license_info.get("id")) + if not expression: continue - license_info = license_choice.get("license") - if isinstance(license_info, dict): - identifier = _cyclonedx_string(license_info.get("id")) - if identifier: - expressions.append(identifier) + if _valid_spdx_expression(expression): + expressions.append(expression) + else: + logger.warning( + "component %s has an invalid SPDX license expression %r; skipping it", + _cyclonedx_component_purl(component) or component.get("name"), + expression, + ) if not expressions: return None return " AND ".join(expressions) +def _valid_spdx_expression(expression: str) -> bool: + """Return True if *expression* is a valid SPDX license expression.""" + try: + _SPDX_LICENSING.parse(expression, validate=True) + except ExpressionError: + return False + return True + + def _cyclonedx_checksums( component: dict[str, typing.Any], ) -> list[dict[str, str]]: @@ -521,6 +543,14 @@ def _cyclonedx_checksums( normalized_algorithm = _CYCLONEDX_HASH_TO_SPDX.get(algorithm.upper()) if normalized_algorithm is None: continue + if not _HEX_RE.fullmatch(content): + logger.warning( + "component %s has a non-hexadecimal %s checksum %r; skipping it", + _cyclonedx_component_purl(component) or component.get("name"), + normalized_algorithm, + content, + ) + continue checksums.append({"algorithm": normalized_algorithm, "checksumValue": content}) return checksums diff --git a/tests/test_sbom.py b/tests/test_sbom.py index 60ddb6c6..03e310fe 100644 --- a/tests/test_sbom.py +++ b/tests/test_sbom.py @@ -895,6 +895,56 @@ def test_merge_cyclonedx_sbom_imports_packages_and_relationships( _validate_spdx(document) +def test_cyclonedx_checksums_skips_unsupported_algorithm() -> None: + """SHA3-224 has no SPDX 2.3 equivalent and must be dropped.""" + component = { + "name": "serde", + "hashes": [ + {"alg": "SHA3-224", "content": "a" * 56}, + {"alg": "SHA-256", "content": "b" * 64}, + ], + } + + checksums = sbom._cyclonedx_checksums(component) + + assert checksums == [{"algorithm": "SHA256", "checksumValue": "b" * 64}] + + +def test_cyclonedx_checksums_skips_non_hex_content() -> None: + """Non-hexadecimal checksum content would fail SPDX validation.""" + component = { + "name": "serde", + "hashes": [ + {"alg": "SHA-256", "content": "not-a-valid-hash"}, + {"alg": "SHA-512", "content": "c" * 128}, + ], + } + + checksums = sbom._cyclonedx_checksums(component) + + assert checksums == [{"algorithm": "SHA512", "checksumValue": "c" * 128}] + + +def test_cyclonedx_license_expression_skips_invalid() -> None: + """Invalid SPDX expressions must not be copied into licenseDeclared.""" + component = { + "name": "serde", + "licenses": [ + {"expression": "MIT"}, + {"expression": "Totally Not A License"}, + ], + } + + assert sbom._cyclonedx_license_expression(component) == "MIT" + + +def test_cyclonedx_license_expression_all_invalid_returns_none() -> None: + """A component with only invalid expressions declares no license.""" + component = {"name": "serde", "licenses": [{"license": {"id": "Bogus-9.9"}}]} + + assert sbom._cyclonedx_license_expression(component) is None + + def test_merge_cyclonedx_sboms_maps_python_root_to_wheel( tmp_path: pathlib.Path, ) -> None: From d2581e687950484943d8c7191cea8d60bf2aa93b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Prpi=C4=8D?= Date: Thu, 15 Oct 2026 09:14:53 -0400 Subject: [PATCH 3/6] fix(sbom): preserve license expression grouping when merging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parenthesize CycloneDX license expressions containing OR before joining them with AND: SPDX gives AND higher precedence than OR, so combining "MIT OR Apache-2.0" with "BSD-3-Clause" previously produced "MIT OR Apache-2.0 AND BSD-3-Clause", which weakens the declaration. Also use a .test host for the repository_url test fixture per the coding guidelines. Co-Authored-By: GLM-5.3 Signed-off-by: Martin Prpič --- src/fromager/sbom.py | 7 ++++++- tests/test_sbom.py | 18 +++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/fromager/sbom.py b/src/fromager/sbom.py index 2bffa406..cb48b196 100644 --- a/src/fromager/sbom.py +++ b/src/fromager/sbom.py @@ -513,7 +513,12 @@ def _cyclonedx_license_expression( if not expressions: return None - return " AND ".join(expressions) + # Parenthesize OR expressions before joining: SPDX gives AND higher + # precedence than OR, so an ungrouped OR would weaken the declaration. + return " AND ".join( + f"({expression})" if " OR " in expression else expression + for expression in expressions + ) def _valid_spdx_expression(expression: str) -> bool: diff --git a/tests/test_sbom.py b/tests/test_sbom.py index 03e310fe..47a99b0d 100644 --- a/tests/test_sbom.py +++ b/tests/test_sbom.py @@ -945,11 +945,27 @@ def test_cyclonedx_license_expression_all_invalid_returns_none() -> None: assert sbom._cyclonedx_license_expression(component) is None +def test_cyclonedx_license_expression_groups_or_operands() -> None: + """OR expressions must be parenthesized so AND-joining cannot weaken them.""" + component = { + "name": "serde", + "licenses": [ + {"expression": "MIT OR Apache-2.0"}, + {"expression": "BSD-3-Clause"}, + ], + } + + assert ( + sbom._cyclonedx_license_expression(component) + == "(MIT OR Apache-2.0) AND BSD-3-Clause" + ) + + def test_merge_cyclonedx_sboms_maps_python_root_to_wheel( tmp_path: pathlib.Path, ) -> None: """Verify an auditwheel Python root is attached to the wheel package.""" - settings = SbomSettings(repository_url=AnyUrl("https://packages.redhat.com")) + settings = SbomSettings(repository_url=AnyUrl("https://pkg.test/simple/")) ctx = make_sbom_ctx(tmp_path, sbom_settings=settings) document = sbom.generate_sbom( ctx=ctx, From 9b054848c9a27197591131a5d197458297c2be7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Prpi=C4=8D?= Date: Mon, 19 Oct 2026 11:33:16 +0100 Subject: [PATCH 4/6] fix(sbom): match CycloneDX root against the real wheel identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_is_python_wheel_root()` compared the CycloneDX root to the name and `versionInfo` of the SPDX wheel entry, which carry `purl.name` and `purl.version` overrides. Build tools report the real package identity, so with an override set the root no longer matched, was imported as a separate package, and the SBOM said the wheel contains itself. Pass the requirement name and the wheel file name version to `merge_cyclonedx_sboms()` and compare the root against those instead. Add a regression test with a `purl.version` override. Co-Authored-By: Claude Signed-off-by: Martin Prpič --- src/fromager/sbom.py | 45 ++++++++++----------- src/fromager/wheels.py | 2 + tests/test_sbom.py | 88 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 105 insertions(+), 30 deletions(-) diff --git a/src/fromager/sbom.py b/src/fromager/sbom.py index cb48b196..6b61354d 100644 --- a/src/fromager/sbom.py +++ b/src/fromager/sbom.py @@ -416,8 +416,16 @@ def _versions_match(left: str, right: str) -> bool: def _is_python_wheel_root( component: dict[str, typing.Any], - sbom: dict[str, typing.Any], + name: str, + version: str, ) -> bool: + """Return True if the CycloneDX root *component* is the wheel itself. + + The root is matched against the real package identity (*name* and + *version* of the built wheel), not the SPDX wheel entry, because + ``purl`` overrides change the entry but not what the build tool + reports. + """ purl = _cyclonedx_component_purl(component) if purl is None: return False @@ -437,29 +445,11 @@ def _is_python_wheel_root( if component_name is None or component_version is None: return False - packages = sbom.get("packages") - if not isinstance(packages, list): - return False - wheel = next( - ( - package - for package in packages - if isinstance(package, dict) and package.get("SPDXID") == "SPDXRef-wheel" - ), - None, - ) - if wheel is None: - return False - - wheel_name = _cyclonedx_string(wheel.get("name")) - wheel_version = _cyclonedx_string(wheel.get("versionInfo")) - if wheel_name is None or wheel_version is None: - return False return ( - canonicalize_name(parsed_purl.name) == canonicalize_name(wheel_name) - and _versions_match(parsed_purl.version, wheel_version) - and canonicalize_name(component_name) == canonicalize_name(wheel_name) - and _versions_match(component_version, wheel_version) + canonicalize_name(parsed_purl.name) == canonicalize_name(name) + and _versions_match(parsed_purl.version, version) + and canonicalize_name(component_name) == canonicalize_name(name) + and _versions_match(component_version, version) ) @@ -687,9 +677,16 @@ def merge_cyclonedx_sboms( *, sbom: dict[str, typing.Any], sboms_dir: pathlib.Path, + name: str, + version: str, ) -> None: """Merge CycloneDX components into a Fromager SPDX document. + *name* and *version* are the real identity of the built wheel (the + requirement name and the version from the wheel file name). They are + used to recognize a build tool's root component as the wheel itself, + independent of any ``purl`` overrides in the SPDX wheel entry. + Imported components are related to the wheel with ``CONTAINS``. Local file download qualifiers are removed from their PURLs. The original CycloneDX files are only read and remain alongside ``fromager.spdx.json``. @@ -740,7 +737,7 @@ def merge_cyclonedx_sboms( identity = _cyclonedx_component_identity(component) purl = _cyclonedx_spdx_purl(component) spdx_id: str | None - if is_root and _is_python_wheel_root(component, sbom): + if is_root and _is_python_wheel_root(component, name, version): spdx_id = "SPDXRef-wheel" if purl: purl_to_spdx_id[purl] = spdx_id diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index dc0a0984..e2f1a210 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -360,6 +360,8 @@ def add_extra_metadata_to_wheels( sbom.merge_cyclonedx_sboms( sbom=sbom_doc, sboms_dir=dist_info_dir / "sboms", + name=req.name, + version=str(dist_version), ) sbom.write_sbom(sbom=sbom_doc, dist_info_dir=dist_info_dir) diff --git a/tests/test_sbom.py b/tests/test_sbom.py index 47a99b0d..d3c35d0f 100644 --- a/tests/test_sbom.py +++ b/tests/test_sbom.py @@ -868,7 +868,9 @@ def test_merge_cyclonedx_sbom_imports_packages_and_relationships( sboms_dir = tmp_path / "sboms" _write_cyclonedx_sbom(sboms_dir, "example.cyclonedx.json") - sbom.merge_cyclonedx_sboms(sbom=document, sboms_dir=sboms_dir) + sbom.merge_cyclonedx_sboms( + sbom=document, sboms_dir=sboms_dir, name="example-pkg", version="1.2.3" + ) packages = {package["name"]: package for package in document["packages"]} assert {"example-pkg", "example", "serde"} <= packages.keys() @@ -1005,7 +1007,9 @@ def test_merge_cyclonedx_sboms_maps_python_root_to_wheel( sboms_dir.mkdir(parents=True) (sboms_dir / "auditwheel.cdx.json").write_text(json.dumps(auditwheel)) - sbom.merge_cyclonedx_sboms(sbom=document, sboms_dir=sboms_dir) + sbom.merge_cyclonedx_sboms( + sbom=document, sboms_dir=sboms_dir, name="pillow", version="12.3.0" + ) packages = document["packages"] assert len(packages) == 3 @@ -1025,6 +1029,70 @@ def test_merge_cyclonedx_sboms_maps_python_root_to_wheel( _validate_spdx(document) +def test_merge_cyclonedx_sboms_maps_python_root_with_purl_version_override( + tmp_path: pathlib.Path, +) -> None: + """Verify a purl version override does not stop the root matching.""" + ctx = make_sbom_ctx( + tmp_path, + sbom_settings=SbomSettings(), + package_overrides={"purl": {"version": "0.5.5+local.1"}}, + ) + document = sbom.generate_sbom( + ctx=ctx, + req=Requirement("test-pkg==0.5.5"), + version=Version("0.5.5"), + ) + wheel = next( + package + for package in document["packages"] + if package["SPDXID"] == "SPDXRef-wheel" + ) + assert wheel["versionInfo"] == "0.5.5+local.1" + root_purl = "pkg:pypi/test-pkg@0.5.5" + maturin = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": { + "component": { + "type": "library", + "bom-ref": root_purl, + "name": "test-pkg", + "version": "0.5.5", + "purl": root_purl, + } + }, + "components": [ + { + "type": "library", + "bom-ref": "pkg:cargo/serde@1.0.0", + "name": "serde", + "version": "1.0.0", + "purl": "pkg:cargo/serde@1.0.0", + } + ], + } + sboms_dir = tmp_path / "sboms" + sboms_dir.mkdir() + (sboms_dir / "maturin.cdx.json").write_text(json.dumps(maturin)) + + sbom.merge_cyclonedx_sboms( + sbom=document, sboms_dir=sboms_dir, name="test-pkg", version="0.5.5" + ) + + assert len(document["packages"]) == 3 + serde = next( + package for package in document["packages"] if package["name"] == "serde" + ) + contains = { + relationship["relatedSpdxElement"] + for relationship in document["relationships"] + if relationship["relationshipType"] == "CONTAINS" + } + assert contains == {serde["SPDXID"]} + _validate_spdx(document) + + def test_merge_cyclonedx_sboms_does_not_map_unrelated_python_root_to_wheel( tmp_path: pathlib.Path, ) -> None: @@ -1067,7 +1135,9 @@ def test_merge_cyclonedx_sboms_does_not_map_unrelated_python_root_to_wheel( sboms_dir.mkdir() (sboms_dir / "custom.cyclonedx.json").write_text(json.dumps(cyclonedx)) - sbom.merge_cyclonedx_sboms(sbom=document, sboms_dir=sboms_dir) + sbom.merge_cyclonedx_sboms( + sbom=document, sboms_dir=sboms_dir, name="example-pkg", version="1.2.3" + ) packages = {package["name"]: package for package in document["packages"]} assert {"unrelated", "libXau"} <= packages.keys() @@ -1149,7 +1219,9 @@ def test_merge_cyclonedx_sboms_imports_nested_components( sboms_dir.mkdir() (sboms_dir / "nested.cyclonedx.json").write_text(json.dumps(cyclonedx)) - sbom.merge_cyclonedx_sboms(sbom=document, sboms_dir=sboms_dir) + sbom.merge_cyclonedx_sboms( + sbom=document, sboms_dir=sboms_dir, name="example-pkg", version="1.2.3" + ) packages = {package["name"]: package for package in document["packages"]} assert {"example", "example-target", "example-nested"} <= packages.keys() @@ -1204,7 +1276,9 @@ def test_merge_cyclonedx_sboms_deduplicates_packages_by_purl( root_name="another", ) - sbom.merge_cyclonedx_sboms(sbom=document, sboms_dir=sboms_dir) + sbom.merge_cyclonedx_sboms( + sbom=document, sboms_dir=sboms_dir, name="example-pkg", version="1.2.3" + ) serde_packages = [ package for package in document["packages"] if package["name"] == "serde" @@ -1238,7 +1312,9 @@ def test_merge_cyclonedx_sboms_ignores_invalid_and_other_formats( (sboms_dir / "broken.json").write_text("{") with caplog.at_level("WARNING", logger="fromager.sbom"): - sbom.merge_cyclonedx_sboms(sbom=document, sboms_dir=sboms_dir) + sbom.merge_cyclonedx_sboms( + sbom=document, sboms_dir=sboms_dir, name="example-pkg", version="1.2.3" + ) assert len(document["packages"]) == 2 assert "could not read" in caplog.text From daba35e6dd447cc531f705d5b566680ac4146b0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Prpi=C4=8D?= Date: Tue, 10 Nov 2026 16:25:41 -0500 Subject: [PATCH 5/6] fix(sbom): find the wheel package through the DESCRIBES relationship MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Package SPDX ids now carry a per-document suffix, so the merge can no longer attach components to a fixed `SPDXRef-wheel` id. Look up the wheel's id through the document's DESCRIBES relationship instead, as the `generate_sbom()` docstring recommends. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Martin Prpič --- src/fromager/sbom.py | 30 +++++++++++++++++++++++++++--- tests/test_sbom.py | 23 +++++++++++++---------- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/src/fromager/sbom.py b/src/fromager/sbom.py index 6b61354d..a29ad61b 100644 --- a/src/fromager/sbom.py +++ b/src/fromager/sbom.py @@ -647,6 +647,25 @@ def _purl_to_spdx_id(sbom: dict[str, typing.Any]) -> dict[str, str]: return purls +def _described_spdx_id(sbom: dict[str, typing.Any]) -> str | None: + """Return the SPDX id of the package the document describes (the wheel). + + Package SPDX ids are not predictable, so the wheel is found through the + document's DESCRIBES relationship. + """ + relationships = sbom.get("relationships") + if not isinstance(relationships, list): + return None + for relationship in relationships: + if ( + isinstance(relationship, dict) + and relationship.get("spdxElementId") == "SPDXRef-DOCUMENT" + and relationship.get("relationshipType") == "DESCRIBES" + ): + return _cyclonedx_string(relationship.get("relatedSpdxElement")) + return None + + def _add_spdx_relationship( sbom: dict[str, typing.Any], relationship: tuple[str, str, str], @@ -694,6 +713,11 @@ def merge_cyclonedx_sboms( if not sboms_dir.is_dir(): return + wheel_spdx_id = _described_spdx_id(sbom) + if wheel_spdx_id is None: + logger.warning("SBOM has no DESCRIBES relationship; skipping CycloneDX merge") + return + purl_to_spdx_id = _purl_to_spdx_id(sbom) identity_to_spdx_id: dict[str, str] = {} packages = sbom.setdefault("packages", []) @@ -738,7 +762,7 @@ def merge_cyclonedx_sboms( purl = _cyclonedx_spdx_purl(component) spdx_id: str | None if is_root and _is_python_wheel_root(component, name, version): - spdx_id = "SPDXRef-wheel" + spdx_id = wheel_spdx_id if purl: purl_to_spdx_id[purl] = spdx_id else: @@ -761,10 +785,10 @@ def merge_cyclonedx_sboms( _merge_cyclonedx_package(package, component) identity_to_spdx_id[identity] = spdx_id - if spdx_id != "SPDXRef-wheel": + if spdx_id != wheel_spdx_id: _add_spdx_relationship( sbom, - ("SPDXRef-wheel", "CONTAINS", spdx_id), + (wheel_spdx_id, "CONTAINS", spdx_id), ) diff --git a/tests/test_sbom.py b/tests/test_sbom.py index d3c35d0f..56dd5c32 100644 --- a/tests/test_sbom.py +++ b/tests/test_sbom.py @@ -865,6 +865,7 @@ def test_merge_cyclonedx_sbom_imports_packages_and_relationships( version=Version("1.2.3"), metadata=_metadata(), ) + wheel_id = sbom._described_spdx_id(document) sboms_dir = tmp_path / "sboms" _write_cyclonedx_sbom(sboms_dir, "example.cyclonedx.json") @@ -889,8 +890,8 @@ def test_merge_cyclonedx_sbom_imports_packages_and_relationships( ) for relationship in document["relationships"] } - assert ("SPDXRef-wheel", "CONTAINS", spdx_ids["example"]) in relationships - assert ("SPDXRef-wheel", "CONTAINS", spdx_ids["serde"]) in relationships + assert (wheel_id, "CONTAINS", spdx_ids["example"]) in relationships + assert (wheel_id, "CONTAINS", spdx_ids["serde"]) in relationships assert not any( relationship_type == "DEPENDS_ON" for _, relationship_type, _ in relationships ) @@ -975,6 +976,7 @@ def test_merge_cyclonedx_sboms_maps_python_root_to_wheel( version=Version("12.3.0"), metadata=_metadata(), ) + wheel_id = sbom._described_spdx_id(document) sboms_dir = tmp_path / "sboms" auditwheel = { "bomFormat": "CycloneDX", @@ -1022,7 +1024,7 @@ def test_merge_cyclonedx_sboms_maps_python_root_to_wheel( ) for relationship in document["relationships"] } - assert ("SPDXRef-wheel", "CONTAINS", rpm["SPDXID"]) in relationships + assert (wheel_id, "CONTAINS", rpm["SPDXID"]) in relationships assert not any( relationship_type == "DEPENDS_ON" for _, relationship_type, _ in relationships ) @@ -1043,10 +1045,9 @@ def test_merge_cyclonedx_sboms_maps_python_root_with_purl_version_override( req=Requirement("test-pkg==0.5.5"), version=Version("0.5.5"), ) + wheel_id = sbom._described_spdx_id(document) wheel = next( - package - for package in document["packages"] - if package["SPDXID"] == "SPDXRef-wheel" + package for package in document["packages"] if package["SPDXID"] == wheel_id ) assert wheel["versionInfo"] == "0.5.5+local.1" root_purl = "pkg:pypi/test-pkg@0.5.5" @@ -1104,6 +1105,7 @@ def test_merge_cyclonedx_sboms_does_not_map_unrelated_python_root_to_wheel( version=Version("12.3.0"), metadata=_metadata(), ) + wheel_id = sbom._described_spdx_id(document) root_purl = "pkg:pypi/unrelated@2.0" dependency_purl = "pkg:rpm/almalinux/libXau@1.0.9-3.el8" cyclonedx = { @@ -1141,7 +1143,7 @@ def test_merge_cyclonedx_sboms_does_not_map_unrelated_python_root_to_wheel( packages = {package["name"]: package for package in document["packages"]} assert {"unrelated", "libXau"} <= packages.keys() - assert packages["unrelated"]["SPDXID"] != "SPDXRef-wheel" + assert packages["unrelated"]["SPDXID"] != wheel_id relationships = { ( relationship["spdxElementId"], @@ -1151,12 +1153,12 @@ def test_merge_cyclonedx_sboms_does_not_map_unrelated_python_root_to_wheel( for relationship in document["relationships"] } assert ( - "SPDXRef-wheel", + wheel_id, "CONTAINS", packages["unrelated"]["SPDXID"], ) in relationships assert ( - "SPDXRef-wheel", + wheel_id, "CONTAINS", packages["libXau"]["SPDXID"], ) in relationships @@ -1177,6 +1179,7 @@ def test_merge_cyclonedx_sboms_imports_nested_components( version=Version("1.2.3"), metadata=_metadata(), ) + wheel_id = sbom._described_spdx_id(document) root_purl = "pkg:cargo/example@1.0.0?download_url=file://." target_purl = "pkg:cargo/example-target@1.0.0?download_url=file://../target" nested_purl = "pkg:cargo/example-nested@1.0.0" @@ -1247,7 +1250,7 @@ def test_merge_cyclonedx_sboms_imports_nested_components( } for package_name in ("example", "example-target", "example-nested"): assert ( - "SPDXRef-wheel", + wheel_id, "CONTAINS", packages[package_name]["SPDXID"], ) in relationships From 979e163192353a466f6ba9aff5ef6b951ca591ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Prpi=C4=8D?= Date: Tue, 10 Nov 2026 16:28:24 -0500 Subject: [PATCH 6/6] fix(sbom): harden CycloneDX merge against invalid and colliding data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reuse only packages the merge created for duplicate components, so build-reported checksums and licenses never land on fromager's own wheel or upstream entries; a wheel root may omit name and version. - Canonicalize license expressions with `canonicalize_license_expression()` (PEP 639 rules), so lowercase operators are grouped correctly and `LicenseRef-` ids are kept. This also drops the import-time `get_spdx_licensing()` call. - Lowercase checksums and check their digest length, as SPDX 2.3 requires; map MD5. - Skip non-UTF-8 and deeply nested SBOM files instead of failing the wheel build. - Always canonicalize purls, drop unparseable ones, and parse each purl once per component. - Skip everything nested under an `excluded` component. - Index imported packages so the merge is linear in component count. - Update the `versionchanged` release in the wheel SBOM docs. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Martin Prpič --- docs/reference/files.md | 14 ++- src/fromager/sbom.py | 256 +++++++++++++++++++++------------------- tests/test_sbom.py | 211 ++++++++++++++++++++++++++++++++- 3 files changed, 353 insertions(+), 128 deletions(-) diff --git a/docs/reference/files.md b/docs/reference/files.md index 0bc6379c..908c1c22 100644 --- a/docs/reference/files.md +++ b/docs/reference/files.md @@ -157,14 +157,16 @@ CycloneDX SBOMs generated by Maturin are read from the same directory and their components are merged into the canonical SPDX document. Each imported component is related to the wheel with `CONTAINS`; the CycloneDX dependency graph is not copied. Nested target components are included, while components -with CycloneDX scope `excluded` are omitted because they are not shipped -runtime dependencies. Local `file://` download qualifiers are removed from -imported PURLs because those paths are only meaningful in the build -environment. +with CycloneDX scope `excluded`, and everything nested under them, are omitted +because they are not shipped runtime dependencies. Local `file://` download +qualifiers are removed from imported PURLs because those paths are only +meaningful in the build environment. A CycloneDX root with a PyPI PURL matching the wheel's normalized name and version is associated with the wheel package, so auditwheel components are not -attached to the upstream source. The original CycloneDX files are preserved. +attached to the upstream source. Any other component is imported as its own +package; build-reported checksums and licenses never change Fromager's wheel or +upstream entries. The original CycloneDX files are preserved. Maturin must be version 1.12.0 or newer and must be built with its `sbom` feature enabled. Fromager does not enable that Maturin feature automatically. @@ -177,7 +179,7 @@ Downloaded prebuilt wheels retain any native SBOM files, but are not guaranteed to receive a merged Fromager SPDX document until they go through a separate post-download processing path. -```{versionchanged} 0.96.0 +```{versionchanged} 0.104.0 Maturin CycloneDX SBOMs are merged into the canonical Fromager SPDX SBOM. ``` diff --git a/src/fromager/sbom.py b/src/fromager/sbom.py index a29ad61b..d582e6b7 100644 --- a/src/fromager/sbom.py +++ b/src/fromager/sbom.py @@ -18,8 +18,8 @@ from datetime import UTC, datetime from urllib.parse import quote -from license_expression import ExpressionError, get_spdx_licensing from packageurl import PackageURL +from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression from packaging.metadata import RawMetadata from packaging.requirements import Requirement from packaging.utils import NormalizedName, canonicalize_name @@ -56,6 +56,10 @@ "Declared in the upstream project's {source}: {raw}. The license text is " "not available in the package metadata." ) +_CYCLONEDX_REF_COMMENT = ( + "Declared in the build tool SBOM {filename}. The license text is not " + "available in that SBOM." +) # characters allowed in SPDX ids (SPDX 2.3, idstring) _SPDX_IDSTRING_CHARS = r"A-Za-z0-9.\-" _LICENSE_REF_RE = re.compile(rf"LicenseRef-[{_SPDX_IDSTRING_CHARS}]+") @@ -340,6 +344,7 @@ def generate_sbom( _CYCLONEDX_HASH_TO_SPDX = { + "MD5": "MD5", "SHA-1": "SHA1", "SHA-224": "SHA224", "SHA-256": "SHA256", @@ -352,12 +357,26 @@ def generate_sbom( "BLAKE2B-256": "BLAKE2b-256", "BLAKE2B-384": "BLAKE2b-384", "BLAKE2B-512": "BLAKE2b-512", + # BLAKE3 is omitted: spdx-tools requires at least 256 hex digits, while + # CycloneDX BLAKE3 hashes are usually 64. } -# Reused across components; parsing SPDX license expressions is relatively costly. -_SPDX_LICENSING = get_spdx_licensing() +# SPDX 2.3 checksum values are lowercase hex digits of the digest length. +_SPDX_CHECKSUM_HEX_LENGTHS = { + "MD5": 32, + "SHA1": 40, + "SHA224": 56, + "SHA256": 64, + "SHA384": 96, + "SHA512": 128, + "SHA3-256": 64, + "SHA3-384": 96, + "SHA3-512": 128, + "BLAKE2b-256": 64, + "BLAKE2b-384": 96, + "BLAKE2b-512": 128, +} -# SPDX checksum values must be lowercase or uppercase hexadecimal digits. _HEX_RE = re.compile(r"[0-9a-fA-F]+") @@ -372,19 +391,29 @@ def _cyclonedx_component_purl(component: dict[str, typing.Any]) -> str | None: def _cyclonedx_spdx_purl(component: dict[str, typing.Any]) -> str | None: + """Return the component's purl in canonical form, or None if it is invalid. + + Round-tripping through `PackageURL` keeps the same package spelled with + different qualifier order from becoming two SPDX packages. Local file + ``download_url`` qualifiers are dropped. + """ purl = _cyclonedx_component_purl(component) if purl is None: return None try: parsed_purl = PackageURL.from_string(purl) except ValueError: - return purl + logger.warning( + "component %s has an invalid purl %r; ignoring it", + component.get("name"), + purl, + ) + return None qualifiers = dict(parsed_purl.qualifiers or {}) download_url = qualifiers.get("download_url") - if download_url is None or not download_url.startswith("file://"): - return purl - del qualifiers["download_url"] + if download_url is not None and download_url.startswith("file://"): + del qualifiers["download_url"] return PackageURL( type=parsed_purl.type, namespace=parsed_purl.namespace, @@ -398,6 +427,13 @@ def _cyclonedx_spdx_purl(component: dict[str, typing.Any]) -> str | None: def _iter_cyclonedx_components( component: dict[str, typing.Any], ) -> typing.Iterator[dict[str, typing.Any]]: + """Yield *component* and its nested components. + + Components with ``excluded`` scope are not shipped, so they are skipped + together with everything nested under them. + """ + if _cyclonedx_string(component.get("scope")) == "excluded": + return yield component nested_components = component.get("components") if not isinstance(nested_components, list): @@ -416,6 +452,7 @@ def _versions_match(left: str, right: str) -> bool: def _is_python_wheel_root( component: dict[str, typing.Any], + purl: str | None, name: str, version: str, ) -> bool: @@ -424,15 +461,12 @@ def _is_python_wheel_root( The root is matched against the real package identity (*name* and *version* of the built wheel), not the SPDX wheel entry, because ``purl`` overrides change the entry but not what the build tool - reports. + reports. *purl* is the component's canonical purl. The component's + ``name`` and ``version`` fields are optional, but must match if present. """ - purl = _cyclonedx_component_purl(component) if purl is None: return False - try: - parsed_purl = PackageURL.from_string(purl) - except ValueError: - return False + parsed_purl = PackageURL.from_string(purl) if ( parsed_purl.type != "pypi" or parsed_purl.name is None @@ -442,19 +476,20 @@ def _is_python_wheel_root( component_name = _cyclonedx_string(component.get("name")) component_version = _cyclonedx_string(component.get("version")) - if component_name is None or component_version is None: - return False - return ( canonicalize_name(parsed_purl.name) == canonicalize_name(name) and _versions_match(parsed_purl.version, version) - and canonicalize_name(component_name) == canonicalize_name(name) - and _versions_match(component_version, version) + and ( + component_name is None + or canonicalize_name(component_name) == canonicalize_name(name) + ) + and (component_version is None or _versions_match(component_version, version)) ) -def _cyclonedx_component_identity(component: dict[str, typing.Any]) -> str: - purl = _cyclonedx_spdx_purl(component) +def _cyclonedx_component_identity( + component: dict[str, typing.Any], purl: str | None +) -> str: if purl: return f"purl:{purl}" @@ -468,8 +503,7 @@ def _cyclonedx_component_identity(component: dict[str, typing.Any]) -> str: ) -def _cyclonedx_spdx_id(component: dict[str, typing.Any]) -> str: - identity = _cyclonedx_component_identity(component) +def _cyclonedx_spdx_id(identity: str) -> str: digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:20] return f"SPDXRef-cyclonedx-{digest}" @@ -492,8 +526,9 @@ def _cyclonedx_license_expression( expression = _cyclonedx_string(license_info.get("id")) if not expression: continue - if _valid_spdx_expression(expression): - expressions.append(expression) + canonical = _canonical_spdx_expression(expression) + if canonical is not None: + expressions.append(canonical) else: logger.warning( "component %s has an invalid SPDX license expression %r; skipping it", @@ -505,19 +540,22 @@ def _cyclonedx_license_expression( return None # Parenthesize OR expressions before joining: SPDX gives AND higher # precedence than OR, so an ungrouped OR would weaken the declaration. + # Canonical expressions always spell the operator in uppercase. return " AND ".join( f"({expression})" if " OR " in expression else expression for expression in expressions ) -def _valid_spdx_expression(expression: str) -> bool: - """Return True if *expression* is a valid SPDX license expression.""" +def _canonical_spdx_expression(expression: str) -> str | None: + """Return *expression* in canonical SPDX form, or None if it is invalid. + + Uses the PEP 639 rules, which allow ``LicenseRef-`` identifiers. + """ try: - _SPDX_LICENSING.parse(expression, validate=True) - except ExpressionError: - return False - return True + return canonicalize_license_expression(expression) + except InvalidLicenseExpression: + return None def _cyclonedx_checksums( @@ -538,23 +576,28 @@ def _cyclonedx_checksums( normalized_algorithm = _CYCLONEDX_HASH_TO_SPDX.get(algorithm.upper()) if normalized_algorithm is None: continue - if not _HEX_RE.fullmatch(content): + if ( + not _HEX_RE.fullmatch(content) + or len(content) != _SPDX_CHECKSUM_HEX_LENGTHS[normalized_algorithm] + ): logger.warning( - "component %s has a non-hexadecimal %s checksum %r; skipping it", + "component %s has an invalid %s checksum %r; skipping it", _cyclonedx_component_purl(component) or component.get("name"), normalized_algorithm, content, ) continue - checksums.append({"algorithm": normalized_algorithm, "checksumValue": content}) + checksums.append( + {"algorithm": normalized_algorithm, "checksumValue": content.lower()} + ) return checksums def _cyclonedx_package( component: dict[str, typing.Any], spdx_id: str, + purl: str | None, ) -> dict[str, typing.Any]: - purl = _cyclonedx_spdx_purl(component) name = ( _cyclonedx_string(component.get("name")) or purl @@ -623,30 +666,6 @@ def _merge_cyclonedx_package( package["comment"] = f"CycloneDX scope: {scope}" -def _purl_to_spdx_id(sbom: dict[str, typing.Any]) -> dict[str, str]: - purls: dict[str, str] = {} - packages = sbom.get("packages") - if not isinstance(packages, list): - return purls - - for package in packages: - if not isinstance(package, dict): - continue - spdx_id = _cyclonedx_string(package.get("SPDXID")) - external_refs = package.get("externalRefs") - if not spdx_id or not isinstance(external_refs, list): - continue - for external_ref in external_refs: - if not isinstance(external_ref, dict): - continue - if external_ref.get("referenceType") != "purl": - continue - purl = _cyclonedx_string(external_ref.get("referenceLocator")) - if purl: - purls[purl] = spdx_id - return purls - - def _described_spdx_id(sbom: dict[str, typing.Any]) -> str | None: """Return the SPDX id of the package the document describes (the wheel). @@ -666,32 +685,6 @@ def _described_spdx_id(sbom: dict[str, typing.Any]) -> str | None: return None -def _add_spdx_relationship( - sbom: dict[str, typing.Any], - relationship: tuple[str, str, str], -) -> None: - relationships = sbom.setdefault("relationships", []) - if not isinstance(relationships, list): - return - existing = { - ( - item.get("spdxElementId"), - item.get("relationshipType"), - item.get("relatedSpdxElement"), - ) - for item in relationships - if isinstance(item, dict) - } - if relationship not in existing: - relationships.append( - { - "spdxElementId": relationship[0], - "relationshipType": relationship[1], - "relatedSpdxElement": relationship[2], - } - ) - - def merge_cyclonedx_sboms( *, sbom: dict[str, typing.Any], @@ -707,8 +700,9 @@ def merge_cyclonedx_sboms( independent of any ``purl`` overrides in the SPDX wheel entry. Imported components are related to the wheel with ``CONTAINS``. Local file - download qualifiers are removed from their PURLs. The original CycloneDX - files are only read and remain alongside ``fromager.spdx.json``. + download qualifiers are removed from their PURLs. Imported ``LicenseRef`` + ids are declared in ``hasExtractedLicensingInfos`` so the document stays + valid SPDX. The original CycloneDX files are only read and remain alongside ``fromager.spdx.json``. """ if not sboms_dir.is_dir(): return @@ -718,11 +712,21 @@ def merge_cyclonedx_sboms( logger.warning("SBOM has no DESCRIBES relationship; skipping CycloneDX merge") return - purl_to_spdx_id = _purl_to_spdx_id(sbom) - identity_to_spdx_id: dict[str, str] = {} packages = sbom.setdefault("packages", []) - if not isinstance(packages, list): + relationships = sbom.setdefault("relationships", []) + if not isinstance(packages, list) or not isinstance(relationships, list): return + # Only packages created by this merge are reused for duplicates, so + # build-reported metadata never lands on fromager's own wheel or + # upstream entries. + imported: dict[str, dict[str, typing.Any]] = {} + # Every LicenseRef used in the document must be declared in it once. + extracted = sbom.setdefault("hasExtractedLicensingInfos", []) + declared_refs = { + info.get("licenseId") for info in extracted if isinstance(info, dict) + } + # Identities of build tool roots recognized as the wheel itself. + wheel_identities: set[str] = set() for sbom_path in sorted(sboms_dir.iterdir()): if not sbom_path.is_file() or sbom_path.name == SBOM_FILENAME: @@ -730,7 +734,12 @@ def merge_cyclonedx_sboms( try: with sbom_path.open(encoding="utf-8") as sbom_file: cyclonedx = json.load(sbom_file) - except (OSError, json.JSONDecodeError) as err: + except ( + OSError, + UnicodeDecodeError, + json.JSONDecodeError, + RecursionError, + ) as err: logger.warning("could not read SBOM file %s: %s", sbom_path, err) continue @@ -755,41 +764,46 @@ def merge_cyclonedx_sboms( ) for component, is_root in components: - if _cyclonedx_string(component.get("scope")) == "excluded": + purl = _cyclonedx_spdx_purl(component) + identity = _cyclonedx_component_identity(component, purl) + if is_root and _is_python_wheel_root(component, purl, name, version): + wheel_identities.add(identity) + continue + if identity in wheel_identities: continue - identity = _cyclonedx_component_identity(component) - purl = _cyclonedx_spdx_purl(component) - spdx_id: str | None - if is_root and _is_python_wheel_root(component, name, version): - spdx_id = wheel_spdx_id - if purl: - purl_to_spdx_id[purl] = spdx_id + package = imported.get(identity) + if package is not None: + _merge_cyclonedx_package(package, component) else: - spdx_id = identity_to_spdx_id.get(identity) - if purl: - mapped_spdx_id = purl_to_spdx_id.get(purl) - if mapped_spdx_id is not None: - spdx_id = mapped_spdx_id - if spdx_id is None: - spdx_id = _cyclonedx_spdx_id(component) - packages.append(_cyclonedx_package(component, spdx_id)) - if purl: - purl_to_spdx_id[purl] = spdx_id - else: - package = next( - package - for package in packages - if package.get("SPDXID") == spdx_id + spdx_id = _cyclonedx_spdx_id(identity) + package = _cyclonedx_package(component, spdx_id, purl) + imported[identity] = package + packages.append(package) + relationships.append( + { + "spdxElementId": wheel_spdx_id, + "relationshipType": "CONTAINS", + "relatedSpdxElement": spdx_id, + } + ) + + for ref in _LICENSE_REF_RE.findall(package.get("licenseDeclared", "")): + if ref not in declared_refs: + declared_refs.add(ref) + extracted.append( + { + "licenseId": ref, + "name": ref.removeprefix("LicenseRef-"), + "extractedText": "NOASSERTION", + "comment": _CYCLONEDX_REF_COMMENT.format( + filename=sbom_path.name + ), + } ) - _merge_cyclonedx_package(package, component) - identity_to_spdx_id[identity] = spdx_id - if spdx_id != wheel_spdx_id: - _add_spdx_relationship( - sbom, - (wheel_spdx_id, "CONTAINS", spdx_id), - ) + if not extracted: + del sbom["hasExtractedLicensingInfos"] def write_sbom( diff --git a/tests/test_sbom.py b/tests/test_sbom.py index 56dd5c32..6083273c 100644 --- a/tests/test_sbom.py +++ b/tests/test_sbom.py @@ -1044,6 +1044,7 @@ def test_merge_cyclonedx_sboms_maps_python_root_with_purl_version_override( ctx=ctx, req=Requirement("test-pkg==0.5.5"), version=Version("0.5.5"), + metadata=_metadata(), ) wheel_id = sbom._described_spdx_id(document) wheel = next( @@ -1313,6 +1314,8 @@ def test_merge_cyclonedx_sboms_ignores_invalid_and_other_formats( sboms_dir.mkdir() (sboms_dir / "not-an-sbom.json").write_text('{"format": "other"}') (sboms_dir / "broken.json").write_text("{") + (sboms_dir / "latin1.json").write_bytes(b'{"name": "caf\xe9"}') + (sboms_dir / "nested.json").write_text("[" * 1_000_000 + "]" * 1_000_000) with caplog.at_level("WARNING", logger="fromager.sbom"): sbom.merge_cyclonedx_sboms( @@ -1320,5 +1323,211 @@ def test_merge_cyclonedx_sboms_ignores_invalid_and_other_formats( ) assert len(document["packages"]) == 2 - assert "could not read" in caplog.text + for filename in ("broken.json", "latin1.json", "nested.json"): + assert f"could not read SBOM file {sboms_dir / filename}" in caplog.text + _validate_spdx(document) + + +def _merge_components( + tmp_path: pathlib.Path, + components: list[dict[str, typing.Any]], + *, + root: dict[str, typing.Any] | None = None, + req: str = "example-pkg==1.2.3", + wheel_version: str = "1.2.3", +) -> dict[str, typing.Any]: + """Merge one CycloneDX document into a fresh SBOM and return the SBOM.""" + requirement = Requirement(req) + ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + document = sbom.generate_sbom( + ctx=ctx, + req=requirement, + version=Version(str(next(iter(requirement.specifier)).version)), + wheel_version=Version(wheel_version), + metadata=_metadata(), + ) + cyclonedx: dict[str, typing.Any] = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "components": components, + } + if root is not None: + cyclonedx["metadata"] = {"component": root} + sboms_dir = tmp_path / "sboms" + sboms_dir.mkdir(exist_ok=True) + (sboms_dir / "tool.cdx.json").write_text(json.dumps(cyclonedx)) + sbom.merge_cyclonedx_sboms( + sbom=document, + sboms_dir=sboms_dir, + name=requirement.name, + version=wheel_version, + ) + return document + + +def test_merge_cyclonedx_sboms_maps_purl_only_root_to_wheel( + tmp_path: pathlib.Path, +) -> None: + """A root without name/version fields matches the wheel by its purl.""" + root = {"type": "library", "purl": "pkg:pypi/example-pkg@1.2.3"} + serde = {"type": "library", "name": "serde", "purl": "pkg:cargo/serde@1.0.0"} + + document = _merge_components(tmp_path, [serde], root=root) + + assert [package["name"] for package in document["packages"][2:]] == ["serde"] + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_never_merges_into_upstream_entry( + tmp_path: pathlib.Path, +) -> None: + """A root whose purl equals the upstream purl stays a separate package. + + The wheel has a local version, so the root does not match the wheel and + its build-reported metadata must not land on the upstream entry. + """ + root = { + "type": "library", + "name": "example-pkg", + "version": "1.2.3", + "purl": "pkg:pypi/example-pkg@1.2.3", + "hashes": [{"alg": "SHA-256", "content": "a" * 64}], + "licenses": [{"expression": "MIT"}], + } + + document = _merge_components(tmp_path, [], root=root, wheel_version="1.2.3+local.1") + + _wheel, upstream, imported = document["packages"] + assert "checksums" not in upstream + assert upstream["licenseDeclared"] == "NOASSERTION" + assert imported["SPDXID"].startswith("SPDXRef-cyclonedx-") + contains = { + relationship["relatedSpdxElement"] + for relationship in document["relationships"] + if relationship["relationshipType"] == "CONTAINS" + } + assert contains == {imported["SPDXID"]} + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_skips_excluded_subtree(tmp_path: pathlib.Path) -> None: + """Children of an excluded component are not shipped either.""" + excluded = { + "type": "library", + "name": "build-tool", + "purl": "pkg:cargo/build-tool@1.0.0", + "scope": "excluded", + "components": [ + {"type": "library", "name": "child", "purl": "pkg:cargo/child@1.0.0"} + ], + } + + document = _merge_components(tmp_path, [excluded]) + + assert len(document["packages"]) == 2 + + +def test_merge_cyclonedx_sboms_deduplicates_by_canonical_purl( + tmp_path: pathlib.Path, +) -> None: + """The same purl with differently ordered qualifiers is one package.""" + components = [ + {"type": "library", "name": "serde", "purl": "pkg:cargo/serde@1.0.0?b=2&a=1"}, + {"type": "library", "name": "serde", "purl": "pkg:cargo/serde@1.0.0?a=1&b=2"}, + ] + + document = _merge_components(tmp_path, components) + + serde = [package for package in document["packages"] if package["name"] == "serde"] + assert len(serde) == 1 + assert serde[0]["externalRefs"][0]["referenceLocator"] == ( + "pkg:cargo/serde@1.0.0?a=1&b=2" + ) + + +def test_merge_cyclonedx_sboms_drops_invalid_purl(tmp_path: pathlib.Path) -> None: + """An unparseable purl is not emitted as a purl locator.""" + component = {"type": "library", "name": "broken", "purl": "not-a-purl"} + + document = _merge_components(tmp_path, [component]) + + broken = document["packages"][2] + assert broken["name"] == "broken" + assert "externalRefs" not in broken + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_declares_imported_license_refs( + tmp_path: pathlib.Path, +) -> None: + """Each imported LicenseRef gets one extracted licensing info entry.""" + components = [ + { + "type": "library", + "name": "a", + "purl": "pkg:cargo/a@1.0.0", + "licenses": [{"expression": "MIT AND LicenseRef-Proprietary-1.0"}], + }, + { + "type": "library", + "name": "b", + "purl": "pkg:cargo/b@1.0.0", + "licenses": [{"expression": "LicenseRef-Proprietary-1.0"}], + }, + ] + + document = _merge_components(tmp_path, components) + + assert document["hasExtractedLicensingInfos"] == [ + { + "licenseId": "LicenseRef-Proprietary-1.0", + "name": "Proprietary-1.0", + "extractedText": "NOASSERTION", + "comment": ( + "Declared in the build tool SBOM tool.cdx.json. The license " + "text is not available in that SBOM." + ), + } + ] _validate_spdx(document) + + +def test_cyclonedx_license_expression_groups_lowercase_or() -> None: + """Lowercase operators are canonicalized and grouped too.""" + component = { + "licenses": [ + {"expression": "MIT or Apache-2.0"}, + {"expression": "BSD-3-Clause"}, + ] + } + + assert ( + sbom._cyclonedx_license_expression(component) + == "(MIT OR Apache-2.0) AND BSD-3-Clause" + ) + + +def test_cyclonedx_license_expression_keeps_license_ref() -> None: + """LicenseRef- identifiers are valid SPDX and must be kept.""" + component = {"licenses": [{"expression": "MIT AND LicenseRef-Proprietary-1.0"}]} + + assert ( + sbom._cyclonedx_license_expression(component) + == "MIT AND LicenseRef-Proprietary-1.0" + ) + + +def test_cyclonedx_checksums_normalizes_case_and_checks_length() -> None: + """SPDX requires lowercase hex of the exact digest length.""" + component = { + "hashes": [ + {"alg": "SHA-256", "content": "A" * 64}, + {"alg": "SHA-1", "content": "b" * 64}, + {"alg": "MD5", "content": "c" * 32}, + ] + } + + assert sbom._cyclonedx_checksums(component) == [ + {"algorithm": "SHA256", "checksumValue": "a" * 64}, + {"algorithm": "MD5", "checksumValue": "c" * 32}, + ]