diff --git a/docs/reference/files.md b/docs/reference/files.md index 0485d2d3..908c1c22 100644 --- a/docs/reference/files.md +++ b/docs/reference/files.md @@ -149,6 +149,40 @@ The following example shows `graph.json` file for the top-level dependency `whee } ``` +## Wheel SBOMs + +When SBOM generation is enabled, Fromager writes the canonical SPDX 2.3 +document to `.dist-info/sboms/fromager.spdx.json`. During source builds, +CycloneDX SBOMs generated by Maturin are read from the same directory and +their components are merged into the canonical SPDX document. Each imported +component is related to the wheel with `CONTAINS`; the CycloneDX dependency +graph is not copied. Nested target components are included, while components +with CycloneDX scope `excluded`, and everything nested under them, are omitted +because they are not shipped runtime dependencies. Local `file://` download +qualifiers are removed from imported PURLs because those paths are only +meaningful in the build environment. + +A CycloneDX root with a PyPI PURL matching the wheel's normalized name and +version is associated with the wheel package, so auditwheel components are not +attached to the upstream source. Any other component is imported as its own +package; build-reported checksums and licenses never change Fromager's wheel or +upstream entries. The original CycloneDX files are preserved. + +Maturin must be version 1.12.0 or newer and must be built with its `sbom` +feature enabled. Fromager does not enable that Maturin feature automatically. +For example, a packaging environment can pass Maturin's +`MATURIN_SETUP_ARGS` with a feature set that includes `sbom`. The exact +feature set depends on the platform and packaging environment. + +This merge is applied to wheels processed by Fromager's source-build path. +Downloaded prebuilt wheels retain any native SBOM files, but are not guaranteed +to receive a merged Fromager SPDX document until they go through a separate +post-download processing path. + +```{versionchanged} 0.104.0 +Maturin CycloneDX SBOMs are merged into the canonical Fromager SPDX SBOM. +``` + ## Output Directories During the wheel building process, fromager generates multiple output directories namely `sdists-repo`, `wheels-repo` and `work-dir`. These directories contain important information related to the wheel build. diff --git a/src/fromager/sbom.py b/src/fromager/sbom.py index 52c3d5b6..d582e6b7 100644 --- a/src/fromager/sbom.py +++ b/src/fromager/sbom.py @@ -19,10 +19,11 @@ from urllib.parse import quote from packageurl import PackageURL +from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression from packaging.metadata import RawMetadata from packaging.requirements import Requirement from packaging.utils import NormalizedName, canonicalize_name -from packaging.version import Version +from packaging.version import InvalidVersion, Version from .pkgmetadata import DeclaredLicense, declared_license_from_raw_metadata @@ -55,6 +56,10 @@ "Declared in the upstream project's {source}: {raw}. The license text is " "not available in the package metadata." ) +_CYCLONEDX_REF_COMMENT = ( + "Declared in the build tool SBOM {filename}. The license text is not " + "available in that SBOM." +) # characters allowed in SPDX ids (SPDX 2.3, idstring) _SPDX_IDSTRING_CHARS = r"A-Za-z0-9.\-" _LICENSE_REF_RE = re.compile(rf"LicenseRef-[{_SPDX_IDSTRING_CHARS}]+") @@ -338,6 +343,469 @@ def generate_sbom( return doc +_CYCLONEDX_HASH_TO_SPDX = { + "MD5": "MD5", + "SHA-1": "SHA1", + "SHA-224": "SHA224", + "SHA-256": "SHA256", + "SHA-384": "SHA384", + "SHA-512": "SHA512", + # SPDX 2.3 has no SHA3-224 algorithm, so it is intentionally omitted. + "SHA3-256": "SHA3-256", + "SHA3-384": "SHA3-384", + "SHA3-512": "SHA3-512", + "BLAKE2B-256": "BLAKE2b-256", + "BLAKE2B-384": "BLAKE2b-384", + "BLAKE2B-512": "BLAKE2b-512", + # BLAKE3 is omitted: spdx-tools requires at least 256 hex digits, while + # CycloneDX BLAKE3 hashes are usually 64. +} + +# SPDX 2.3 checksum values are lowercase hex digits of the digest length. +_SPDX_CHECKSUM_HEX_LENGTHS = { + "MD5": 32, + "SHA1": 40, + "SHA224": 56, + "SHA256": 64, + "SHA384": 96, + "SHA512": 128, + "SHA3-256": 64, + "SHA3-384": 96, + "SHA3-512": 128, + "BLAKE2b-256": 64, + "BLAKE2b-384": 96, + "BLAKE2b-512": 128, +} + +_HEX_RE = re.compile(r"[0-9a-fA-F]+") + + +def _cyclonedx_string(value: typing.Any) -> str | None: + if isinstance(value, str) and value: + return value + return None + + +def _cyclonedx_component_purl(component: dict[str, typing.Any]) -> str | None: + return _cyclonedx_string(component.get("purl")) + + +def _cyclonedx_spdx_purl(component: dict[str, typing.Any]) -> str | None: + """Return the component's purl in canonical form, or None if it is invalid. + + Round-tripping through `PackageURL` keeps the same package spelled with + different qualifier order from becoming two SPDX packages. Local file + ``download_url`` qualifiers are dropped. + """ + purl = _cyclonedx_component_purl(component) + if purl is None: + return None + try: + parsed_purl = PackageURL.from_string(purl) + except ValueError: + logger.warning( + "component %s has an invalid purl %r; ignoring it", + component.get("name"), + purl, + ) + return None + + qualifiers = dict(parsed_purl.qualifiers or {}) + download_url = qualifiers.get("download_url") + if download_url is not None and download_url.startswith("file://"): + del qualifiers["download_url"] + return PackageURL( + type=parsed_purl.type, + namespace=parsed_purl.namespace, + name=parsed_purl.name, + version=parsed_purl.version, + qualifiers=qualifiers or None, + subpath=parsed_purl.subpath, + ).to_string() + + +def _iter_cyclonedx_components( + component: dict[str, typing.Any], +) -> typing.Iterator[dict[str, typing.Any]]: + """Yield *component* and its nested components. + + Components with ``excluded`` scope are not shipped, so they are skipped + together with everything nested under them. + """ + if _cyclonedx_string(component.get("scope")) == "excluded": + return + yield component + nested_components = component.get("components") + if not isinstance(nested_components, list): + return + for nested_component in nested_components: + if isinstance(nested_component, dict): + yield from _iter_cyclonedx_components(nested_component) + + +def _versions_match(left: str, right: str) -> bool: + try: + return Version(left) == Version(right) + except InvalidVersion: + return left == right + + +def _is_python_wheel_root( + component: dict[str, typing.Any], + purl: str | None, + name: str, + version: str, +) -> bool: + """Return True if the CycloneDX root *component* is the wheel itself. + + The root is matched against the real package identity (*name* and + *version* of the built wheel), not the SPDX wheel entry, because + ``purl`` overrides change the entry but not what the build tool + reports. *purl* is the component's canonical purl. The component's + ``name`` and ``version`` fields are optional, but must match if present. + """ + if purl is None: + return False + parsed_purl = PackageURL.from_string(purl) + if ( + parsed_purl.type != "pypi" + or parsed_purl.name is None + or parsed_purl.version is None + ): + return False + + component_name = _cyclonedx_string(component.get("name")) + component_version = _cyclonedx_string(component.get("version")) + return ( + canonicalize_name(parsed_purl.name) == canonicalize_name(name) + and _versions_match(parsed_purl.version, version) + and ( + component_name is None + or canonicalize_name(component_name) == canonicalize_name(name) + ) + and (component_version is None or _versions_match(component_version, version)) + ) + + +def _cyclonedx_component_identity( + component: dict[str, typing.Any], purl: str | None +) -> str: + if purl: + return f"purl:{purl}" + + return "component:" + "\x00".join( + [ + _cyclonedx_string(component.get("type")) or "", + _cyclonedx_string(component.get("group")) or "", + _cyclonedx_string(component.get("name")) or "", + _cyclonedx_string(component.get("version")) or "", + ] + ) + + +def _cyclonedx_spdx_id(identity: str) -> str: + digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:20] + return f"SPDXRef-cyclonedx-{digest}" + + +def _cyclonedx_license_expression( + component: dict[str, typing.Any], +) -> str | None: + licenses = component.get("licenses") + if not isinstance(licenses, list): + return None + + expressions: list[str] = [] + for license_choice in licenses: + if not isinstance(license_choice, dict): + continue + expression = _cyclonedx_string(license_choice.get("expression")) + if not expression: + license_info = license_choice.get("license") + if isinstance(license_info, dict): + expression = _cyclonedx_string(license_info.get("id")) + if not expression: + continue + canonical = _canonical_spdx_expression(expression) + if canonical is not None: + expressions.append(canonical) + else: + logger.warning( + "component %s has an invalid SPDX license expression %r; skipping it", + _cyclonedx_component_purl(component) or component.get("name"), + expression, + ) + + if not expressions: + return None + # Parenthesize OR expressions before joining: SPDX gives AND higher + # precedence than OR, so an ungrouped OR would weaken the declaration. + # Canonical expressions always spell the operator in uppercase. + return " AND ".join( + f"({expression})" if " OR " in expression else expression + for expression in expressions + ) + + +def _canonical_spdx_expression(expression: str) -> str | None: + """Return *expression* in canonical SPDX form, or None if it is invalid. + + Uses the PEP 639 rules, which allow ``LicenseRef-`` identifiers. + """ + try: + return canonicalize_license_expression(expression) + except InvalidLicenseExpression: + return None + + +def _cyclonedx_checksums( + component: dict[str, typing.Any], +) -> list[dict[str, str]]: + hashes = component.get("hashes") + if not isinstance(hashes, list): + return [] + + checksums: list[dict[str, str]] = [] + for hash_info in hashes: + if not isinstance(hash_info, dict): + continue + algorithm = _cyclonedx_string(hash_info.get("alg")) + content = _cyclonedx_string(hash_info.get("content")) + if not algorithm or not content: + continue + normalized_algorithm = _CYCLONEDX_HASH_TO_SPDX.get(algorithm.upper()) + if normalized_algorithm is None: + continue + if ( + not _HEX_RE.fullmatch(content) + or len(content) != _SPDX_CHECKSUM_HEX_LENGTHS[normalized_algorithm] + ): + logger.warning( + "component %s has an invalid %s checksum %r; skipping it", + _cyclonedx_component_purl(component) or component.get("name"), + normalized_algorithm, + content, + ) + continue + checksums.append( + {"algorithm": normalized_algorithm, "checksumValue": content.lower()} + ) + return checksums + + +def _cyclonedx_package( + component: dict[str, typing.Any], + spdx_id: str, + purl: str | None, +) -> dict[str, typing.Any]: + name = ( + _cyclonedx_string(component.get("name")) + or purl + or _cyclonedx_string(component.get("bom-ref")) + or "unknown" + ) + version = _cyclonedx_string(component.get("version")) or "NOASSERTION" + package: dict[str, typing.Any] = { + "SPDXID": spdx_id, + "name": name, + "versionInfo": version, + "downloadLocation": "NOASSERTION", + "supplier": "NOASSERTION", + } + + if purl: + package["externalRefs"] = [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": purl, + } + ] + + checksums = _cyclonedx_checksums(component) + if checksums: + package["checksums"] = checksums + + license_expression = _cyclonedx_license_expression(component) + if license_expression: + package["licenseDeclared"] = license_expression + + scope = _cyclonedx_string(component.get("scope")) + if scope and scope != "required": + package["comment"] = f"CycloneDX scope: {scope}" + + return package + + +def _merge_cyclonedx_package( + package: dict[str, typing.Any], + component: dict[str, typing.Any], +) -> None: + """Add non-conflicting metadata from a duplicate CycloneDX component.""" + component_checksums = _cyclonedx_checksums(component) + if component_checksums: + checksums = package.setdefault("checksums", []) + if isinstance(checksums, list): + existing_checksums = { + (item.get("algorithm"), item.get("checksumValue")) + for item in checksums + if isinstance(item, dict) + } + for checksum in component_checksums: + key = (checksum["algorithm"], checksum["checksumValue"]) + if key not in existing_checksums: + checksums.append(checksum) + existing_checksums.add(key) + + license_expression = _cyclonedx_license_expression(component) + if license_expression and "licenseDeclared" not in package: + package["licenseDeclared"] = license_expression + + scope = _cyclonedx_string(component.get("scope")) + if scope and scope != "required" and "comment" not in package: + package["comment"] = f"CycloneDX scope: {scope}" + + +def _described_spdx_id(sbom: dict[str, typing.Any]) -> str | None: + """Return the SPDX id of the package the document describes (the wheel). + + Package SPDX ids are not predictable, so the wheel is found through the + document's DESCRIBES relationship. + """ + relationships = sbom.get("relationships") + if not isinstance(relationships, list): + return None + for relationship in relationships: + if ( + isinstance(relationship, dict) + and relationship.get("spdxElementId") == "SPDXRef-DOCUMENT" + and relationship.get("relationshipType") == "DESCRIBES" + ): + return _cyclonedx_string(relationship.get("relatedSpdxElement")) + return None + + +def merge_cyclonedx_sboms( + *, + sbom: dict[str, typing.Any], + sboms_dir: pathlib.Path, + name: str, + version: str, +) -> None: + """Merge CycloneDX components into a Fromager SPDX document. + + *name* and *version* are the real identity of the built wheel (the + requirement name and the version from the wheel file name). They are + used to recognize a build tool's root component as the wheel itself, + independent of any ``purl`` overrides in the SPDX wheel entry. + + Imported components are related to the wheel with ``CONTAINS``. Local file + download qualifiers are removed from their PURLs. Imported ``LicenseRef`` + ids are declared in ``hasExtractedLicensingInfos`` so the document stays + valid SPDX. The original CycloneDX files are only read and remain alongside ``fromager.spdx.json``. + """ + if not sboms_dir.is_dir(): + return + + wheel_spdx_id = _described_spdx_id(sbom) + if wheel_spdx_id is None: + logger.warning("SBOM has no DESCRIBES relationship; skipping CycloneDX merge") + return + + packages = sbom.setdefault("packages", []) + relationships = sbom.setdefault("relationships", []) + if not isinstance(packages, list) or not isinstance(relationships, list): + return + # Only packages created by this merge are reused for duplicates, so + # build-reported metadata never lands on fromager's own wheel or + # upstream entries. + imported: dict[str, dict[str, typing.Any]] = {} + # Every LicenseRef used in the document must be declared in it once. + extracted = sbom.setdefault("hasExtractedLicensingInfos", []) + declared_refs = { + info.get("licenseId") for info in extracted if isinstance(info, dict) + } + # Identities of build tool roots recognized as the wheel itself. + wheel_identities: set[str] = set() + + for sbom_path in sorted(sboms_dir.iterdir()): + if not sbom_path.is_file() or sbom_path.name == SBOM_FILENAME: + continue + try: + with sbom_path.open(encoding="utf-8") as sbom_file: + cyclonedx = json.load(sbom_file) + except ( + OSError, + UnicodeDecodeError, + json.JSONDecodeError, + RecursionError, + ) as err: + logger.warning("could not read SBOM file %s: %s", sbom_path, err) + continue + + if not isinstance(cyclonedx, dict) or cyclonedx.get("bomFormat") != "CycloneDX": + continue + + metadata = cyclonedx.get("metadata") + root = metadata.get("component") if isinstance(metadata, dict) else None + components: list[tuple[dict[str, typing.Any], bool]] = [] + if isinstance(root, dict): + components.extend( + (component, component is root) + for component in _iter_cyclonedx_components(root) + ) + raw_components = cyclonedx.get("components") + if isinstance(raw_components, list): + for component in raw_components: + if isinstance(component, dict): + components.extend( + (nested_component, False) + for nested_component in _iter_cyclonedx_components(component) + ) + + for component, is_root in components: + purl = _cyclonedx_spdx_purl(component) + identity = _cyclonedx_component_identity(component, purl) + if is_root and _is_python_wheel_root(component, purl, name, version): + wheel_identities.add(identity) + continue + if identity in wheel_identities: + continue + + package = imported.get(identity) + if package is not None: + _merge_cyclonedx_package(package, component) + else: + spdx_id = _cyclonedx_spdx_id(identity) + package = _cyclonedx_package(component, spdx_id, purl) + imported[identity] = package + packages.append(package) + relationships.append( + { + "spdxElementId": wheel_spdx_id, + "relationshipType": "CONTAINS", + "relatedSpdxElement": spdx_id, + } + ) + + for ref in _LICENSE_REF_RE.findall(package.get("licenseDeclared", "")): + if ref not in declared_refs: + declared_refs.add(ref) + extracted.append( + { + "licenseId": ref, + "name": ref.removeprefix("LicenseRef-"), + "extractedText": "NOASSERTION", + "comment": _CYCLONEDX_REF_COMMENT.format( + filename=sbom_path.name + ), + } + ) + + if not extracted: + del sbom["hasExtractedLicensingInfos"] + + def write_sbom( *, sbom: dict[str, typing.Any], diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index e442ffed..e2f1a210 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -357,6 +357,12 @@ def add_extra_metadata_to_wheels( # identifies the wheel that actually contains it wheel_version=dist_version, ) + sbom.merge_cyclonedx_sboms( + sbom=sbom_doc, + sboms_dir=dist_info_dir / "sboms", + name=req.name, + version=str(dist_version), + ) sbom.write_sbom(sbom=sbom_doc, dist_info_dir=dist_info_dir) build_tag = get_build_tag( diff --git a/tests/test_sbom.py b/tests/test_sbom.py index ddfa1184..6083273c 100644 --- a/tests/test_sbom.py +++ b/tests/test_sbom.py @@ -10,6 +10,7 @@ from packaging.metadata import RawMetadata from packaging.requirements import Requirement from packaging.version import Version +from pydantic import AnyUrl from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import ( JsonLikeDictParser, ) @@ -33,6 +34,58 @@ def _metadata(**fields: typing.Any) -> RawMetadata: return raw +def _write_cyclonedx_sbom( + sboms_dir: pathlib.Path, + filename: str, + *, + root_name: str = "example", + root_version: str = "1.0.0", +) -> None: + """Write a representative Maturin CycloneDX 1.5 document.""" + root_purl = f"pkg:cargo/{root_name}@{root_version}" + serde_purl = "pkg:cargo/serde@1.0.0" + pyo3_purl = "pkg:cargo/pyo3@0.21.0" + document = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "serialNumber": f"urn:uuid:{root_name}", + "metadata": { + "tools": [{"vendor": "PyO3", "name": "maturin", "version": "1.15.0"}], + "component": { + "type": "library", + "bom-ref": root_purl, + "name": root_name, + "version": root_version, + "purl": root_purl, + }, + }, + "components": [ + { + "type": "library", + "bom-ref": serde_purl, + "name": "serde", + "version": "1.0.0", + "scope": "required", + "purl": serde_purl, + "hashes": [{"alg": "SHA-256", "content": "a" * 64}], + "licenses": [{"license": {"id": "MIT"}}], + }, + { + "type": "library", + "bom-ref": pyo3_purl, + "name": "pyo3", + "version": "0.21.0", + "scope": "excluded", + "purl": pyo3_purl, + "licenses": [{"expression": "Apache-2.0"}], + }, + ], + "dependencies": [{"ref": root_purl, "dependsOn": [serde_purl, pyo3_purl]}], + } + sboms_dir.mkdir(parents=True, exist_ok=True) + (sboms_dir / filename).write_text(json.dumps(document)) + + def test_generate_sbom_structure(tmp_path: pathlib.Path) -> None: """Verify the generated SBOM has the required SPDX 2.3 fields.""" ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) @@ -799,3 +852,682 @@ def test_generate_sbom_license_with_unparsed_field(tmp_path: pathlib.Path) -> No + " Unparseable fields: Classifier." ) _validate_spdx(doc) + + +def test_merge_cyclonedx_sbom_imports_packages_and_relationships( + tmp_path: pathlib.Path, +) -> None: + """Verify Maturin CycloneDX data is represented in SPDX.""" + ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + document = sbom.generate_sbom( + ctx=ctx, + req=Requirement("example-pkg==1.2.3"), + version=Version("1.2.3"), + metadata=_metadata(), + ) + wheel_id = sbom._described_spdx_id(document) + sboms_dir = tmp_path / "sboms" + _write_cyclonedx_sbom(sboms_dir, "example.cyclonedx.json") + + sbom.merge_cyclonedx_sboms( + sbom=document, sboms_dir=sboms_dir, name="example-pkg", version="1.2.3" + ) + + packages = {package["name"]: package for package in document["packages"]} + assert {"example-pkg", "example", "serde"} <= packages.keys() + assert "pyo3" not in packages + serde = packages["serde"] + assert serde["externalRefs"][0]["referenceLocator"] == "pkg:cargo/serde@1.0.0" + assert serde["checksums"] == [{"algorithm": "SHA256", "checksumValue": "a" * 64}] + assert serde["licenseDeclared"] == "MIT" + assert "comment" not in serde + spdx_ids = {package["name"]: package["SPDXID"] for package in document["packages"]} + relationships = { + ( + relationship["spdxElementId"], + relationship["relationshipType"], + relationship["relatedSpdxElement"], + ) + for relationship in document["relationships"] + } + assert (wheel_id, "CONTAINS", spdx_ids["example"]) in relationships + assert (wheel_id, "CONTAINS", spdx_ids["serde"]) in relationships + assert not any( + relationship_type == "DEPENDS_ON" for _, relationship_type, _ in relationships + ) + _validate_spdx(document) + + +def test_cyclonedx_checksums_skips_unsupported_algorithm() -> None: + """SHA3-224 has no SPDX 2.3 equivalent and must be dropped.""" + component = { + "name": "serde", + "hashes": [ + {"alg": "SHA3-224", "content": "a" * 56}, + {"alg": "SHA-256", "content": "b" * 64}, + ], + } + + checksums = sbom._cyclonedx_checksums(component) + + assert checksums == [{"algorithm": "SHA256", "checksumValue": "b" * 64}] + + +def test_cyclonedx_checksums_skips_non_hex_content() -> None: + """Non-hexadecimal checksum content would fail SPDX validation.""" + component = { + "name": "serde", + "hashes": [ + {"alg": "SHA-256", "content": "not-a-valid-hash"}, + {"alg": "SHA-512", "content": "c" * 128}, + ], + } + + checksums = sbom._cyclonedx_checksums(component) + + assert checksums == [{"algorithm": "SHA512", "checksumValue": "c" * 128}] + + +def test_cyclonedx_license_expression_skips_invalid() -> None: + """Invalid SPDX expressions must not be copied into licenseDeclared.""" + component = { + "name": "serde", + "licenses": [ + {"expression": "MIT"}, + {"expression": "Totally Not A License"}, + ], + } + + assert sbom._cyclonedx_license_expression(component) == "MIT" + + +def test_cyclonedx_license_expression_all_invalid_returns_none() -> None: + """A component with only invalid expressions declares no license.""" + component = {"name": "serde", "licenses": [{"license": {"id": "Bogus-9.9"}}]} + + assert sbom._cyclonedx_license_expression(component) is None + + +def test_cyclonedx_license_expression_groups_or_operands() -> None: + """OR expressions must be parenthesized so AND-joining cannot weaken them.""" + component = { + "name": "serde", + "licenses": [ + {"expression": "MIT OR Apache-2.0"}, + {"expression": "BSD-3-Clause"}, + ], + } + + assert ( + sbom._cyclonedx_license_expression(component) + == "(MIT OR Apache-2.0) AND BSD-3-Clause" + ) + + +def test_merge_cyclonedx_sboms_maps_python_root_to_wheel( + tmp_path: pathlib.Path, +) -> None: + """Verify an auditwheel Python root is attached to the wheel package.""" + settings = SbomSettings(repository_url=AnyUrl("https://pkg.test/simple/")) + ctx = make_sbom_ctx(tmp_path, sbom_settings=settings) + document = sbom.generate_sbom( + ctx=ctx, + req=Requirement("pillow==12.3.0"), + version=Version("12.3.0"), + metadata=_metadata(), + ) + wheel_id = sbom._described_spdx_id(document) + sboms_dir = tmp_path / "sboms" + auditwheel = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": { + "component": { + "type": "library", + "bom-ref": "pkg:pypi/pillow@12.3.0?file_name=pillow.whl", + "name": "Pillow", + "version": "12.3.0", + "purl": "pkg:pypi/pillow@12.3.0?file_name=pillow.whl", + } + }, + "components": [ + { + "type": "library", + "bom-ref": "pkg:rpm/almalinux/libXau@1.0.9-3.el8", + "name": "libXau", + "version": "1.0.9-3.el8", + "purl": "pkg:rpm/almalinux/libXau@1.0.9-3.el8", + } + ], + "dependencies": [ + { + "ref": "pkg:pypi/pillow@12.3.0?file_name=pillow.whl", + "dependsOn": ["pkg:rpm/almalinux/libXau@1.0.9-3.el8"], + } + ], + } + sboms_dir.mkdir(parents=True) + (sboms_dir / "auditwheel.cdx.json").write_text(json.dumps(auditwheel)) + + sbom.merge_cyclonedx_sboms( + sbom=document, sboms_dir=sboms_dir, name="pillow", version="12.3.0" + ) + + packages = document["packages"] + assert len(packages) == 3 + rpm = next(package for package in packages if package["name"] == "libXau") + relationships = { + ( + relationship["spdxElementId"], + relationship["relationshipType"], + relationship["relatedSpdxElement"], + ) + for relationship in document["relationships"] + } + assert (wheel_id, "CONTAINS", rpm["SPDXID"]) in relationships + assert not any( + relationship_type == "DEPENDS_ON" for _, relationship_type, _ in relationships + ) + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_maps_python_root_with_purl_version_override( + tmp_path: pathlib.Path, +) -> None: + """Verify a purl version override does not stop the root matching.""" + ctx = make_sbom_ctx( + tmp_path, + sbom_settings=SbomSettings(), + package_overrides={"purl": {"version": "0.5.5+local.1"}}, + ) + document = sbom.generate_sbom( + ctx=ctx, + req=Requirement("test-pkg==0.5.5"), + version=Version("0.5.5"), + metadata=_metadata(), + ) + wheel_id = sbom._described_spdx_id(document) + wheel = next( + package for package in document["packages"] if package["SPDXID"] == wheel_id + ) + assert wheel["versionInfo"] == "0.5.5+local.1" + root_purl = "pkg:pypi/test-pkg@0.5.5" + maturin = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": { + "component": { + "type": "library", + "bom-ref": root_purl, + "name": "test-pkg", + "version": "0.5.5", + "purl": root_purl, + } + }, + "components": [ + { + "type": "library", + "bom-ref": "pkg:cargo/serde@1.0.0", + "name": "serde", + "version": "1.0.0", + "purl": "pkg:cargo/serde@1.0.0", + } + ], + } + sboms_dir = tmp_path / "sboms" + sboms_dir.mkdir() + (sboms_dir / "maturin.cdx.json").write_text(json.dumps(maturin)) + + sbom.merge_cyclonedx_sboms( + sbom=document, sboms_dir=sboms_dir, name="test-pkg", version="0.5.5" + ) + + assert len(document["packages"]) == 3 + serde = next( + package for package in document["packages"] if package["name"] == "serde" + ) + contains = { + relationship["relatedSpdxElement"] + for relationship in document["relationships"] + if relationship["relationshipType"] == "CONTAINS" + } + assert contains == {serde["SPDXID"]} + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_does_not_map_unrelated_python_root_to_wheel( + tmp_path: pathlib.Path, +) -> None: + """Verify an unrelated PyPI root remains its own SPDX package.""" + ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + document = sbom.generate_sbom( + ctx=ctx, + req=Requirement("pillow==12.3.0"), + version=Version("12.3.0"), + metadata=_metadata(), + ) + wheel_id = sbom._described_spdx_id(document) + root_purl = "pkg:pypi/unrelated@2.0" + dependency_purl = "pkg:rpm/almalinux/libXau@1.0.9-3.el8" + cyclonedx = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": { + "component": { + "type": "library", + "bom-ref": root_purl, + "name": "unrelated", + "version": "2.0", + "purl": root_purl, + } + }, + "components": [ + { + "type": "library", + "bom-ref": dependency_purl, + "name": "libXau", + "version": "1.0.9-3.el8", + "purl": dependency_purl, + } + ], + "dependencies": [ + {"ref": root_purl, "dependsOn": [dependency_purl]}, + ], + } + sboms_dir = tmp_path / "sboms" + sboms_dir.mkdir() + (sboms_dir / "custom.cyclonedx.json").write_text(json.dumps(cyclonedx)) + + sbom.merge_cyclonedx_sboms( + sbom=document, sboms_dir=sboms_dir, name="example-pkg", version="1.2.3" + ) + + packages = {package["name"]: package for package in document["packages"]} + assert {"unrelated", "libXau"} <= packages.keys() + assert packages["unrelated"]["SPDXID"] != wheel_id + relationships = { + ( + relationship["spdxElementId"], + relationship["relationshipType"], + relationship["relatedSpdxElement"], + ) + for relationship in document["relationships"] + } + assert ( + wheel_id, + "CONTAINS", + packages["unrelated"]["SPDXID"], + ) in relationships + assert ( + wheel_id, + "CONTAINS", + packages["libXau"]["SPDXID"], + ) in relationships + assert not any( + relationship_type == "DEPENDS_ON" for _, relationship_type, _ in relationships + ) + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_imports_nested_components( + tmp_path: pathlib.Path, +) -> None: + """Verify nested target components are imported and linked.""" + ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + document = sbom.generate_sbom( + ctx=ctx, + req=Requirement("example-pkg==1.2.3"), + version=Version("1.2.3"), + metadata=_metadata(), + ) + wheel_id = sbom._described_spdx_id(document) + root_purl = "pkg:cargo/example@1.0.0?download_url=file://." + target_purl = "pkg:cargo/example-target@1.0.0?download_url=file://../target" + nested_purl = "pkg:cargo/example-nested@1.0.0" + cyclonedx = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": { + "component": { + "type": "application", + "bom-ref": root_purl, + "name": "example", + "version": "1.0.0", + "purl": root_purl, + } + }, + "components": [ + { + "type": "library", + "bom-ref": target_purl, + "name": "example-target", + "version": "1.0.0", + "purl": target_purl, + "components": [ + { + "type": "library", + "bom-ref": nested_purl, + "name": "example-nested", + "version": "1.0.0", + "purl": nested_purl, + } + ], + } + ], + "dependencies": [ + {"ref": root_purl, "dependsOn": [target_purl]}, + {"ref": target_purl, "dependsOn": [nested_purl]}, + ], + } + sboms_dir = tmp_path / "sboms" + sboms_dir.mkdir() + (sboms_dir / "nested.cyclonedx.json").write_text(json.dumps(cyclonedx)) + + sbom.merge_cyclonedx_sboms( + sbom=document, sboms_dir=sboms_dir, name="example-pkg", version="1.2.3" + ) + + packages = {package["name"]: package for package in document["packages"]} + assert {"example", "example-target", "example-nested"} <= packages.keys() + relationships = { + ( + relationship["spdxElementId"], + relationship["relationshipType"], + relationship["relatedSpdxElement"], + ) + for relationship in document["relationships"] + } + assert { + package["externalRefs"][0]["referenceLocator"] + for package in ( + packages["example"], + packages["example-target"], + packages["example-nested"], + ) + } == { + "pkg:cargo/example@1.0.0", + "pkg:cargo/example-target@1.0.0", + "pkg:cargo/example-nested@1.0.0", + } + for package_name in ("example", "example-target", "example-nested"): + assert ( + wheel_id, + "CONTAINS", + packages[package_name]["SPDXID"], + ) in relationships + assert not any( + relationship_type == "DEPENDS_ON" for _, relationship_type, _ in relationships + ) + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_deduplicates_packages_by_purl( + tmp_path: pathlib.Path, +) -> None: + """Verify repeated Cargo components produce one SPDX package.""" + ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + document = sbom.generate_sbom( + ctx=ctx, + req=Requirement("example-pkg==1.2.3"), + version=Version("1.2.3"), + metadata=_metadata(), + ) + sboms_dir = tmp_path / "sboms" + _write_cyclonedx_sbom(sboms_dir, "first.cyclonedx.json") + _write_cyclonedx_sbom( + sboms_dir, + "second.cyclonedx.json", + root_name="another", + ) + + sbom.merge_cyclonedx_sboms( + sbom=document, sboms_dir=sboms_dir, name="example-pkg", version="1.2.3" + ) + + serde_packages = [ + package for package in document["packages"] if package["name"] == "serde" + ] + assert len(serde_packages) == 1 + assert ( + sum( + relationship["relationshipType"] == "CONTAINS" + and relationship["relatedSpdxElement"] == serde_packages[0]["SPDXID"] + for relationship in document["relationships"] + ) + == 1 + ) + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_ignores_invalid_and_other_formats( + tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture +) -> None: + """Verify unrelated or unreadable SBOM files do not break the merge.""" + ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + document = sbom.generate_sbom( + ctx=ctx, + req=Requirement("example-pkg==1.2.3"), + version=Version("1.2.3"), + metadata=_metadata(), + ) + sboms_dir = tmp_path / "sboms" + sboms_dir.mkdir() + (sboms_dir / "not-an-sbom.json").write_text('{"format": "other"}') + (sboms_dir / "broken.json").write_text("{") + (sboms_dir / "latin1.json").write_bytes(b'{"name": "caf\xe9"}') + (sboms_dir / "nested.json").write_text("[" * 1_000_000 + "]" * 1_000_000) + + with caplog.at_level("WARNING", logger="fromager.sbom"): + sbom.merge_cyclonedx_sboms( + sbom=document, sboms_dir=sboms_dir, name="example-pkg", version="1.2.3" + ) + + assert len(document["packages"]) == 2 + for filename in ("broken.json", "latin1.json", "nested.json"): + assert f"could not read SBOM file {sboms_dir / filename}" in caplog.text + _validate_spdx(document) + + +def _merge_components( + tmp_path: pathlib.Path, + components: list[dict[str, typing.Any]], + *, + root: dict[str, typing.Any] | None = None, + req: str = "example-pkg==1.2.3", + wheel_version: str = "1.2.3", +) -> dict[str, typing.Any]: + """Merge one CycloneDX document into a fresh SBOM and return the SBOM.""" + requirement = Requirement(req) + ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + document = sbom.generate_sbom( + ctx=ctx, + req=requirement, + version=Version(str(next(iter(requirement.specifier)).version)), + wheel_version=Version(wheel_version), + metadata=_metadata(), + ) + cyclonedx: dict[str, typing.Any] = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "components": components, + } + if root is not None: + cyclonedx["metadata"] = {"component": root} + sboms_dir = tmp_path / "sboms" + sboms_dir.mkdir(exist_ok=True) + (sboms_dir / "tool.cdx.json").write_text(json.dumps(cyclonedx)) + sbom.merge_cyclonedx_sboms( + sbom=document, + sboms_dir=sboms_dir, + name=requirement.name, + version=wheel_version, + ) + return document + + +def test_merge_cyclonedx_sboms_maps_purl_only_root_to_wheel( + tmp_path: pathlib.Path, +) -> None: + """A root without name/version fields matches the wheel by its purl.""" + root = {"type": "library", "purl": "pkg:pypi/example-pkg@1.2.3"} + serde = {"type": "library", "name": "serde", "purl": "pkg:cargo/serde@1.0.0"} + + document = _merge_components(tmp_path, [serde], root=root) + + assert [package["name"] for package in document["packages"][2:]] == ["serde"] + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_never_merges_into_upstream_entry( + tmp_path: pathlib.Path, +) -> None: + """A root whose purl equals the upstream purl stays a separate package. + + The wheel has a local version, so the root does not match the wheel and + its build-reported metadata must not land on the upstream entry. + """ + root = { + "type": "library", + "name": "example-pkg", + "version": "1.2.3", + "purl": "pkg:pypi/example-pkg@1.2.3", + "hashes": [{"alg": "SHA-256", "content": "a" * 64}], + "licenses": [{"expression": "MIT"}], + } + + document = _merge_components(tmp_path, [], root=root, wheel_version="1.2.3+local.1") + + _wheel, upstream, imported = document["packages"] + assert "checksums" not in upstream + assert upstream["licenseDeclared"] == "NOASSERTION" + assert imported["SPDXID"].startswith("SPDXRef-cyclonedx-") + contains = { + relationship["relatedSpdxElement"] + for relationship in document["relationships"] + if relationship["relationshipType"] == "CONTAINS" + } + assert contains == {imported["SPDXID"]} + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_skips_excluded_subtree(tmp_path: pathlib.Path) -> None: + """Children of an excluded component are not shipped either.""" + excluded = { + "type": "library", + "name": "build-tool", + "purl": "pkg:cargo/build-tool@1.0.0", + "scope": "excluded", + "components": [ + {"type": "library", "name": "child", "purl": "pkg:cargo/child@1.0.0"} + ], + } + + document = _merge_components(tmp_path, [excluded]) + + assert len(document["packages"]) == 2 + + +def test_merge_cyclonedx_sboms_deduplicates_by_canonical_purl( + tmp_path: pathlib.Path, +) -> None: + """The same purl with differently ordered qualifiers is one package.""" + components = [ + {"type": "library", "name": "serde", "purl": "pkg:cargo/serde@1.0.0?b=2&a=1"}, + {"type": "library", "name": "serde", "purl": "pkg:cargo/serde@1.0.0?a=1&b=2"}, + ] + + document = _merge_components(tmp_path, components) + + serde = [package for package in document["packages"] if package["name"] == "serde"] + assert len(serde) == 1 + assert serde[0]["externalRefs"][0]["referenceLocator"] == ( + "pkg:cargo/serde@1.0.0?a=1&b=2" + ) + + +def test_merge_cyclonedx_sboms_drops_invalid_purl(tmp_path: pathlib.Path) -> None: + """An unparseable purl is not emitted as a purl locator.""" + component = {"type": "library", "name": "broken", "purl": "not-a-purl"} + + document = _merge_components(tmp_path, [component]) + + broken = document["packages"][2] + assert broken["name"] == "broken" + assert "externalRefs" not in broken + _validate_spdx(document) + + +def test_merge_cyclonedx_sboms_declares_imported_license_refs( + tmp_path: pathlib.Path, +) -> None: + """Each imported LicenseRef gets one extracted licensing info entry.""" + components = [ + { + "type": "library", + "name": "a", + "purl": "pkg:cargo/a@1.0.0", + "licenses": [{"expression": "MIT AND LicenseRef-Proprietary-1.0"}], + }, + { + "type": "library", + "name": "b", + "purl": "pkg:cargo/b@1.0.0", + "licenses": [{"expression": "LicenseRef-Proprietary-1.0"}], + }, + ] + + document = _merge_components(tmp_path, components) + + assert document["hasExtractedLicensingInfos"] == [ + { + "licenseId": "LicenseRef-Proprietary-1.0", + "name": "Proprietary-1.0", + "extractedText": "NOASSERTION", + "comment": ( + "Declared in the build tool SBOM tool.cdx.json. The license " + "text is not available in that SBOM." + ), + } + ] + _validate_spdx(document) + + +def test_cyclonedx_license_expression_groups_lowercase_or() -> None: + """Lowercase operators are canonicalized and grouped too.""" + component = { + "licenses": [ + {"expression": "MIT or Apache-2.0"}, + {"expression": "BSD-3-Clause"}, + ] + } + + assert ( + sbom._cyclonedx_license_expression(component) + == "(MIT OR Apache-2.0) AND BSD-3-Clause" + ) + + +def test_cyclonedx_license_expression_keeps_license_ref() -> None: + """LicenseRef- identifiers are valid SPDX and must be kept.""" + component = {"licenses": [{"expression": "MIT AND LicenseRef-Proprietary-1.0"}]} + + assert ( + sbom._cyclonedx_license_expression(component) + == "MIT AND LicenseRef-Proprietary-1.0" + ) + + +def test_cyclonedx_checksums_normalizes_case_and_checks_length() -> None: + """SPDX requires lowercase hex of the exact digest length.""" + component = { + "hashes": [ + {"alg": "SHA-256", "content": "A" * 64}, + {"alg": "SHA-1", "content": "b" * 64}, + {"alg": "MD5", "content": "c" * 32}, + ] + } + + assert sbom._cyclonedx_checksums(component) == [ + {"algorithm": "SHA256", "checksumValue": "a" * 64}, + {"algorithm": "MD5", "checksumValue": "c" * 32}, + ] diff --git a/tests/test_wheels.py b/tests/test_wheels.py index f9fd22f0..fb14557b 100644 --- a/tests/test_wheels.py +++ b/tests/test_wheels.py @@ -418,6 +418,34 @@ def test_add_extra_metadata_generates_sbom_when_enabled( wheel_dir.mkdir() wheel_file = wheel_dir / "test_pkg-1.0.0+local.1-py3-none-any.whl" + native_sbom = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": { + "component": { + "type": "library", + "bom-ref": "pkg:cargo/test-pkg@1.0.0", + "name": "test-pkg", + "version": "1.0.0", + "purl": "pkg:cargo/test-pkg@1.0.0", + } + }, + "components": [ + { + "type": "library", + "bom-ref": "pkg:cargo/serde@1.0.0", + "name": "serde", + "version": "1.0.0", + "purl": "pkg:cargo/serde@1.0.0", + } + ], + "dependencies": [ + { + "ref": "pkg:cargo/test-pkg@1.0.0", + "dependsOn": ["pkg:cargo/serde@1.0.0"], + } + ], + } with zipfile.ZipFile(wheel_file, "w") as zf: zf.writestr("test_pkg/__init__.py", "") zf.writestr( @@ -428,6 +456,10 @@ def test_add_extra_metadata_generates_sbom_when_enabled( "test_pkg-1.0.0+local.1.dist-info/WHEEL", "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", ) + zf.writestr( + "test_pkg-1.0.0+local.1.dist-info/sboms/rust.cyclonedx.json", + json.dumps(native_sbom), + ) mock_run.return_value = "" @@ -438,22 +470,14 @@ def test_add_extra_metadata_generates_sbom_when_enabled( sdist_dir.mkdir() # Capture the wheel contents before repack by inspecting what wheel pack receives - captured_contents: list[str] = [] - captured_sbom: dict[str, typing.Any] = {} + captured_contents: dict[str, str] = {} def fake_run(cmd: list[str], **kwargs: object) -> str: # wheel pack is called with the unpacked dir as second arg unpacked_dir = pathlib.Path(cmd[2]) for f in unpacked_dir.rglob("*"): if f.is_file(): - captured_contents.append(str(f.relative_to(unpacked_dir))) - sbom_file = ( - unpacked_dir - / "test_pkg-1.0.0+local.1.dist-info" - / "sboms" - / ("fromager.spdx.json") - ) - captured_sbom.update(json.loads(sbom_file.read_text())) + captured_contents[str(f.relative_to(unpacked_dir))] = f.read_text() repacked.touch() return "" @@ -468,12 +492,25 @@ def fake_run(cmd: list[str], **kwargs: object) -> str: wheel_file=wheel_file, ) - # Verify the SBOM file was added to the unpacked wheel before repacking - assert any("sboms/fromager.spdx.json" in c for c in captured_contents) + # Verify the canonical SBOM was merged and the native file was preserved. + assert ( + "test_pkg-1.0.0+local.1.dist-info/sboms/fromager.spdx.json" in captured_contents + ) + assert ( + "test_pkg-1.0.0+local.1.dist-info/sboms/rust.cyclonedx.json" + in captured_contents + ) + merged_sbom = json.loads( + captured_contents["test_pkg-1.0.0+local.1.dist-info/sboms/fromager.spdx.json"] + ) + assert {package["name"] for package in merged_sbom["packages"]} >= { + "test-pkg", + "serde", + } # The SBOM must identify the wheel with its local version - assert captured_sbom["name"] == "test-pkg-1.0.0+local.1" - wheel_entry, upstream_entry = captured_sbom["packages"] + assert merged_sbom["name"] == "test-pkg-1.0.0+local.1" + wheel_entry, upstream_entry = merged_sbom["packages"][:2] assert wheel_entry["versionInfo"] == "1.0.0+local.1" assert wheel_entry["externalRefs"][0]["referenceLocator"] == ( "pkg:pypi/test-pkg@1.0.0%2Blocal.1"