Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions e2e/ci_config_suite.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ test_section "configuration tests"
run_test "build_settings"
run_test "override"
run_test "extra_metadata"
run_test "sbom_licenses"
run_test "lint_requirements"

finish_suite
3 changes: 3 additions & 0 deletions e2e/sbom_settings.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
sbom:
supplier: "Organization: ExampleCo"
namespace: "https://sbom.test"
25 changes: 25 additions & 0 deletions e2e/test_sbom_licenses.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#!/bin/bash
# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-

# Test that the SBOM embedded in each built wheel carries the license the
# wheel declares in its METADATA (issue #1378).
#
# idna 3.6 only has the ambiguous "BSD License" classifier, so it must get
# a LicenseRef with extracted licensing info. tomli 2.0.1 has the "MIT
# License" classifier, which maps cleanly to MIT.

SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "$SCRIPTDIR/common.sh"

fromager \
--log-file="$OUTDIR/bootstrap.log" \
--error-log-file="$OUTDIR/fromager-errors.log" \
--sdists-repo="$OUTDIR/sdists-repo" \
--wheels-repo="$OUTDIR/wheels-repo" \
--work-dir="$OUTDIR/work-dir" \
--settings-file="$SCRIPTDIR/sbom_settings.yaml" \
bootstrap 'idna==3.6' 'tomli==2.0.1'

find "$OUTDIR/wheels-repo/downloads/" -name '*.whl'

python3 "$SCRIPTDIR/verify_sbom_licenses.py" "$OUTDIR/wheels-repo/downloads"
111 changes: 111 additions & 0 deletions e2e/verify_sbom_licenses.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
#!/usr/bin/env python3
"""Verify that wheel SBOMs carry the license declared in wheel metadata.

`test_sbom_licenses.sh` passes the directory holding the built wheels. Every
failed check is reported, and the exit status is non-zero when any check
fails.
"""

import json
import pathlib
import re
import sys
import zipfile
from typing import Any

SPDX_FILE_SUFFIX = ".dist-info/sboms/fromager.spdx.json"
LICENSE_REF_RE = re.compile(r"LicenseRef-[A-Za-z0-9.\-]+")

# idna 3.6 only declares the ambiguous "BSD License" classifier, so its
# license becomes a LicenseRef with the classifier as extracted text.
IDNA_LICENSE_REF = "LicenseRef-license-osi-approved-bsd-license-a9d12a80"
IDNA_EXTRACTED_INFO = [
{
"licenseId": IDNA_LICENSE_REF,
"extractedText": "License :: OSI Approved :: BSD License",
}
]


def sbom_of(wheel: pathlib.Path) -> dict[str, Any]:
"""Return the SBOM embedded in *wheel*."""
with zipfile.ZipFile(wheel) as zf:
(name,) = [n for n in zf.namelist() if n.endswith(SPDX_FILE_SUFFIX)]
doc: dict[str, Any] = json.loads(zf.read(name))
return doc


def check_license_fields(
wheel: pathlib.Path, doc: dict[str, Any], failures: list[str]
) -> None:
"""Check the invariants that must hold for every built wheel."""
wheel_pkg, upstream_pkg = doc["packages"]
for key in ("licenseDeclared", "licenseConcluded", "licenseComments"):
if key not in wheel_pkg or key not in upstream_pkg:
failures.append(f"{wheel.name}: {key} missing from a package")
elif wheel_pkg[key] != upstream_pkg[key]:
failures.append(f"{wheel.name}: {key} differs between packages")
if wheel_pkg.get("licenseConcluded") != "NOASSERTION":
failures.append(f"{wheel.name}: licenseConcluded is not NOASSERTION")
declared_ids = {
info["licenseId"] for info in doc.get("hasExtractedLicensingInfos", [])
}
for ref in LICENSE_REF_RE.findall(wheel_pkg.get("licenseDeclared", "")):
if ref not in declared_ids:
failures.append(f"{wheel.name}: {ref} has no extracted licensing info")


def check_idna(downloads: pathlib.Path, failures: list[str]) -> None:
"""Check the ambiguous-classifier case in the idna 3.6 wheel."""
idna = sbom_of(next(downloads.glob("idna-3.6-*.whl")))
pkg = idna["packages"][0]
declared = pkg.get("licenseDeclared")
if declared != IDNA_LICENSE_REF:
failures.append(f"idna: licenseDeclared is {declared!r}")
extracted = idna.get("hasExtractedLicensingInfos")
if extracted != IDNA_EXTRACTED_INFO:
failures.append(f"idna: unexpected hasExtractedLicensingInfos {extracted!r}")
if "(License classifiers)" not in pkg.get("licenseComments", ""):
failures.append("idna: comment does not name the source")


def check_tomli(downloads: pathlib.Path, failures: list[str]) -> None:
"""Check that the unambiguous "MIT License" classifier maps cleanly to MIT."""
tomli = sbom_of(next(downloads.glob("tomli-2.0.1-*.whl")))
declared = tomli["packages"][0].get("licenseDeclared")
if declared != "MIT":
failures.append(f"tomli: licenseDeclared is {declared!r}")
if "hasExtractedLicensingInfos" in tomli:
failures.append("tomli: unexpected hasExtractedLicensingInfos")


def verify(downloads: pathlib.Path) -> tuple[list[str], int]:
"""Return the failed checks and the number of wheels checked."""
failures: list[str] = []
wheels = sorted(downloads.glob("*.whl"))
if not wheels:
failures.append(f"no wheels found in {downloads}")
for wheel in wheels:
check_license_fields(wheel, sbom_of(wheel), failures)

check_idna(downloads, failures)
check_tomli(downloads, failures)
return failures, len(wheels)


def main(argv: list[str]) -> int:
"""Run the checks against the downloads directory in *argv*."""
if len(argv) != 1:
program = pathlib.Path(sys.argv[0]).name
print(f"usage: {program} DOWNLOADS_DIR", file=sys.stderr)
return 2

failures, wheel_count = verify(pathlib.Path(argv[0]))
for failure in failures:
print(f"FAIL: {failure}", file=sys.stderr)
print(f"checked {wheel_count} wheels, {len(failures)} failure(s)")
return 1 if failures else 0


if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
8 changes: 4 additions & 4 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -250,13 +250,13 @@ dependencies = [
]
[tool.hatch.envs.lint.scripts]
check = [
"ruff check src tests",
"ruff format --check src tests",
"ruff check src tests e2e",
"ruff format --check src tests e2e",
"python ./e2e/mergify_lint.py",
]
fix = [
"ruff format src tests docs",
"ruff check --fix src tests docs",
"ruff format src tests e2e docs",
"ruff check --fix src tests e2e docs",
]
precommit = "pre-commit run --all-files"
install-hooks = "pre-commit install"
Expand Down
13 changes: 13 additions & 0 deletions src/fromager/packagesettings/_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,19 @@ class SbomSettings(pydantic.BaseModel):
repository_url: "https://example.com/simple"
creators:
- "Organization: ExampleCo"

Both package entries in the SBOM carry ``licenseDeclared`` from the
license the built wheel declares in its ``METADATA``. Sources are tried
in order: ``License-Expression``, unambiguous license classifiers, then
a ``License`` value that is an exact SPDX expression or a known common
spelling. Anything else is recorded as a ``LicenseRef-`` id with the raw
declaration in ``hasExtractedLicensingInfos``. ``licenseConcluded`` is
always ``NOASSERTION`` because fromager does not audit licenses, and
``licenseComments`` says the value is inherited from upstream. A built
wheel without ``METADATA`` fails when SBOMs are enabled.

.. versionchanged:: 0.104.0
Populate SBOM license fields from wheel metadata.
"""

model_config = MODEL_CONFIG
Expand Down
11 changes: 10 additions & 1 deletion src/fromager/pkgmetadata/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,19 @@
"""

from .pep376 import dist_info_name, verbatim_dist_name
from .pep639 import license_from_metadata, license_from_metadata_values
from .pep639 import (
DeclaredLicense,
LicenseSource,
declared_license_from_raw_metadata,
license_from_metadata,
license_from_metadata_values,
)
from .pep753 import normalize_project_urls, project_urls_from_metadata

__all__ = (
"DeclaredLicense",
"LicenseSource",
"declared_license_from_raw_metadata",
"dist_info_name",
"license_from_metadata",
"license_from_metadata_values",
Expand Down
136 changes: 134 additions & 2 deletions src/fromager/pkgmetadata/pep639.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,13 @@
- https://spdx.org/licenses/
"""

import dataclasses
import enum
import inspect

from license_expression import ExpressionError, LicenseExpression, get_spdx_licensing
from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression
from packaging.metadata import Metadata
from packaging.metadata import Metadata, RawMetadata


def license_from_metadata(metadata: Metadata) -> LicenseExpression:
Expand All @@ -32,7 +36,9 @@ def license_from_metadata_values(
2. Fall back to *license_text*. Perform some unambiguous translatons
(e.g. ``Apache 2`` to ``Apache-2.0``) and attempt to parse the string
as SPDX license expression.
3. Finally fall back to trove classifiers.
3. Finally fall back to trove classifiers. Several mapped classifiers
are joined with ``AND``; `declared_license_from_raw_metadata()`
treats them as ambiguous instead.

Raises an exception if license is missing, ambiguous, or not a valid
SPDX license expression.
Expand All @@ -59,6 +65,108 @@ def license_from_metadata_values(
raise ExceptionGroup("unable to detect license", errors)


class LicenseSource(enum.StrEnum):
"""Core metadata field a declared license was taken from"""

LICENSE_EXPRESSION = "License-Expression"
CLASSIFIERS = "License classifiers"
LICENSE = "License"


@dataclasses.dataclass(frozen=True)
class DeclaredLicense:
"""License as declared in a package's core metadata

``raw`` keeps the declaration (license classifiers joined by newlines)
so it can be quoted when ``spdx`` is ``None``, which means it has no
unambiguous SPDX mapping.
"""

source: LicenseSource
raw: str
spdx: str | None


def declared_license_from_raw_metadata(
metadata: RawMetadata,
) -> DeclaredLicense | None:
"""Detect the declared license for SBOMs without guessing

Unlike `license_from_metadata_values()`, this never raises and reports
Comment thread
rd4398 marked this conversation as resolved.
Resolved
an ambiguous declaration instead of guessing an SPDX id, so the caller
can record it verbatim. Sources are tried in the order agreed in
issue #1378: ``License-Expression``, unambiguous license classifiers,
then free text that is an exact SPDX expression or a curated mapping.
Without a clean mapping, the first declaration present is returned
with ``spdx=None``, preferring free text over ambiguous classifiers
because it is usually more specific. Classifiers that name more than
one license are ambiguous here, while `license_from_metadata_values()`
joins them with ``AND``.

Returns ``None`` when the metadata declares no license at all.
"""
expression = _declared_value(metadata.get("license_expression", "").strip())
text = _declared_value(inspect.cleandoc(metadata.get("license", "")).strip())
classifiers = [
c for c in metadata.get("classifiers", []) if c.startswith("License ::")
]
joined_classifiers = "\n".join(classifiers)

canonical_expression = _canonical_spdx(expression) if expression else None
if expression and canonical_expression:
return DeclaredLicense(
LicenseSource.LICENSE_EXPRESSION, expression, canonical_expression
)
classifier_spdx = _single_classifier_license(classifiers)
if classifier_spdx:
return DeclaredLicense(
LicenseSource.CLASSIFIERS, joined_classifiers, classifier_spdx
)
if text:
text_spdx = _strict_text_to_spdx(text)
if text_spdx is not None:
return DeclaredLicense(LicenseSource.LICENSE, text, text_spdx)

if expression:
return DeclaredLicense(LicenseSource.LICENSE_EXPRESSION, expression, None)
if text:
return DeclaredLicense(LicenseSource.LICENSE, text, None)
if classifiers:
return DeclaredLicense(LicenseSource.CLASSIFIERS, joined_classifiers, None)
return None


# license classifiers that do not name a license
_NON_LICENSE_CLASSIFIERS = frozenset(
{"License :: DFSG approved", "License :: OSI Approved"}
)


def _single_classifier_license(classifiers: list[str]) -> str | None:
"""SPDX id of the license classifiers if they name exactly one license

Several different licenses may mean a choice (OR) or a combination
(AND); classifiers cannot say which, so that is not guessed.
"""
mapped: set[str] = set()
for classifier in classifiers:
if classifier in _NON_LICENSE_CLASSIFIERS:
continue
# ambiguous, or a license classifier missing from the table
spdx = _TROVE_SPDX.get(classifier)
if spdx is None:
return None
mapped.add(spdx)
return mapped.pop() if len(mapped) == 1 else None


def _declared_value(value: str) -> str | None:
"""Treat empty values and the legacy setuptools ``UNKNOWN`` as absent"""
if not value or value.casefold() == "unknown":
return None
return value


_SPDX = get_spdx_licensing()


Expand Down Expand Up @@ -106,6 +214,30 @@ def _license_text_to_spdx(text: str) -> LicenseExpression:
raise ValueError(text[:100])


def _canonical_spdx(text: str) -> str | None:
"""Canonical form of a valid SPDX expression, else ``None``

Uses the same validation build backends apply to ``License-Expression``
(PEP 639). Unlike the license library, it rejects shorthands such as
``GPL`` and ``WITH`` misuse, and keeps deprecated SPDX ids as written.
"""
try:
return canonicalize_license_expression(text)
except InvalidLicenseExpression:
return None


def _strict_text_to_spdx(text: str) -> str | None:
"""Map free text only when the mapping is certain, else ``None``

Free text must be a curated mapping or a valid SPDX expression;
anything else would overstate what upstream declared.
"""
if text in _LICENSE_STRING_TO_SPDX:
return _canonical_spdx(_LICENSE_STRING_TO_SPDX[text])
return _canonical_spdx(text)


# unambiguous text to SPDX
# The keys are common cases seen in the wild on PyPI.org
_LICENSE_STRING_TO_SPDX: dict[str, str] = {
Expand Down
Loading
Loading