Skip to content

Commit 022019a

Browse files
Merge pull request #5346 from aboutcode-org/maven-package-assembly
Fix maven package assembly in uberjars
2 parents d7dd246 + 9f91bef commit 022019a

518 files changed

Lines changed: 3915 additions & 718 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎src/packagedcode/jar_manifest.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -341,12 +341,14 @@ def dget(s):
341341

342342
comment = dget('Comment')
343343

344-
if comment or doc_url:
344+
if comment or doc_url or i_title:
345345
package["extra_data"] = {}
346346
if comment:
347347
package["extra_data"]['notes'] = comment
348348
if doc_url:
349349
package["extra_data"]['documentation_url'] = doc_url
350+
if i_title:
351+
package["extra_data"]['pname'] = i_title
350352

351353
return package
352354

‎src/packagedcode/maven.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,12 +102,11 @@ def assemble(cls, package_data, resource, codebase, package_adder=models.add_to_
102102
if fnmatchcase(r.path, nested_pom_xml_path_pattern):
103103
pom_xmls.append(r)
104104

105-
if len(pom_xmls) > 1:
105+
if len(pom_xmls) > 1 and not manifests:
106106
yield from MavenPomXmlHandlerMixin.assemble(package_data, resource, codebase)
107107
return
108108

109-
if manifests and pom_xmls:
110-
# raise Exception(resource.path, meta_inf_resource, datafile_name_patterns, package_adder)
109+
elif manifests and pom_xmls:
111110
parent_resource = meta_inf_resource.parent(codebase)
112111
if not parent_resource:
113112
parent_resource = meta_inf_resource
@@ -116,9 +115,10 @@ def assemble(cls, package_data, resource, codebase, package_adder=models.add_to_
116115
directory=meta_inf_resource,
117116
codebase=codebase,
118117
package_adder=package_adder,
119-
ignore_name_check=True,
118+
relaxed_purl_comparision=True,
120119
parent_resource=parent_resource,
121120
)
121+
122122
elif manifests and not pom_xmls:
123123
yield from JavaJarManifestHandlerMixin.assemble(package_data, resource, codebase, package_adder)
124124
elif pom_xmls and not manifests:
@@ -1293,6 +1293,10 @@ def _parse(
12931293
base_url=base_url,
12941294
))
12951295

1296+
extra_data = {}
1297+
if pname:
1298+
extra_data['pname'] = pname
1299+
12961300
# FIXME: there are still other data to map in a PackageData
12971301
package_data = dict(
12981302
datasource_id=datasource_id,
@@ -1309,6 +1313,7 @@ def _parse(
13091313
dependencies=get_dependencies(pom),
13101314
source_packages=source_packages,
13111315
bug_tracking_url=bug_tracking_url,
1316+
extra_data=extra_data,
13121317
**urls,
13131318
)
13141319
return MavenPackageData.from_data(package_data, package_only)

‎src/packagedcode/models.py‎

Lines changed: 85 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1254,7 +1254,7 @@ def assemble_from_many(
12541254
pkgdata_resources,
12551255
codebase,
12561256
package_adder=add_to_package,
1257-
ignore_name_check=False,
1257+
relaxed_purl_comparision=False,
12581258
parent_resource=None,
12591259
):
12601260
"""
@@ -1287,34 +1287,40 @@ def assemble_from_many(
12871287
base_resource = parent_resource
12881288

12891289
# process each package in sequence. The first item creates a package and
1290-
# the other only update
1290+
# thereafter we create new packages to update the old packages
12911291
# We are saving the Packages, Dependencies, and Resources in lists until
12921292
# after we go through `pkgdata_resources` for all Package data, then we
12931293
# yield Packages, then Dependencies, then Resources.
12941294
dependencies = []
12951295
resources = []
12961296
resources_from_package = []
1297+
packages_by_purl = {}
1298+
12971299
for package_data, resource in pkgdata_resources:
12981300
if not base_resource:
12991301
base_resource = resource
13001302

1301-
if not package:
1302-
# create package from the first item first package_data
1303-
if package_data.purl:
1304-
package = Package.from_package_data(
1305-
package_data=package_data,
1306-
datafile_path=resource.path,
1307-
)
1308-
package_uid = package.package_uid
1309-
else:
1310-
# FIXME: What is the package_data is NOT for the same package as package?
1311-
# FIXME: What if the update did not do anything? (it does return True or False)
1312-
# FIXME: There we would be missing out packges AND/OR errors
1303+
compatible_package = get_compatible_package(
1304+
package_data=package_data,
1305+
packages_by_purl=packages_by_purl,
1306+
relaxed_purl_comparision=True,
1307+
)
1308+
if package_data.purl and not compatible_package:
1309+
package = Package.from_package_data(
1310+
package_data=package_data,
1311+
datafile_path=resource.path,
1312+
)
1313+
package_uid = package.package_uid
1314+
packages_by_purl[package.purl] = package
1315+
1316+
elif compatible_package:
1317+
package = compatible_package
13131318
package.update(
13141319
package_data=package_data,
13151320
datafile_path=resource.path,
1316-
ignore_name_check=ignore_name_check,
1321+
relaxed_purl_comparision=relaxed_purl_comparision,
13171322
)
1323+
package_uid = package.package_uid
13181324

13191325
if package_uid:
13201326
resources_from_package.append((package_uid, resource,))
@@ -1334,7 +1340,7 @@ def assemble_from_many(
13341340
resources.append(resource)
13351341

13361342
# Yield Packages, Dependencies, and Resources
1337-
if package:
1343+
for package in packages_by_purl.values():
13381344
package.populate_license_fields()
13391345
yield package
13401346
yield from dependencies
@@ -1419,7 +1425,7 @@ def assemble_from_many_datafiles_in_directory(
14191425
directory,
14201426
codebase,
14211427
package_adder=add_to_package,
1422-
ignore_name_check=False,
1428+
relaxed_purl_comparision=False,
14231429
parent_resource=None,
14241430
):
14251431
"""
@@ -1440,7 +1446,7 @@ def assemble_from_many_datafiles_in_directory(
14401446
multiple PackageData for unrelated Packages.
14411447
"""
14421448
if TRACE:
1443-
logger_debug(f'assemble_from_many_datafiles: datafile_name_patterns: {datafile_name_patterns!r}')
1449+
logger_debug(f'assemble_from_many_datafiles: directory: {directory} datafile_name_patterns: {datafile_name_patterns!r}')
14441450

14451451
if not codebase.has_single_resource:
14461452
resources = [
@@ -1471,7 +1477,7 @@ def assemble_from_many_datafiles_in_directory(
14711477
pkgdata_resources=pkgdata_resources,
14721478
codebase=codebase,
14731479
package_adder=package_adder,
1474-
ignore_name_check=ignore_name_check,
1480+
relaxed_purl_comparision=relaxed_purl_comparision,
14751481
parent_resource=parent_resource,
14761482
)
14771483

@@ -1713,7 +1719,7 @@ def update(
17131719
include_version=True,
17141720
include_qualifiers=False,
17151721
include_subpath=False,
1716-
ignore_name_check=False,
1722+
relaxed_purl_comparision=False,
17171723
default_relation='AND',
17181724
licensing=Licensing(),
17191725
):
@@ -1748,7 +1754,8 @@ def update(
17481754
include_version=include_version,
17491755
include_qualifiers=include_qualifiers,
17501756
include_subpath=include_subpath,
1751-
ignore_name_check=ignore_name_check,
1757+
relaxed_purl_comparision=relaxed_purl_comparision,
1758+
pname_match=does_pname_match(self, package_data),
17521759
):
17531760
if TRACE_UPDATE:
17541761
logger_debug(f'update: skipping: {self.purl} is not compatible with: {package_data.purl}')
@@ -1853,18 +1860,27 @@ def get_packages_files(self, codebase):
18531860
yield resource
18541861

18551862

1863+
# In certain cases package manifests should be assembled even if
1864+
# the PURLs are slightly different
1865+
compatible_types = {
1866+
"maven": "jar",
1867+
"jar": "maven",
1868+
}
1869+
1870+
18561871
def is_compatible(
18571872
purl1,
18581873
purl2,
18591874
include_version=True,
18601875
include_qualifiers=True,
18611876
include_subpath=True,
1862-
ignore_name_check=False,
1877+
relaxed_purl_comparision=False,
1878+
pname_match=False,
18631879
):
18641880
"""
18651881
Return True if the ``purl1`` PackageURL-like object is compatible with
18661882
the ``purl2`` PackageURL-like object, e.g. it is about the same package.
1867-
PackageData objectys are PackageURL-like.
1883+
PackageData objects are PackageURL-like.
18681884
18691885
For example::
18701886
>>> p1 = PackageURL.from_string('pkg:deb/libncurses5@6.1-1ubuntu1.18.04?arch=arm64')
@@ -1887,17 +1903,23 @@ def is_compatible(
18871903
>>> is_compatible(p1, p5, include_subpath=False)
18881904
True
18891905
"""
1890-
if ignore_name_check:
1891-
is_compatible = (
1892-
purl1.type == purl2.type
1893-
and purl1.namespace == purl2.namespace
1906+
is_compatible_purl = (
1907+
purl1.type == purl2.type
1908+
and purl1.namespace == purl2.namespace
1909+
and purl1.name == purl2.name
1910+
)
1911+
if relaxed_purl_comparision:
1912+
is_compatible_type = (
1913+
purl1.type in compatible_types
1914+
and purl2.type == compatible_types.get(purl1.type)
18941915
)
1895-
else:
18961916
is_compatible = (
1897-
purl1.type == purl2.type
1898-
and purl1.namespace == purl2.namespace
1899-
and purl1.name == purl2.name
1917+
is_compatible_purl or (is_compatible_type and pname_match) or
1918+
(is_compatible_type and purl1.type != purl2.type and purl1.namespace == purl2.namespace)
19001919
)
1920+
else:
1921+
is_compatible = is_compatible_purl
1922+
19011923
if include_version:
19021924
is_compatible = is_compatible and (purl1.version == purl2.version)
19031925

@@ -1910,6 +1932,38 @@ def is_compatible(
19101932
return is_compatible
19111933

19121934

1935+
def get_compatible_package(
1936+
package_data,
1937+
packages_by_purl,
1938+
include_version=True,
1939+
include_qualifiers=True,
1940+
include_subpath=True,
1941+
relaxed_purl_comparision=False,
1942+
):
1943+
if not package_data.purl:
1944+
return
1945+
1946+
for purl_to_compare, package in packages_by_purl.items():
1947+
if is_compatible(
1948+
purl1=PackageURL.from_string(purl_to_compare),
1949+
purl2=PackageURL.from_string(package_data.purl),
1950+
include_version=include_version,
1951+
include_qualifiers=include_qualifiers,
1952+
include_subpath=include_subpath,
1953+
relaxed_purl_comparision=relaxed_purl_comparision,
1954+
pname_match=does_pname_match(package, package_data),
1955+
):
1956+
return package
1957+
1958+
1959+
def does_pname_match(package1, package2):
1960+
pname1 = package1.extra_data.get("pname")
1961+
pname2 = package2.extra_data.get("pname")
1962+
if pname1 and pname2 and pname1 == pname2:
1963+
return True
1964+
1965+
return False
1966+
19131967
@attr.attributes(slots=True)
19141968
class PackageWithResources(Package):
19151969
"""

‎tests/formattedcode/data/common/manifests-expected.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,9 @@
6060
],
6161
"is_private": false,
6262
"is_virtual": false,
63-
"extra_data": {},
63+
"extra_data": {
64+
"pname": "Enterprise JavaBeans (EJB) 3.0"
65+
},
6466
"repository_homepage_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/",
6567
"repository_download_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/persistence-api-1.0.jar",
6668
"api_data_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/persistence-api-1.0.pom",
@@ -1162,7 +1164,9 @@
11621164
"file_references": [],
11631165
"is_private": false,
11641166
"is_virtual": false,
1165-
"extra_data": {},
1167+
"extra_data": {
1168+
"pname": "Enterprise JavaBeans (EJB) 3.0"
1169+
},
11661170
"dependencies": [],
11671171
"repository_homepage_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/",
11681172
"repository_download_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/persistence-api-1.0.jar",

‎tests/formattedcode/data/common/manifests-expected.jsonlines‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,21 @@
99
"--info": true,
1010
"--json-lines": "<file>",
1111
"--license": true,
12-
"--package": true
12+
"--package": true,
13+
"--processes": "-1"
1314
},
1415
"notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
15-
"output_format_version": "4.0.0",
16+
"output_format_version": "4.1.0",
1617
"message": null,
1718
"errors": [],
1819
"warnings": [],
1920
"extra_data": {
2021
"system_environment": {
2122
"operating_system": "linux",
2223
"cpu_architecture": "64",
23-
"platform": "Linux-5.15.0-141-generic-x86_64-with-glibc2.35",
24-
"platform_version": "#151-Ubuntu SMP Sun May 18 21:35:19 UTC 2025",
25-
"python_version": "3.10.12 (main, Feb 4 2025, 14:57:36) [GCC 11.4.0]"
24+
"platform": "Linux-5.15.0-198-generic-x86_64-with-glibc2.35",
25+
"platform_version": "#208-Ubuntu SMP Fri Sep 4 10:22:38 UTC 2026",
26+
"python_version": "3.10.12 (main, Aug 31 2026, 10:18:17) [GCC 11.4.0]"
2627
},
2728
"spdx_license_list_version": "3.28",
2829
"files_count": 4
@@ -92,7 +93,9 @@
9293
],
9394
"is_private": false,
9495
"is_virtual": false,
95-
"extra_data": {},
96+
"extra_data": {
97+
"pname": "Enterprise JavaBeans (EJB) 3.0"
98+
},
9699
"repository_homepage_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/",
97100
"repository_download_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/persistence-api-1.0.jar",
98101
"api_data_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/persistence-api-1.0.pom",
@@ -1208,7 +1211,9 @@
12081211
"file_references": [],
12091212
"is_private": false,
12101213
"is_virtual": false,
1211-
"extra_data": {},
1214+
"extra_data": {
1215+
"pname": "Enterprise JavaBeans (EJB) 3.0"
1216+
},
12121217
"dependencies": [],
12131218
"repository_homepage_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/",
12141219
"repository_download_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/persistence-api-1.0.jar",

0 commit comments

Comments
 (0)