Skip to content

Commit a686c61

Browse files
committed
Add test for remove_ghost_packages pipeline
Signed-off-by: Keshav Priyadarshi <git@keshav.space>
1 parent 7a72929 commit a686c61

4 files changed

Lines changed: 124 additions & 30 deletions

File tree

‎vulnerabilities/pipelines/__init__.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,29 @@
66
# See https://github.com/nexB/vulnerablecode for support or download.
77
# See https://aboutcode.org for more information about nexB OSS projects.
88
#
9+
import logging
10+
from datetime import datetime
11+
from datetime import timezone
912

1013
from aboutcode.pipeline import BasePipeline
1114

1215
from vulnerabilities.utils import classproperty
1316

17+
module_logger = logging.getLogger(__name__)
18+
1419

1520
class VulnerableCodePipeline(BasePipeline):
21+
def log(self, message, level=logging.INFO):
22+
"""Log the given `message` to the current module logger and execution_log."""
23+
now_local = datetime.now(timezone.utc).astimezone()
24+
timestamp = now_local.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
25+
message = f"{timestamp} {message}"
26+
module_logger.log(level, message)
27+
self.append_to_log(message)
28+
1629
@classproperty
1730
def qualified_name(cls):
1831
"""
19-
Fully qualified name prefixed with the module name of the improver used in logging.
32+
Fully qualified name prefixed with the module name of the pipeline used in logging.
2033
"""
2134
return f"{cls.__module__}.{cls.__qualname__}"

‎vulnerabilities/pipelines/remove_ghost_packages.py‎

Lines changed: 38 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
# See https://aboutcode.org for more information about nexB OSS projects.
88
#
99

10+
import logging
11+
1012
from aboutcode.pipeline import LoopProgress
1113
from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS
1214
from fetchcode.package_versions import versions
@@ -18,47 +20,61 @@
1820

1921

2022
class RemoveGhostPackagePipeline(VulnerableCodePipeline):
23+
"""Detect and remove packages that do not exist upstream."""
24+
2125
@classmethod
2226
def steps(cls):
2327
return (cls.remove_ghost_packages,)
2428

2529
def remove_ghost_packages(self):
26-
"""
27-
Use fetchcode to validate the package indeed exists upstream.
28-
"""
29-
interesting_packages_qs = (
30-
Package.objects.filter(type__in=SUPPORTED_ECOSYSTEMS)
31-
.exclude(qualifiers__isnull=True)
32-
.exclude(subpath__isnull=True)
33-
)
30+
detect_and_remove_ghost_packages(logger=self.log)
3431

35-
distinct_packages = interesting_packages_qs.values("type", "namespace", "name").distinct(
36-
"type", "namespace", "name"
37-
)
3832

39-
distinct_packages_count = distinct_packages.count()
40-
package_iterator = distinct_packages.iterator(chunk_size=2000)
41-
progress = LoopProgress(total_iterations=distinct_packages_count, logger=self.log)
33+
def detect_and_remove_ghost_packages(logger=None):
34+
"""Use fetchcode to validate the package indeed exists upstream."""
35+
interesting_packages_qs = (
36+
Package.objects.filter(type__in=SUPPORTED_ECOSYSTEMS)
37+
.filter(qualifiers="")
38+
.filter(subpath="")
39+
)
40+
41+
distinct_packages = interesting_packages_qs.values("type", "namespace", "name").distinct(
42+
"type", "namespace", "name"
43+
)
4244

43-
ghost_package_count = 0
45+
distinct_packages_count = distinct_packages.count()
46+
package_iterator = distinct_packages.iterator(chunk_size=2000)
47+
progress = LoopProgress(total_iterations=distinct_packages_count, logger=logger)
4448

45-
for package in progress.iter(package_iterator):
46-
ghost_package_count += _remove_ghost_packages(package, interesting_packages_qs)
49+
ghost_package_count = 0
4750

48-
if self.log:
49-
self.log(f"Successfully removed {ghost_package_count:,d} ghost Packages")
51+
for package in progress.iter(package_iterator):
52+
ghost_package_count += remove_ghost_package(
53+
package=package,
54+
interesting_packages_qs=interesting_packages_qs,
55+
logger=logger,
56+
)
5057

58+
if logger:
59+
logger(f"Successfully removed {ghost_package_count:,d} ghost Packages")
5160

52-
def _remove_ghost_packages(package, interesting_packages_qs):
61+
62+
def remove_ghost_package(package, interesting_packages_qs, logger=None):
5363
if not package["type"] in RANGE_CLASS_BY_SCHEMES:
5464
return 0
5565

5666
versionless_purl = PackageURL(**package)
5767
purl_type = package["type"]
5868
version_class = RANGE_CLASS_BY_SCHEMES[purl_type].version_class
59-
known_versions = [version_class(v.value) for v in versions(str(versionless_purl))]
60-
package_versions = interesting_packages_qs.filter(**package)
6169

70+
try:
71+
known_versions = [version_class(v.value) for v in versions(str(versionless_purl))]
72+
except Exception as e:
73+
if logger:
74+
logger(f"An error occurred: {e}", level=logging.ERROR)
75+
return 0
76+
77+
package_versions = interesting_packages_qs.filter(**package)
6278
removed_packages = 0
6379
for pkg in package_versions:
6480
if version_class(pkg.version) not in known_versions:
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/nexB/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
10+
import io
11+
from pathlib import Path
12+
from unittest import mock
13+
14+
from django.test import TestCase
15+
from fetchcode.package_versions import PackageVersion
16+
17+
from vulnerabilities.models import Package
18+
from vulnerabilities.pipelines import remove_ghost_packages
19+
20+
21+
class RemoveGhostPackagePipelineTest(TestCase):
22+
data = Path(__file__).parent.parent / "test_data"
23+
24+
@mock.patch("vulnerabilities.pipelines.remove_ghost_packages.versions")
25+
def test_remove_ghost_package(self, mock_fetchcode_versions):
26+
Package.objects.create(type="pypi", name="foo", version="2.3.0")
27+
Package.objects.create(type="pypi", name="foo", version="3.0.0")
28+
29+
mock_fetchcode_versions.return_value = [
30+
PackageVersion(value="2.3.0"),
31+
]
32+
interesting_packages_qs = Package.objects.all()
33+
target_package = {
34+
"type": "pypi",
35+
"namespace": "",
36+
"name": "foo",
37+
}
38+
39+
self.assertEqual(2, Package.objects.count())
40+
41+
removed_package_count = remove_ghost_packages.remove_ghost_package(
42+
package=target_package,
43+
interesting_packages_qs=interesting_packages_qs,
44+
)
45+
self.assertEqual(1, removed_package_count)
46+
self.assertEqual(1, Package.objects.count())
47+
48+
@mock.patch("vulnerabilities.pipelines.remove_ghost_packages.versions")
49+
def test_remove_ghost_package(self, mock_fetchcode_versions):
50+
Package.objects.create(type="pypi", name="foo", version="2.3.0")
51+
Package.objects.create(type="pypi", name="foo", version="3.0.0")
52+
Package.objects.create(
53+
type="deb",
54+
namespace="debian",
55+
name="foo",
56+
version="3.0.0",
57+
qualifiers={"distro": "trixie"},
58+
)
59+
60+
mock_fetchcode_versions.return_value = [
61+
PackageVersion(value="2.3.0"),
62+
]
63+
64+
self.assertEqual(3, Package.objects.count())
65+
66+
buffer = io.StringIO()
67+
remove_ghost_packages.detect_and_remove_ghost_packages(logger=buffer.write)
68+
expected = "Successfully removed 1 ghost Packages"
69+
70+
self.assertIn(expected, buffer.getvalue())
71+
self.assertEqual(2, Package.objects.count())

‎vulnerablecode/settings.py‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,6 @@
320320
"127.0.0.1",
321321
]
322322

323-
# Logging
324323

325324
LOGGING = {
326325
"version": 1,
@@ -341,12 +340,7 @@
341340
},
342341
},
343342
"loggers": {
344-
"aboutcode.pipeline": {
345-
"handlers": ["console"],
346-
"level": VULNERABLECODE_LOG_LEVEL,
347-
"propagate": False,
348-
},
349-
"aboutcode.pipeline": {
343+
"vulnerabilities.pipelines": {
350344
"handlers": ["console"],
351345
"level": VULNERABLECODE_LOG_LEVEL,
352346
"propagate": False,

0 commit comments

Comments
 (0)