Skip to content

Commit f761a78

Browse files
committed
Add separate pipeline for marking unfurls
Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>
1 parent 7eae034 commit f761a78

5 files changed

Lines changed: 178 additions & 142 deletions

File tree

‎vulnerabilities/api_v3.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,7 @@ def get_affected_by_vulnerabilities(self, package):
319319
)
320320

321321
return result
322+
return []
322323

323324
def get_fixing_vulnerabilities(self, package):
324325
advisories = self.context["fixing_advisory_map"].get(package.id, [])

‎vulnerabilities/improvers/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
enhance_with_metasploit as enhance_with_metasploit_v2,
2828
)
2929
from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2
30+
from vulnerabilities.pipelines.v2_improvers import mark_unfurl_version_range
3031
from vulnerabilities.pipelines.v2_improvers import reference_collect_commits
3132
from vulnerabilities.pipelines.v2_improvers import relate_severities
3233
from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2
@@ -46,6 +47,7 @@
4647
compute_advisory_todo_v2.ComputeToDo,
4748
reference_collect_commits.CollectReferencesFixCommitsPipeline,
4849
enhance_with_github_poc.GithubPocsImproverPipeline,
50+
mark_unfurl_version_range.MarkUnfurlVersionRangePipeline,
4951
valid_versions.GitHubBasicImprover,
5052
valid_versions.GitLabBasicImprover,
5153
valid_versions.NginxBasicImprover,
Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/aboutcode-org/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
10+
from django.db import transaction
11+
from django.db.models import Exists
12+
from django.db.models import OuterRef
13+
from django.db.models import Q
14+
15+
from vulnerabilities.models import AdvisoryV2
16+
from vulnerabilities.models import ImpactedPackage
17+
from vulnerabilities.models import ImpactedPackageAffecting
18+
from vulnerabilities.models import ImpactedPackageFixedBy
19+
from vulnerabilities.models import PackageV2
20+
from vulnerabilities.models import PipelineSchedule
21+
from vulnerabilities.pipelines import VulnerableCodePipeline
22+
from vulnerabilities.pipes.group_advisories import group_advisory_for_package
23+
from vulnerabilities.pipes.risk_score import compute_package_risk_score_bulk
24+
from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS
25+
26+
27+
class MarkUnfurlVersionRangePipeline(VulnerableCodePipeline):
28+
"""
29+
Mark advisories as unfurled whose all impacts have been unfurled
30+
"""
31+
32+
pipeline_id = "mark_unfurl_version_range_v2"
33+
34+
run_interval = 1
35+
run_priority = PipelineSchedule.ExecutionPriority.HIGH
36+
37+
@classmethod
38+
def steps(cls):
39+
return (
40+
cls.mark_all_impacts_unfurled_sucessfully,
41+
cls.mark_all_impacts_unfurl_attempted,
42+
)
43+
44+
def mark_all_impacts_unfurled_sucessfully(self):
45+
impacted_packages = ImpactedPackage.objects.all()
46+
while True:
47+
advisory_ids = list(
48+
latest_advisories_with_all_impacts_unfurled_successfully(
49+
impacted_packages=impacted_packages
50+
)[:100]
51+
)
52+
53+
if not advisory_ids:
54+
break
55+
56+
complete_advisories_import(advisory_ids=advisory_ids, success=True)
57+
58+
def mark_all_impacts_unfurl_attempted(self):
59+
impacted_packages = ImpactedPackage.objects.all()
60+
advisories_qs = latest_advisories_with_all_impacts_unfurled_attempted(
61+
impacted_packages=impacted_packages
62+
)
63+
64+
batch_size = 100
65+
batch = []
66+
67+
for advisory_id in advisories_qs.iterator(chunk_size=100):
68+
batch.append(advisory_id)
69+
70+
if len(batch) >= batch_size:
71+
complete_advisories_import(
72+
advisory_ids=list(batch),
73+
)
74+
batch.clear()
75+
76+
if batch:
77+
complete_advisories_import(
78+
advisory_ids=list(batch),
79+
)
80+
81+
82+
def latest_advisories_with_all_impacts_unfurled_successfully(
83+
impacted_packages=None,
84+
):
85+
unsuccessful_impacts = impacted_packages.filter(
86+
advisory_id=OuterRef("pk"),
87+
advisory__is_latest=True,
88+
).filter(Q(last_range_unfurl_at__isnull=True) | Q(last_successful_range_unfurl_at__isnull=True))
89+
90+
return (
91+
AdvisoryV2.objects.filter(
92+
_all_impacts_unfurled_successfully=False,
93+
is_latest=True,
94+
)
95+
.annotate(has_unsuccessful_impacts=Exists(unsuccessful_impacts))
96+
.filter(has_unsuccessful_impacts=False)
97+
.order_by("id")
98+
.values_list("id", flat=True)
99+
)
100+
101+
102+
def latest_advisories_with_all_impacts_unfurled_attempted(
103+
impacted_packages=None,
104+
):
105+
impacts_not_attempted = impacted_packages.filter(
106+
advisory_id=OuterRef("pk"),
107+
advisory__is_latest=True,
108+
last_range_unfurl_at__isnull=True,
109+
)
110+
111+
return (
112+
AdvisoryV2.objects.filter(
113+
_all_impacts_unfurled_successfully=False,
114+
is_latest=True,
115+
)
116+
.annotate(has_unattempted_impacts=Exists(impacts_not_attempted))
117+
.filter(has_unattempted_impacts=False)
118+
.order_by("_all_impacts_unfurled", "id")
119+
.values_list("id", flat=True)
120+
)
121+
122+
123+
@transaction.atomic
124+
def complete_advisories_import(advisory_ids, success=False):
125+
if not advisory_ids:
126+
return
127+
128+
AdvisoryV2.objects.filter(id__in=advisory_ids).update(_all_impacts_unfurled=True)
129+
130+
if success:
131+
AdvisoryV2.objects.filter(id__in=advisory_ids).update(
132+
_all_impacts_unfurled_successfully=True
133+
)
134+
135+
affecting_package_ids = set(
136+
ImpactedPackageAffecting.objects.filter(
137+
impacted_package__advisory_id__in=advisory_ids
138+
).values_list(
139+
"package_id",
140+
flat=True,
141+
)
142+
)
143+
144+
fixed_by_package_ids = set(
145+
ImpactedPackageFixedBy.objects.filter(
146+
impacted_package__advisory_id__in=advisory_ids
147+
).values_list(
148+
"package_id",
149+
flat=True,
150+
)
151+
)
152+
153+
compute_package_risk_score_bulk(PackageV2.objects.filter(id__in=affecting_package_ids))
154+
155+
group_package_ids = affecting_package_ids | fixed_by_package_ids
156+
157+
for package in PackageV2.objects.filter(
158+
id__in=group_package_ids, type__in=TYPES_WITH_MULTIPLE_IMPORTERS
159+
).iterator(chunk_size=2000):
160+
group_advisory_for_package(package)

‎vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py‎

Lines changed: 1 addition & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,7 @@
1313

1414
from aboutcode.pipeline import LoopProgress
1515
from django.db import transaction
16-
from django.db.models import Exists
1716
from django.db.models import F
18-
from django.db.models import OuterRef
1917
from django.db.models import Q
2018
from django.utils import timezone
2119
from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS as FETCHCODE_SUPPORTED_ECOSYSTEMS
@@ -55,11 +53,7 @@ class UnfurlVersionRangePipeline(VulnerableCodePipeline):
5553

5654
@classmethod
5755
def steps(cls):
58-
return (
59-
cls.unfurl_version_range,
60-
cls.mark_all_impacts_unfurled_sucessfully,
61-
cls.mark_all_impacts_unfurl_attempted,
62-
)
56+
return (cls.unfurl_version_range,)
6357

6458
def unfurl_version_range(self):
6559
cur_time = timezone.now()
@@ -135,43 +129,6 @@ def unfurl_version_range(self):
135129
self.log(f"Successfully processed {processed_impacted_packages_count:,d} ImpactedPackage.")
136130
self.log(f"{processed_affected_packages_count:,d} new Impact-Package relation created.")
137131

138-
def mark_all_impacts_unfurled_sucessfully(self):
139-
impacted_packages = ImpactedPackage.objects.all()
140-
while True:
141-
advisory_ids = list(
142-
latest_advisories_with_all_impacts_unfurled_successfully(
143-
impacted_packages=impacted_packages
144-
)[:100]
145-
)
146-
147-
if not advisory_ids:
148-
break
149-
150-
complete_advisories_import(advisory_ids=advisory_ids, success=True)
151-
152-
def mark_all_impacts_unfurl_attempted(self):
153-
impacted_packages = ImpactedPackage.objects.all()
154-
advisories_qs = latest_advisories_with_all_impacts_unfurled_attempted(
155-
impacted_packages=impacted_packages
156-
)
157-
158-
batch_size = 100
159-
batch = []
160-
161-
for advisory_id in advisories_qs.iterator(chunk_size=100):
162-
batch.append(advisory_id)
163-
164-
if len(batch) >= batch_size:
165-
complete_advisories_import(
166-
advisory_ids=list(batch),
167-
)
168-
batch.clear()
169-
170-
if batch:
171-
complete_advisories_import(
172-
advisory_ids=list(batch),
173-
)
174-
175132

176133
def get_affected_purls(versions, impact, logger):
177134
affecting_version_range = VersionRange.from_string(impact.affecting_vers)
@@ -257,84 +214,3 @@ def impacted_package_qs(cutoff_day=2):
257214
.order_by("advisory__id", F("last_range_unfurl_at").asc(nulls_first=True))
258215
.only("pk", "affecting_vers", "advisory", "base_purl")
259216
)
260-
261-
262-
@transaction.atomic
263-
def complete_advisories_import(advisory_ids, success=False):
264-
if not advisory_ids:
265-
return
266-
267-
AdvisoryV2.objects.filter(id__in=advisory_ids).update(_all_impacts_unfurled=True)
268-
269-
if success:
270-
AdvisoryV2.objects.filter(id__in=advisory_ids).update(
271-
_all_impacts_unfurled_successfully=True
272-
)
273-
274-
affecting_package_ids = set(
275-
ImpactedPackageAffecting.objects.filter(
276-
impacted_package__advisory_id__in=advisory_ids
277-
).values_list(
278-
"package_id",
279-
flat=True,
280-
)
281-
)
282-
283-
fixed_by_package_ids = set(
284-
ImpactedPackageFixedBy.objects.filter(
285-
impacted_package__advisory_id__in=advisory_ids
286-
).values_list(
287-
"package_id",
288-
flat=True,
289-
)
290-
)
291-
292-
compute_package_risk_score_bulk(PackageV2.objects.filter(id__in=affecting_package_ids))
293-
294-
group_package_ids = affecting_package_ids | fixed_by_package_ids
295-
296-
for package in PackageV2.objects.filter(
297-
id__in=group_package_ids, type__in=TYPES_WITH_MULTIPLE_IMPORTERS
298-
).iterator(chunk_size=2000):
299-
group_advisory_for_package(package)
300-
301-
302-
def latest_advisories_with_all_impacts_unfurled_successfully(
303-
impacted_packages=None,
304-
):
305-
unsuccessful_impacts = impacted_packages.filter(
306-
advisory_id=OuterRef("pk"),
307-
advisory__is_latest=True,
308-
).filter(Q(last_range_unfurl_at__isnull=True) | Q(last_successful_range_unfurl_at__isnull=True))
309-
310-
return (
311-
AdvisoryV2.objects.filter(
312-
_all_impacts_unfurled_successfully=False,
313-
is_latest=True,
314-
)
315-
.annotate(has_unsuccessful_impacts=Exists(unsuccessful_impacts))
316-
.filter(has_unsuccessful_impacts=False)
317-
.order_by("id")
318-
.values_list("id", flat=True)
319-
)
320-
321-
322-
def latest_advisories_with_all_impacts_unfurled_attempted(
323-
impacted_packages=None,
324-
):
325-
impacts_not_attempted = impacted_packages.filter(
326-
advisory_id=OuterRef("pk"),
327-
advisory__is_latest=True,
328-
last_range_unfurl_at__isnull=True,
329-
)
330-
331-
return (
332-
AdvisoryV2.objects.filter(
333-
_all_impacts_unfurled_successfully=False,
334-
is_latest=True,
335-
)
336-
.annotate(has_unattempted_impacts=Exists(impacts_not_attempted))
337-
.filter(has_unattempted_impacts=False)
338-
.order_by("id")
339-
.values_list("id", flat=True)
340-
)

0 commit comments

Comments
 (0)