|
13 | 13 |
|
14 | 14 | from aboutcode.pipeline import LoopProgress |
15 | 15 | from django.db import transaction |
16 | | -from django.db.models import Exists |
17 | 16 | from django.db.models import F |
18 | | -from django.db.models import OuterRef |
19 | 17 | from django.db.models import Q |
20 | 18 | from django.utils import timezone |
21 | 19 | from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS as FETCHCODE_SUPPORTED_ECOSYSTEMS |
@@ -55,11 +53,7 @@ class UnfurlVersionRangePipeline(VulnerableCodePipeline): |
55 | 53 |
|
56 | 54 | @classmethod |
57 | 55 | def steps(cls): |
58 | | - return ( |
59 | | - cls.unfurl_version_range, |
60 | | - cls.mark_all_impacts_unfurled_sucessfully, |
61 | | - cls.mark_all_impacts_unfurl_attempted, |
62 | | - ) |
| 56 | + return (cls.unfurl_version_range,) |
63 | 57 |
|
64 | 58 | def unfurl_version_range(self): |
65 | 59 | cur_time = timezone.now() |
@@ -135,43 +129,6 @@ def unfurl_version_range(self): |
135 | 129 | self.log(f"Successfully processed {processed_impacted_packages_count:,d} ImpactedPackage.") |
136 | 130 | self.log(f"{processed_affected_packages_count:,d} new Impact-Package relation created.") |
137 | 131 |
|
138 | | - def mark_all_impacts_unfurled_sucessfully(self): |
139 | | - impacted_packages = ImpactedPackage.objects.all() |
140 | | - while True: |
141 | | - advisory_ids = list( |
142 | | - latest_advisories_with_all_impacts_unfurled_successfully( |
143 | | - impacted_packages=impacted_packages |
144 | | - )[:100] |
145 | | - ) |
146 | | - |
147 | | - if not advisory_ids: |
148 | | - break |
149 | | - |
150 | | - complete_advisories_import(advisory_ids=advisory_ids, success=True) |
151 | | - |
152 | | - def mark_all_impacts_unfurl_attempted(self): |
153 | | - impacted_packages = ImpactedPackage.objects.all() |
154 | | - advisories_qs = latest_advisories_with_all_impacts_unfurled_attempted( |
155 | | - impacted_packages=impacted_packages |
156 | | - ) |
157 | | - |
158 | | - batch_size = 100 |
159 | | - batch = [] |
160 | | - |
161 | | - for advisory_id in advisories_qs.iterator(chunk_size=100): |
162 | | - batch.append(advisory_id) |
163 | | - |
164 | | - if len(batch) >= batch_size: |
165 | | - complete_advisories_import( |
166 | | - advisory_ids=list(batch), |
167 | | - ) |
168 | | - batch.clear() |
169 | | - |
170 | | - if batch: |
171 | | - complete_advisories_import( |
172 | | - advisory_ids=list(batch), |
173 | | - ) |
174 | | - |
175 | 132 |
|
176 | 133 | def get_affected_purls(versions, impact, logger): |
177 | 134 | affecting_version_range = VersionRange.from_string(impact.affecting_vers) |
@@ -257,84 +214,3 @@ def impacted_package_qs(cutoff_day=2): |
257 | 214 | .order_by("advisory__id", F("last_range_unfurl_at").asc(nulls_first=True)) |
258 | 215 | .only("pk", "affecting_vers", "advisory", "base_purl") |
259 | 216 | ) |
260 | | - |
261 | | - |
262 | | -@transaction.atomic |
263 | | -def complete_advisories_import(advisory_ids, success=False): |
264 | | - if not advisory_ids: |
265 | | - return |
266 | | - |
267 | | - AdvisoryV2.objects.filter(id__in=advisory_ids).update(_all_impacts_unfurled=True) |
268 | | - |
269 | | - if success: |
270 | | - AdvisoryV2.objects.filter(id__in=advisory_ids).update( |
271 | | - _all_impacts_unfurled_successfully=True |
272 | | - ) |
273 | | - |
274 | | - affecting_package_ids = set( |
275 | | - ImpactedPackageAffecting.objects.filter( |
276 | | - impacted_package__advisory_id__in=advisory_ids |
277 | | - ).values_list( |
278 | | - "package_id", |
279 | | - flat=True, |
280 | | - ) |
281 | | - ) |
282 | | - |
283 | | - fixed_by_package_ids = set( |
284 | | - ImpactedPackageFixedBy.objects.filter( |
285 | | - impacted_package__advisory_id__in=advisory_ids |
286 | | - ).values_list( |
287 | | - "package_id", |
288 | | - flat=True, |
289 | | - ) |
290 | | - ) |
291 | | - |
292 | | - compute_package_risk_score_bulk(PackageV2.objects.filter(id__in=affecting_package_ids)) |
293 | | - |
294 | | - group_package_ids = affecting_package_ids | fixed_by_package_ids |
295 | | - |
296 | | - for package in PackageV2.objects.filter( |
297 | | - id__in=group_package_ids, type__in=TYPES_WITH_MULTIPLE_IMPORTERS |
298 | | - ).iterator(chunk_size=2000): |
299 | | - group_advisory_for_package(package) |
300 | | - |
301 | | - |
302 | | -def latest_advisories_with_all_impacts_unfurled_successfully( |
303 | | - impacted_packages=None, |
304 | | -): |
305 | | - unsuccessful_impacts = impacted_packages.filter( |
306 | | - advisory_id=OuterRef("pk"), |
307 | | - advisory__is_latest=True, |
308 | | - ).filter(Q(last_range_unfurl_at__isnull=True) | Q(last_successful_range_unfurl_at__isnull=True)) |
309 | | - |
310 | | - return ( |
311 | | - AdvisoryV2.objects.filter( |
312 | | - _all_impacts_unfurled_successfully=False, |
313 | | - is_latest=True, |
314 | | - ) |
315 | | - .annotate(has_unsuccessful_impacts=Exists(unsuccessful_impacts)) |
316 | | - .filter(has_unsuccessful_impacts=False) |
317 | | - .order_by("id") |
318 | | - .values_list("id", flat=True) |
319 | | - ) |
320 | | - |
321 | | - |
322 | | -def latest_advisories_with_all_impacts_unfurled_attempted( |
323 | | - impacted_packages=None, |
324 | | -): |
325 | | - impacts_not_attempted = impacted_packages.filter( |
326 | | - advisory_id=OuterRef("pk"), |
327 | | - advisory__is_latest=True, |
328 | | - last_range_unfurl_at__isnull=True, |
329 | | - ) |
330 | | - |
331 | | - return ( |
332 | | - AdvisoryV2.objects.filter( |
333 | | - _all_impacts_unfurled_successfully=False, |
334 | | - is_latest=True, |
335 | | - ) |
336 | | - .annotate(has_unattempted_impacts=Exists(impacts_not_attempted)) |
337 | | - .filter(has_unattempted_impacts=False) |
338 | | - .order_by("id") |
339 | | - .values_list("id", flat=True) |
340 | | - ) |
0 commit comments