Skip to content

Commit c040f59

Browse files
committed
Fix ehancement pipelines
Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>
1 parent a191687 commit c040f59

6 files changed

Lines changed: 190 additions & 46 deletions

File tree

‎vulnerabilities/pipelines/v2_improvers/enhance_with_exploitdb.py‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
from vulnerabilities.models import AdvisoryReference
2323
from vulnerabilities.models import AdvisoryV2
2424
from vulnerabilities.pipelines import VulnerableCodePipeline
25+
from vulnerabilities.utils import relate_aliases_with_advisories
2526

2627

2728
class ExploitDBImproverPipeline(VulnerableCodePipeline):
@@ -83,17 +84,7 @@ def add_vulnerability_exploit(row, logger):
8384
if not aliases:
8485
return 0
8586

86-
for raw_alias in aliases:
87-
try:
88-
if alias := AdvisoryAlias.objects.get(alias=raw_alias):
89-
for adv in alias.advisories.all():
90-
advisories.add(adv)
91-
else:
92-
advs = AdvisoryV2.objects.filter(advisory_id=raw_alias).latest_per_avid()
93-
for adv in advs:
94-
advisories.add(adv)
95-
except AdvisoryAlias.DoesNotExist:
96-
continue
87+
advisories = relate_aliases_with_advisories(aliases)
9788

9889
if not advisories:
9990
logger(f"No advisory found for aliases {aliases}")

‎vulnerabilities/pipelines/v2_improvers/enhance_with_github_poc.py‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from vulnerabilities.models import AdvisoryPOC
1818
from vulnerabilities.models import AdvisoryV2
1919
from vulnerabilities.pipelines import VulnerableCodePipeline
20+
from vulnerabilities.utils import relate_aliases_with_advisories
2021

2122

2223
class GithubPocsImproverPipeline(VulnerableCodePipeline):
@@ -61,18 +62,7 @@ def collect_and_store_exploits(self):
6162

6263
filename = file_path.stem.strip()
6364

64-
advisories = set()
65-
try:
66-
if alias := AdvisoryAlias.objects.get(alias=filename):
67-
for adv in alias.advisories.all():
68-
advisories.add(adv)
69-
else:
70-
advs = AdvisoryV2.objects.filter(advisory_id=filename).latest_per_avid()
71-
for adv in advs:
72-
advisories.add(adv)
73-
except AdvisoryAlias.DoesNotExist:
74-
self.log(f"Advisory {filename} not found.")
75-
continue
65+
advisories = relate_aliases_with_advisories([filename])
7666

7767
for advisory in advisories:
7868
for exploit_data in exploits_data:

‎vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from vulnerabilities.models import AdvisoryExploit
1818
from vulnerabilities.models import AdvisoryV2
1919
from vulnerabilities.pipelines import VulnerableCodePipeline
20+
from vulnerabilities.utils import relate_aliases_with_advisories
2021

2122

2223
class VulnerabilityKevPipeline(VulnerableCodePipeline):
@@ -72,18 +73,7 @@ def add_vulnerability_exploit(kev_vul, logger):
7273
if not cve_id:
7374
return 0
7475

75-
advisories = set()
76-
try:
77-
if alias := AdvisoryAlias.objects.get(alias=cve_id):
78-
for adv in alias.advisories.all():
79-
advisories.add(adv)
80-
else:
81-
advs = AdvisoryV2.objects.filter(advisory_id=cve_id).latest_per_avid()
82-
for adv in advs:
83-
advisories.add(adv)
84-
except AdvisoryAlias.DoesNotExist:
85-
logger(f"No advisory found for aliases {cve_id}")
86-
return 0
76+
advisories = relate_aliases_with_advisories([cve_id])
8777

8878
for advisory in advisories:
8979
AdvisoryExploit.objects.update_or_create(

‎vulnerabilities/pipelines/v2_improvers/enhance_with_metasploit.py‎

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from vulnerabilities.models import AdvisoryExploit
2020
from vulnerabilities.models import AdvisoryV2
2121
from vulnerabilities.pipelines import VulnerableCodePipeline
22+
from vulnerabilities.utils import relate_aliases_with_advisories
2223

2324

2425
class MetasploitImproverPipeline(VulnerableCodePipeline):
@@ -77,17 +78,9 @@ def add_advisory_exploit(record, logger):
7778
if not interesting_references:
7879
return 0
7980

80-
for ref in interesting_references:
81-
try:
82-
if alias := AdvisoryAlias.objects.get(alias=ref):
83-
for adv in alias.advisories.all():
84-
advisories.add(adv)
85-
else:
86-
advs = AdvisoryV2.objects.filter(advisory_id=ref).latest_per_avid()
87-
for adv in advs:
88-
advisories.add(adv)
89-
except AdvisoryAlias.DoesNotExist:
90-
continue
81+
advisories = []
82+
83+
advisories = relate_aliases_with_advisories(interesting_references)
9184

9285
if not advisories:
9386
logger(f"No advisories found for aliases {interesting_references}")

‎vulnerabilities/tests/test_utils.py‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from datetime import datetime
1111
from datetime import timedelta
1212

13+
import pytest
1314
from django.test import TestCase
1415
from fetchcode.package_versions import PackageVersion
1516
from packageurl import PackageURL
@@ -24,6 +25,7 @@
2425
from vulnerabilities.importer import PackageCommitPatchData
2526
from vulnerabilities.importer import PatchData
2627
from vulnerabilities.importer import VulnerabilitySeverity
28+
from vulnerabilities.models import AdvisoryAlias
2729
from vulnerabilities.models import AdvisoryV2
2830
from vulnerabilities.pipelines import insert_advisory_v2
2931
from vulnerabilities.references import XsaReferenceV2
@@ -33,6 +35,7 @@
3335
from vulnerabilities.utils import get_item
3436
from vulnerabilities.utils import get_severity_range
3537
from vulnerabilities.utils import nearest_patched_package
38+
from vulnerabilities.utils import relate_aliases_with_advisories
3639
from vulnerabilities.utils import resolve_version_range
3740
from vulnerabilities.utils import split_markdown_front_matter
3841

@@ -262,3 +265,152 @@ def test_content_id_from_adv_data_roundtrip_are_same(self):
262265
id_from_roundtrip = utils.compute_content_id_v2(adv_roundtrip)
263266

264267
self.assertEqual(id_from_data, id_from_roundtrip)
268+
269+
270+
@pytest.mark.django_db
271+
def test_returns_advisories_from_alias():
272+
advisory = AdvisoryV2.objects.create(
273+
advisory_id="GHSA-123",
274+
datasource_id="ds",
275+
pipeline_id="ds_importer_v2",
276+
unique_content_id="i3giu",
277+
url="https://test.com",
278+
avid="GHSA-123",
279+
is_latest=True,
280+
_all_impacts_unfurled_at="2025-01-01",
281+
)
282+
283+
alias = AdvisoryAlias.objects.create(alias="CVE-2025-123")
284+
alias.advisories.add(advisory)
285+
286+
result = relate_aliases_with_advisories(["CVE-2025-123"])
287+
288+
assert result == {advisory}
289+
290+
291+
@pytest.mark.django_db
292+
def test_returns_latest_advisory_from_advisory_id():
293+
advisory = AdvisoryV2.objects.create(
294+
advisory_id="GHSA-456",
295+
avid="GHSA-456",
296+
is_latest=True,
297+
_all_impacts_unfurled_at="2025-01-01",
298+
datasource_id="ds",
299+
pipeline_id="ds_importer_v2",
300+
unique_content_id="i3giu2w",
301+
url="https://test.com",
302+
)
303+
304+
result = relate_aliases_with_advisories(["GHSA-456"])
305+
306+
assert advisory in result
307+
assert len(result) == 1
308+
309+
310+
@pytest.mark.django_db
311+
def test_handles_mixed_aliases_and_advisory_ids():
312+
alias_adv = AdvisoryV2.objects.create(
313+
advisory_id="GHSA-1",
314+
avid="GHSA-1",
315+
is_latest=True,
316+
_all_impacts_unfurled_at="2025-01-01",
317+
datasource_id="ds",
318+
pipeline_id="ds_importer_v2",
319+
unique_content_id="i3gswiu",
320+
url="https://test.com",
321+
)
322+
323+
advisory_id_adv = AdvisoryV2.objects.create(
324+
advisory_id="GHSA-2",
325+
avid="GHSA-2",
326+
is_latest=True,
327+
_all_impacts_unfurled_at="2025-01-01",
328+
datasource_id="ds",
329+
pipeline_id="ds_importer_v2",
330+
unique_content_id="i3giu2ww",
331+
url="https://test.com",
332+
)
333+
334+
alias = AdvisoryAlias.objects.create(alias="CVE-2025-1")
335+
alias.advisories.add(alias_adv)
336+
337+
result = relate_aliases_with_advisories(["CVE-2025-1", "GHSA-2"])
338+
339+
assert result == {alias_adv, advisory_id_adv}
340+
341+
342+
@pytest.mark.django_db
343+
def test_excludes_non_latest_alias_advisories():
344+
latest = AdvisoryV2.objects.create(
345+
advisory_id="GHSA-1",
346+
avid="GHSA-1",
347+
is_latest=True,
348+
_all_impacts_unfurled_at="2025-01-01",
349+
datasource_id="ds",
350+
pipeline_id="ds_importer_v2",
351+
unique_content_id="i3giu",
352+
url="https://test.com",
353+
)
354+
355+
old = AdvisoryV2.objects.create(
356+
advisory_id="GHSA-1-old",
357+
avid="GHSA-1",
358+
is_latest=False,
359+
_all_impacts_unfurled_at="2025-01-01",
360+
datasource_id="ds",
361+
pipeline_id="ds_importer_v2",
362+
unique_content_id="i3giu2w",
363+
url="https://test.com",
364+
)
365+
366+
alias = AdvisoryAlias.objects.create(alias="CVE-2025-1")
367+
alias.advisories.add(latest, old)
368+
369+
result = relate_aliases_with_advisories(["CVE-2025-1"])
370+
371+
assert result == {latest}
372+
373+
374+
@pytest.mark.django_db
375+
def test_excludes_unfurled_advisories():
376+
advisory = AdvisoryV2.objects.create(
377+
advisory_id="GHSA-1",
378+
avid="GHSA-1",
379+
is_latest=True,
380+
_all_impacts_unfurled_at=None,
381+
datasource_id="ds",
382+
pipeline_id="ds_importer_v2",
383+
unique_content_id="i3giu",
384+
url="https://test.com",
385+
)
386+
387+
alias = AdvisoryAlias.objects.create(alias="CVE-2025-1")
388+
alias.advisories.add(advisory)
389+
390+
result = relate_aliases_with_advisories(["CVE-2025-1"])
391+
392+
assert result == set()
393+
394+
395+
@pytest.mark.django_db
396+
def test_deduplicates_results():
397+
advisory = AdvisoryV2.objects.create(
398+
advisory_id="GHSA-1",
399+
avid="GHSA-1",
400+
is_latest=True,
401+
_all_impacts_unfurled_at="2025-01-01",
402+
datasource_id="ds",
403+
pipeline_id="ds_importer_v2",
404+
unique_content_id="i3giu",
405+
url="https://test.com",
406+
)
407+
408+
alias1 = AdvisoryAlias.objects.create(alias="CVE-1")
409+
alias2 = AdvisoryAlias.objects.create(alias="CVE-2")
410+
411+
alias1.advisories.add(advisory)
412+
alias2.advisories.add(advisory)
413+
414+
result = relate_aliases_with_advisories(["CVE-1", "CVE-2"])
415+
416+
assert result == {advisory}

‎vulnerabilities/utils.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1055,3 +1055,31 @@ def get_advisories_from_groups(groups, include_ssvc_trees=False):
10551055
"gem",
10561056
"conan",
10571057
]
1058+
1059+
1060+
def relate_aliases_with_advisories(aliases):
1061+
from vulnerabilities.models import AdvisoryAlias
1062+
from vulnerabilities.models import AdvisoryV2
1063+
1064+
aliases = set(aliases)
1065+
1066+
found_aliases = AdvisoryAlias.objects.filter(alias__in=aliases).prefetch_related("advisories")
1067+
1068+
found_alias_values = set(found_aliases.values_list("alias", flat=True))
1069+
1070+
alias_advisories = AdvisoryV2.objects.filter(
1071+
aliases__alias__in=found_alias_values,
1072+
is_latest=True,
1073+
_all_impacts_unfurled_at__isnull=False,
1074+
).distinct()
1075+
1076+
missing_aliases = aliases - found_alias_values
1077+
1078+
advisory_id_advisories = AdvisoryV2.objects.filter(
1079+
advisory_id__in=missing_aliases,
1080+
_all_impacts_unfurled_at__isnull=False,
1081+
).latest_per_avid()
1082+
1083+
advisories = set(alias_advisories)
1084+
advisories.update(advisory_id_advisories)
1085+
return advisories

0 commit comments

Comments
 (0)