Skip to content

Commit 4d758e0

Browse files
committed
Add migration to drop malformed advisories
Add a migration test Signed-off-by: ziad hany <ziadhany2016@gmail.com>
1 parent 30fa8e4 commit 4d758e0

4 files changed

Lines changed: 156 additions & 9 deletions

File tree

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
from django.db import migrations
2+
from django.db.models import Q
3+
4+
5+
class Migration(migrations.Migration):
6+
dependencies = [
7+
("vulnerabilities", "0121_advisoryv2_is_latest_alter_advisoryv2_advisory_id_and_more"),
8+
]
9+
10+
def drop_malformed_advisory_v2(apps, _):
11+
AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2")
12+
AdvisoryAlias = apps.get_model("vulnerabilities", "AdvisoryAlias")
13+
14+
valid_alias_prefix = [
15+
"cve-", "osv-", "xsa-", "vsv", "zbx-", "zf2", "vu#", "gms-", "usn-",
16+
"sw-", "ss-", "ts-", "osvdb-", "ysa-", "se-core-", "pysec-", "alpine-",
17+
"dw2", "go-", "mal-", "zdi-can", "asa-", "ezsa-", "ghsl-", "ghsa-",
18+
"talos-", "srcclr-sid-", "bit-", "gnutls-", "rustsec-", "snyk-",
19+
"temp-", "TYPO3-", "wnpa-sec-", "sa-core-", "skcsirt-", "flow-", "gsd-"
20+
]
21+
query = Q()
22+
for alias_prefix in valid_alias_prefix:
23+
query |= Q(alias__istartswith=alias_prefix)
24+
25+
malformed_aliases = AdvisoryAlias.objects.exclude(query)
26+
AdvisoryV2.objects.filter(aliases__in=malformed_aliases).delete()
27+
malformed_aliases.delete()
28+
29+
operations = [
30+
migrations.RunPython(drop_malformed_advisory_v2, reverse_code=migrations.RunPython.noop),
31+
]

‎vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ def load_advisories(
166166
# fixed_vulns is a list of strings and each string is a space-separated
167167
# list of aliases and CVES
168168
for vuln_ids in fixed_vulns:
169-
vuln_id, aliases = parse_vuln_ids(vuln_ids)
169+
vuln_id, aliases = parse_vuln_ids(vuln_ids, logger=logger)
170170

171171
if not vuln_id:
172172
continue
@@ -258,7 +258,7 @@ def load_advisories(
258258
PARENTHESES_RE = re.compile(r"\(.*?\)")
259259

260260

261-
def parse_vuln_ids(vuln_ids_string):
261+
def parse_vuln_ids(vuln_ids_string, logger=print):
262262
"""
263263
Parses a raw vulnerability ids, removes parentheses and returns the advisory_id and a list of all valid aliases.
264264
"""
@@ -271,13 +271,30 @@ def parse_vuln_ids(vuln_ids_string):
271271
clean_alias = alias.replace("_", "-").replace(".patch", "")
272272
cleaned_vuln_ids.append(clean_alias)
273273

274-
aliases = [
275-
alias
276-
for alias in cleaned_vuln_ids
277-
if alias
278-
and alias not in ["N/A", "CVE"]
279-
and not (alias.startswith("CVE") and not is_cve(alias))
280-
]
274+
aliases = []
275+
valid_prefixes = (
276+
"XSA-",
277+
"GHSL-",
278+
"TALOS-",
279+
"RUSTSEC-",
280+
"GHSA-",
281+
"GNUTLS-",
282+
"VSV",
283+
"ZDI-CAN-",
284+
"DW",
285+
"YSA-",
286+
"ZBX-",
287+
"ALPINE-",
288+
"TS-",
289+
"wnpa-sec-",
290+
)
291+
for alias in cleaned_vuln_ids:
292+
if alias and (
293+
(alias.startswith("CVE-") and is_cve(alias)) or alias.startswith(valid_prefixes)
294+
):
295+
aliases.append(alias)
296+
else:
297+
logger(f"Malformed aliases found: {alias}")
281298

282299
if not aliases:
283300
return None, []

‎vulnerabilities/pipelines/v2_importers/istio_importer.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,9 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
109109
)
110110

111111
title = data.get("title") or ""
112+
if not title.startswith("ISTIO-"):
113+
self.log(f"Invalid advisory_id: {title}")
114+
112115
summary = data.get("description") or ""
113116
references = []
114117
if title:

‎vulnerabilities/tests/test_data_migrations.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
# See https://github.com/aboutcode-org/vulnerablecode for support or download.
77
# See https://aboutcode.org for more information about nexB OSS projects.
88
#
9+
910
from datetime import datetime
1011

1112
from django.apps import apps
@@ -1088,3 +1089,98 @@ def test_latest_is_actually_recent(self):
10881089

10891090
latest = AdvisoryV2.objects.get(avid="test_pipeline/test_adv", is_latest=True)
10901091
self.assertEqual("New advisory", latest.summary)
1092+
1093+
1094+
class TestMalformedAliasesAVIDMigration(TestMigrations):
1095+
app_name = "vulnerabilities"
1096+
migrate_from = "0121_advisoryv2_is_latest_alter_advisoryv2_advisory_id_and_more"
1097+
migrate_to = "0122_advisoryv2_remove_malformed_aliases_and_dvisory_id"
1098+
raw_alias_inputs = [
1099+
("CVE-2023-1111", True),
1100+
("GHSA-abcd-1234", True),
1101+
("", False),
1102+
("(not", False),
1103+
("applicable)", False),
1104+
("(BABEL)", False),
1105+
("(was", False),
1106+
("--with-systemd)", False),
1107+
("fixed", False),
1108+
("printing", False),
1109+
("(AFS/RX)", False),
1110+
("unreliably", False),
1111+
("(ICMP)", False),
1112+
("CVE", False),
1113+
("(Not", False),
1114+
("(RSVP)", False),
1115+
("libpcap)", False),
1116+
("(SMB", False),
1117+
("fix)", False),
1118+
("(DCCP)", False),
1119+
("(HNCP)", False),
1120+
("(+", False),
1121+
("(IKEv1)", False),
1122+
("(FrameRelay)", False),
1123+
("XPTI", False),
1124+
("CVE_2019-2426", False),
1125+
("(BGP)", False),
1126+
("disabled)", False),
1127+
("(RPL)", False),
1128+
("regression", False),
1129+
("actually", False),
1130+
("(VRRP)", False),
1131+
("-V)", False),
1132+
("2025-48379", False),
1133+
("fixed,", False),
1134+
("(802.11)", False),
1135+
("affected,", False),
1136+
("SMB", False),
1137+
("(OSPF6)", False),
1138+
("too", False),
1139+
("partially", False),
1140+
("in", False),
1141+
("(SMB)", False),
1142+
("but", False),
1143+
("-", False),
1144+
("(LDP)", False),
1145+
("reproduced,", False),
1146+
("N/A", False),
1147+
("(tcpdump", False),
1148+
("requires", False),
1149+
("(AoE)", False),
1150+
("(LMP)", False),
1151+
(" CVE-2025-55070", False),
1152+
("n/a", False),
1153+
("No CVE assigned", False),
1154+
("- CVE-2026-26365", False),
1155+
]
1156+
1157+
def setUpBeforeMigration(self, apps):
1158+
AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2")
1159+
AdvisoryAlias = apps.get_model("vulnerabilities", "AdvisoryAlias")
1160+
1161+
for i, (raw_input, _) in enumerate(self.raw_alias_inputs):
1162+
adv = AdvisoryV2.objects.create(
1163+
unique_content_id=f"content_{i}",
1164+
url="https://example.com",
1165+
summary=f"Advisory for {raw_input}",
1166+
advisory_id=raw_input,
1167+
avid=f"test_pipeline/{raw_input}",
1168+
datasource_id="test_pipeline",
1169+
)
1170+
alias = AdvisoryAlias.objects.create(alias=raw_input)
1171+
adv.aliases.add(alias)
1172+
1173+
def test_migration_processes_malformed_aliases(self):
1174+
AdvisoryV2 = self.apps.get_model("vulnerabilities", "AdvisoryV2")
1175+
AdvisoryAlias = self.apps.get_model("vulnerabilities", "AdvisoryAlias")
1176+
1177+
for i, (raw_input, expected_to_survive) in enumerate(self.raw_alias_inputs):
1178+
adv_exists = AdvisoryV2.objects.filter(unique_content_id=f"content_{i}").exists()
1179+
alias_exists = AdvisoryAlias.objects.filter(alias=raw_input).exists()
1180+
1181+
if expected_to_survive:
1182+
assert adv_exists == True
1183+
assert alias_exists == True
1184+
else:
1185+
assert adv_exists == False
1186+
assert alias_exists == False

0 commit comments

Comments
 (0)