Skip to content

Commit db82a62

Browse files
committed
Refactor vulnerability severity and data migrations
- Add 'severities' field to AffectedByPackageRelatedVulnerability. - Add 'severities' field to Vulnerability. - Add 'url' field to VulnerabilitySeverity. - Data migration to remove corrupted SUSE scores. - Data migration to populate new VulnerabilitySeverity url field using reference. - Data migration to populate Vulnerability 'severities' M2M relationship. - Delete VulnerabilitySeverity reference field. Signed-off-by: Keshav Priyadarshi <git@keshav.space>
1 parent 8a68c97 commit db82a62

2 files changed

Lines changed: 242 additions & 39 deletions

File tree

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
# Generated by Django 4.2.16 on 2024-11-01 12:40
2+
3+
from django.db import migrations, models
4+
from aboutcode.pipeline import LoopProgress
5+
6+
7+
class Migration(migrations.Migration):
8+
"""
9+
Add url field to the VulnerabilitySeverity model and remove reference foreignkey
10+
relationship. Also, add new M2M 'severities' field in Vulnerability and
11+
AffectedByPackageRelatedVulnerability relationship.
12+
13+
To achieve this following model changes and data migrations are applied in chronological order:
14+
- Make VulnerabilitySeverity reference field nullable (to make the migration reversible).
15+
- Add 'severities' field to AffectedByPackageRelatedVulnerability.
16+
- Add 'severities' field to Vulnerability.
17+
- Add 'url' field to VulnerabilitySeverity.
18+
- Data migration to remove corrupted SUSE scores.
19+
- Data migration to populate new VulnerabilitySeverity url field using reference.
20+
- Data migration to populate Vulnerability 'severities' M2M relationship.
21+
- Delete VulnerabilitySeverity reference field.
22+
"""
23+
def remove_inaccurate_suse_score(apps, schema_editor):
24+
"""
25+
Remove inaccurate suse severity scores.
26+
See https://github.com/aboutcode-org/vulnerablecode/issues/1597
27+
"""
28+
print(f"\nRemoving inaccurate suse severity scores.")
29+
VulnerabilitySeverity = apps.get_model("vulnerabilities", "VulnerabilitySeverity")
30+
VulnerabilitySeverity.objects.filter(
31+
reference__url="https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml"
32+
).delete()
33+
34+
def reverse_remove_inaccurate_suse_score(apps, schema_editor):
35+
"""Reverse data migration not needed for inaccurate severity scores."""
36+
pass
37+
38+
def populate_severity_model_with_url(apps, schema_editor):
39+
"""Populate the new VulnerabilitySeverity `url` field using reference url"""
40+
41+
VulnerabilitySeverity = apps.get_model("vulnerabilities", "VulnerabilitySeverity")
42+
chunk_size = 10000
43+
batch = []
44+
vulnerability_severity_query = VulnerabilitySeverity.objects.select_related("reference")
45+
46+
print(
47+
f"\nPopulating {vulnerability_severity_query.count():,d} VulnerabilitySeverity"
48+
"`url` field using reference url."
49+
)
50+
progress = LoopProgress(
51+
total_iterations=vulnerability_severity_query.count(),
52+
progress_step=10,
53+
logger=print,
54+
)
55+
for severity in progress.iter(vulnerability_severity_query.iterator(chunk_size=chunk_size)):
56+
severity.url = severity.reference.url
57+
batch.append(severity)
58+
59+
if len(batch) >= chunk_size:
60+
VulnerabilitySeverity.objects.bulk_update(
61+
objs=batch,
62+
fields=["url"],
63+
)
64+
batch.clear()
65+
66+
VulnerabilitySeverity.objects.bulk_update(
67+
objs=batch,
68+
fields=["url"],
69+
)
70+
71+
def reverse_populate_severity_model_reference_using_url(apps, schema_editor):
72+
"""Reverse: Populate the reference using `url` to ensure proper reverse migration."""
73+
VulnerabilitySeverity = apps.get_model("vulnerabilities", "VulnerabilitySeverity")
74+
VulnerabilityReference = apps.get_model("vulnerabilities", "VulnerabilityReference")
75+
chunk_size = 10000
76+
batch = []
77+
vulnerability_severity_query = VulnerabilitySeverity.objects.select_related("reference")
78+
79+
print(
80+
f"\nReverse: Populating {vulnerability_severity_query.count():,d} VulnerabilitySeverity"
81+
"`reference` relation using url."
82+
)
83+
progress = LoopProgress(
84+
total_iterations=vulnerability_severity_query.count(),
85+
progress_step=10,
86+
logger=print,
87+
)
88+
for severity in progress.iter(vulnerability_severity_query.iterator(chunk_size=chunk_size)):
89+
severity.reference = VulnerabilityReference.objects.get(url=severity.url)
90+
batch.append(severity)
91+
92+
if len(batch) >= chunk_size:
93+
VulnerabilitySeverity.objects.bulk_update(
94+
objs=batch,
95+
fields=["reference"],
96+
)
97+
batch.clear()
98+
99+
VulnerabilitySeverity.objects.bulk_update(
100+
objs=batch,
101+
fields=["reference"],
102+
)
103+
104+
def populate_vulnerability_model_with_severities(apps, schema_editor):
105+
"""Populate the new Vulnerability `severities` relation using referenced severity."""
106+
Vulnerability = apps.get_model("vulnerabilities", "Vulnerability")
107+
chunk_size = 10000
108+
109+
vulnerability_query = Vulnerability.objects.prefetch_related(
110+
"references__vulnerabilityseverity_set"
111+
)
112+
print(
113+
f"\nPopulating {vulnerability_query.count():,d} Vulnerability `severities`"
114+
"relation using referenced severity."
115+
)
116+
progress = LoopProgress(
117+
total_iterations=vulnerability_query.count(),
118+
progress_step=10,
119+
logger=print,
120+
)
121+
for vulnerability in progress.iter(vulnerability_query.iterator(chunk_size=chunk_size)):
122+
references = vulnerability.references.all()
123+
severity_ids = [
124+
severity.id
125+
for reference in references
126+
for severity in reference.vulnerabilityseverity_set.all()
127+
if reference.vulnerabilityseverity_set.exists()
128+
]
129+
vulnerability.severities.set(severity_ids)
130+
131+
def reverse_populate_vulnerability_model_with_severities(apps, schema_editor):
132+
"""Reverse data migration not needed for new `severities` relationship."""
133+
pass
134+
135+
dependencies = [
136+
("vulnerabilities", "0074_update_pysec_advisory_created_by"),
137+
]
138+
139+
operations = [
140+
# reference field should nullable to properly support reverse migration.
141+
migrations.AlterField(
142+
model_name="vulnerabilityseverity",
143+
name="reference",
144+
field=models.ForeignKey(
145+
to="vulnerabilities.vulnerabilityreference",
146+
on_delete=models.CASCADE,
147+
null=True,
148+
),
149+
),
150+
migrations.AlterModelOptions(
151+
name="vulnerabilityseverity",
152+
options={"ordering": ["url", "scoring_system", "value"]},
153+
),
154+
migrations.AlterUniqueTogether(
155+
name="vulnerabilityseverity",
156+
unique_together=set(),
157+
),
158+
migrations.AddField(
159+
model_name="affectedbypackagerelatedvulnerability",
160+
name="severities",
161+
field=models.ManyToManyField(
162+
related_name="affected_package_vulnerability_relations",
163+
to="vulnerabilities.vulnerabilityseverity",
164+
),
165+
),
166+
migrations.AddField(
167+
model_name="vulnerability",
168+
name="severities",
169+
field=models.ManyToManyField(
170+
related_name="vulnerabilities", to="vulnerabilities.vulnerabilityseverity"
171+
),
172+
),
173+
migrations.AddField(
174+
model_name="vulnerabilityseverity",
175+
name="url",
176+
field=models.URLField(
177+
help_text="URL to the vulnerability severity", max_length=1024, null=True
178+
),
179+
),
180+
migrations.RunPython(
181+
code=remove_inaccurate_suse_score,
182+
reverse_code=reverse_remove_inaccurate_suse_score,
183+
),
184+
migrations.RunPython(
185+
code=populate_severity_model_with_url,
186+
reverse_code=reverse_populate_severity_model_reference_using_url,
187+
),
188+
migrations.RunPython(
189+
code=populate_vulnerability_model_with_severities,
190+
reverse_code=reverse_populate_vulnerability_model_with_severities,
191+
),
192+
migrations.RemoveField(
193+
model_name="vulnerabilityseverity",
194+
name="reference",
195+
),
196+
]

‎vulnerabilities/models.py‎

Lines changed: 46 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,42 @@ def with_package_counts(self):
156156
)
157157

158158

159+
class VulnerabilitySeverity(models.Model):
160+
url = models.URLField(
161+
max_length=1024,
162+
null=True,
163+
help_text="URL to the vulnerability severity",
164+
)
165+
166+
scoring_system_choices = tuple(
167+
(system.identifier, system.name) for system in SCORING_SYSTEMS.values()
168+
)
169+
170+
scoring_system = models.CharField(
171+
max_length=50,
172+
choices=scoring_system_choices,
173+
help_text="Identifier for the scoring system used. Available choices are: {} ".format(
174+
",\n".join(f"{sid}: {sname}" for sid, sname in scoring_system_choices)
175+
),
176+
)
177+
178+
value = models.CharField(max_length=50, help_text="Example: 9.0, Important, High")
179+
180+
scoring_elements = models.CharField(
181+
max_length=150,
182+
null=True,
183+
help_text="Supporting scoring elements used to compute the score values. "
184+
"For example a CVSS vector string as used to compute a CVSS score.",
185+
)
186+
187+
published_at = models.DateTimeField(
188+
blank=True, null=True, help_text="UTC Date of publication of the vulnerability severity"
189+
)
190+
191+
class Meta:
192+
ordering = ["url", "scoring_system", "value"]
193+
194+
159195
class VulnerabilityStatusType(models.IntegerChoices):
160196
"""List of vulnerability statuses."""
161197

@@ -202,6 +238,11 @@ class Vulnerability(models.Model):
202238
choices=VulnerabilityStatusType.choices, default=VulnerabilityStatusType.PUBLISHED
203239
)
204240

241+
severities = models.ManyToManyField(
242+
VulnerabilitySeverity,
243+
related_name="vulnerabilities",
244+
)
245+
205246
objects = VulnerabilityQuerySet.as_manager()
206247

207248
class Meta:
@@ -215,13 +256,6 @@ def __str__(self):
215256
def vcid(self):
216257
return self.vulnerability_id
217258

218-
@property
219-
def severities(self):
220-
"""
221-
Return a queryset of VulnerabilitySeverity for this vulnerability.
222-
"""
223-
return VulnerabilitySeverity.objects.filter(reference__in=self.references.all())
224-
225259
@property
226260
def affected_packages(self):
227261
"""
@@ -986,41 +1020,14 @@ class Meta(PackageRelatedVulnerabilityBase.Meta):
9861020

9871021

9881022
class AffectedByPackageRelatedVulnerability(PackageRelatedVulnerabilityBase):
989-
class Meta(PackageRelatedVulnerabilityBase.Meta):
990-
verbose_name_plural = "Affected By Package Related Vulnerabilities"
991-
992-
993-
class VulnerabilitySeverity(models.Model):
994-
reference = models.ForeignKey(VulnerabilityReference, on_delete=models.CASCADE)
9951023

996-
scoring_system_choices = tuple(
997-
(system.identifier, system.name) for system in SCORING_SYSTEMS.values()
1024+
severities = models.ManyToManyField(
1025+
VulnerabilitySeverity,
1026+
related_name="affected_package_vulnerability_relations",
9981027
)
9991028

1000-
scoring_system = models.CharField(
1001-
max_length=50,
1002-
choices=scoring_system_choices,
1003-
help_text="Identifier for the scoring system used. Available choices are: {} ".format(
1004-
",\n".join(f"{sid}: {sname}" for sid, sname in scoring_system_choices)
1005-
),
1006-
)
1007-
1008-
value = models.CharField(max_length=50, help_text="Example: 9.0, Important, High")
1009-
1010-
scoring_elements = models.CharField(
1011-
max_length=150,
1012-
null=True,
1013-
help_text="Supporting scoring elements used to compute the score values. "
1014-
"For example a CVSS vector string as used to compute a CVSS score.",
1015-
)
1016-
1017-
published_at = models.DateTimeField(
1018-
blank=True, null=True, help_text="UTC Date of publication of the vulnerability severity"
1019-
)
1020-
1021-
class Meta:
1022-
unique_together = ["reference", "scoring_system", "value"]
1023-
ordering = ["reference", "scoring_system", "value"]
1029+
class Meta(PackageRelatedVulnerabilityBase.Meta):
1030+
verbose_name_plural = "Affected By Package Related Vulnerabilities"
10241031

10251032

10261033
class AliasQuerySet(BaseQuerySet):

0 commit comments

Comments
 (0)