Refcator with AdvisoryHistoryDiff model and pipeline

Signed-off-by: Sampurna Pyne <sampurnapyne1710@gmail.com>

Author: Samk1710

vulnerabilities/improvers/__init__.py

@@ -21,6 +21,7 @@
 from vulnerabilities.pipelines.v2_improvers import (
     group_advisories_for_packages as group_advisories_for_packages_v2,
 )
+from vulnerabilities.pipelines.v2_improvers import history_diff_backfill
 from vulnerabilities.pipelines.v2_improvers import mark_unfurl_version_range
 from vulnerabilities.pipelines.v2_improvers import reference_collect_commits
 from vulnerabilities.pipelines.v2_improvers import relate_severities
@@ -42,6 +43,7 @@
         reference_collect_commits.CollectReferencesFixCommitsPipeline,
         enhance_with_github_poc.GithubPocsImproverPipeline,
         mark_unfurl_version_range.MarkUnfurlVersionRangePipeline,
+        history_diff_backfill.HistoryDiffImproverPipeline,
         group_advisories_for_packages_v2.GroupAdvisoriesForPackages,
     ]
 )

vulnerabilities/migrations/0138_advisoryhistorydiff.py

@@ -0,0 +1,38 @@
+# Generated by Django 5.2.11 on 2026-07-25 16:54
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vulnerabilities', '0137_alter_pipelineschedule_run_interval'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AdvisoryHistoryDiff',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('summary_added', models.TextField(blank=True, null=True)),
+                ('summary_removed', models.TextField(blank=True, null=True)),
+                ('url_added', models.URLField(blank=True, null=True)),
+                ('url_removed', models.URLField(blank=True, null=True)),
+                ('added_aliases', models.ManyToManyField(blank=True, related_name='added_in_history', to='vulnerabilities.advisoryalias')),
+                ('added_impacted_packages', models.ManyToManyField(blank=True, related_name='added_in_history', to='vulnerabilities.impactedpackage')),
+                ('added_patches', models.ManyToManyField(blank=True, related_name='added_in_history', to='vulnerabilities.patch')),
+                ('added_references', models.ManyToManyField(blank=True, related_name='added_in_history', to='vulnerabilities.advisoryreference')),
+                ('added_severities', models.ManyToManyField(blank=True, related_name='added_in_history', to='vulnerabilities.advisoryseverity')),
+                ('added_weaknesses', models.ManyToManyField(blank=True, related_name='added_in_history', to='vulnerabilities.advisoryweakness')),
+                ('advisory_after', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='history_diff', to='vulnerabilities.advisoryv2')),
+                ('advisory_before', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='history_diff_before', to='vulnerabilities.advisoryv2')),
+                ('removed_aliases', models.ManyToManyField(blank=True, related_name='removed_in_history', to='vulnerabilities.advisoryalias')),
+                ('removed_impacted_packages', models.ManyToManyField(blank=True, related_name='removed_in_history', to='vulnerabilities.impactedpackage')),
+                ('removed_patches', models.ManyToManyField(blank=True, related_name='removed_in_history', to='vulnerabilities.patch')),
+                ('removed_references', models.ManyToManyField(blank=True, related_name='removed_in_history', to='vulnerabilities.advisoryreference')),
+                ('removed_severities', models.ManyToManyField(blank=True, related_name='removed_in_history', to='vulnerabilities.advisoryseverity')),
+                ('removed_weaknesses', models.ManyToManyField(blank=True, related_name='removed_in_history', to='vulnerabilities.advisoryweakness')),
+            ],
+        ),
+    ]

vulnerabilities/models.py

@@ -3367,6 +3367,138 @@ def get_aliases(self):
     alias = get_aliases
 
 
+class AdvisoryHistoryDiff(models.Model):
+    """
+    Stores the pre-computed diff between an AdvisoryV2 snapshot (advisory_after)
+    and its immediate predecessor (advisory_before).
+    Created for every snapshot.
+    """
+
+    # Points to the older snapshot in the comparison
+    advisory_before = models.OneToOneField(
+        AdvisoryV2,
+        on_delete=models.SET_NULL,
+        related_name="history_diff_before",
+        null=True,  # The very first snapshot won't have a predecessor
+        blank=True,
+    )
+
+    # Points to the newer snapshot in the comparison
+    advisory_after = models.OneToOneField(
+        AdvisoryV2, on_delete=models.SET_NULL, related_name="history_diff", null=True, blank=True
+    )
+
+    summary_added = models.TextField(blank=True, null=True)
+    summary_removed = models.TextField(blank=True, null=True)
+
+    url_added = models.URLField(blank=True, null=True)
+    url_removed = models.URLField(blank=True, null=True)
+
+    added_severities = models.ManyToManyField(
+        "AdvisorySeverity", related_name="added_in_history", blank=True
+    )
+    removed_severities = models.ManyToManyField(
+        "AdvisorySeverity", related_name="removed_in_history", blank=True
+    )
+
+    added_impacted_packages = models.ManyToManyField(
+        "ImpactedPackage", related_name="added_in_history", blank=True
+    )
+    removed_impacted_packages = models.ManyToManyField(
+        "ImpactedPackage", related_name="removed_in_history", blank=True
+    )
+
+    added_references = models.ManyToManyField(
+        "AdvisoryReference", related_name="added_in_history", blank=True
+    )
+    removed_references = models.ManyToManyField(
+        "AdvisoryReference", related_name="removed_in_history", blank=True
+    )
+
+    added_aliases = models.ManyToManyField(
+        "AdvisoryAlias", related_name="added_in_history", blank=True
+    )
+    removed_aliases = models.ManyToManyField(
+        "AdvisoryAlias", related_name="removed_in_history", blank=True
+    )
+
+    added_weaknesses = models.ManyToManyField(
+        "AdvisoryWeakness", related_name="added_in_history", blank=True
+    )
+    removed_weaknesses = models.ManyToManyField(
+        "AdvisoryWeakness", related_name="removed_in_history", blank=True
+    )
+
+    added_patches = models.ManyToManyField("Patch", related_name="added_in_history", blank=True)
+    removed_patches = models.ManyToManyField("Patch", related_name="removed_in_history", blank=True)
+
+    def to_diff_dict(self) -> dict:
+        """
+        Serialize the relational diff back to the dict format consumed by
+        format_diff_for_ui and the existing template.
+        """
+        diff = {}
+
+        if self.summary_added is not None or self.summary_removed is not None:
+            diff["summary"] = {"old": self.summary_removed, "new": self.summary_added}
+
+        if self.url_added is not None or self.url_removed is not None:
+            diff["url"] = {"old": self.url_removed, "new": self.url_added}
+
+        def _serialize_history_diff(key, added_qs, removed_qs, serializer):
+            added = list(added_qs)
+            removed = list(removed_qs)
+            if added or removed:
+                diff[key] = {
+                    "added": [serializer(obj) for obj in added],
+                    "removed": [serializer(obj) for obj in removed],
+                }
+
+        _serialize_history_diff(
+            "severities",
+            self.added_severities.all(),
+            self.removed_severities.all(),
+            lambda s: s.to_dict(),
+        )
+
+        _serialize_history_diff(
+            "affected_packages",
+            self.added_impacted_packages.all(),
+            self.removed_impacted_packages.all(),
+            lambda p: p.to_dict(),
+        )
+
+        _serialize_history_diff(
+            "references",
+            self.added_references.all(),
+            self.removed_references.all(),
+            lambda r: r.to_dict(),
+        )
+
+        _serialize_history_diff(
+            "aliases",
+            self.added_aliases.all(),
+            self.removed_aliases.all(),
+            lambda a: a.alias,
+        )
+
+        _serialize_history_diff(
+            "weaknesses",
+            self.added_weaknesses.all(),
+            self.removed_weaknesses.all(),
+            lambda w: w.cwe_id,
+        )
+
+        _serialize_history_diff(
+            "patches",
+            self.added_patches.all(),
+            self.removed_patches.all(),
+            lambda p: p.to_dict(),
+        )
+
+        return diff
+
+
 class ImpactedPackage(models.Model):
     """
     Represents a single impact for an advisory, including affected range and fixed version and

vulnerabilities/pipelines/__init__.py

@@ -284,6 +284,10 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline):
     # pull requests, commit messages, EPSS, exploits, etc.
     exclude_from_package_todo = False
 
+    # Set this to True if computing history diff is not fruitful for this source.
+    # An example of such advisory would be pipeline dedicated to EPSS
+    exclude_from_history_diff = False
+
     # Control how often progress log is shown (range: 1–100, higher value = less frequent log)
     progress_step = 10
 
@@ -344,6 +348,7 @@ def collect_and_store_advisories(self):
                     datasource_id=self.datasource_id,
                     logger=self.log,
                     precedence=self.precedence,
+                    calculate_history_diff=not self.exclude_from_history_diff,
                 ):
                     collected_advisory_count += 1
             except Exception as e:

vulnerabilities/pipelines/v2_improvers/history_diff_backfill.py

@@ -0,0 +1,81 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from aboutcode.pipeline import LoopProgress
+
+from vulnerabilities.importers import IMPORTERS_REGISTRY
+from vulnerabilities.models import AdvisoryHistoryDiff
+from vulnerabilities.models import AdvisoryV2
+from vulnerabilities.pipelines import VulnerableCodePipeline
+from vulnerabilities.utils import compute_advisory_v2_diff
+
+
+class HistoryDiffImproverPipeline(VulnerableCodePipeline):
+    """
+    Pipeline to compute and store relational history snapshot diffs.
+    """
+
+    pipeline_id = "history_diff_improver_v2"
+    run_once = True
+
+    @classmethod
+    def steps(cls):
+        return (cls.calculate_history_diffs,)
+
+    def calculate_history_diffs(self):
+        # Skip excluded importers
+        excluded_ids = [
+            cls.pipeline_id
+            for cls in IMPORTERS_REGISTRY.values()
+            if getattr(cls, "exclude_from_history_diff", False)
+            and getattr(cls, "pipeline_id", None)
+        ]
+        qs = (
+            AdvisoryV2.objects.exclude(pipeline_id__in=excluded_ids)
+            if excluded_ids
+            else AdvisoryV2.objects.all()
+        )
+
+        avids = qs.filter(history_diff__isnull=True).values_list("avid", flat=True).distinct()
+        self.log(f"Computing history diffs for {len(avids)} advisories")
+
+        for avid in LoopProgress(
+            total_iterations=len(avids), logger=self.log, progress_step=10
+        ).iter(avids):
+            snapshots = list(
+                qs.filter(avid=avid)
+                .order_by("date_collected", "unique_content_id")
+                .prefetch_related(
+                    "aliases",
+                    "references",
+                    "weaknesses",
+                    "severities",
+                    "patches",
+                    "impacted_packages",
+                )
+            )
+
+            previous_snapshot = None
+
+            # TODO:
+            # Bulk diff creation
+
+            for snapshot in snapshots:
+                if not hasattr(snapshot, "history_diff"):  # Skip already processed advisories
+                    if previous_snapshot is not None:
+                        compute_advisory_v2_diff(previous_snapshot, snapshot)
+                    else:
+                        # For the very first snapshot, we create an empty diff to mark it as processed
+                        AdvisoryHistoryDiff.objects.create(
+                            advisory_after=snapshot, advisory_before=None
+                        )
+
+                previous_snapshot = snapshot
+
+        self.log("Successfully finished history diff backfill.")

vulnerabilities/pipes/advisory.py

@@ -33,6 +33,7 @@
 from vulnerabilities.improver import MAX_CONFIDENCE
 from vulnerabilities.models import Advisory
 from vulnerabilities.models import AdvisoryAlias
+from vulnerabilities.models import AdvisoryHistoryDiff
 from vulnerabilities.models import AdvisoryReference
 from vulnerabilities.models import AdvisorySeverity
 from vulnerabilities.models import AdvisoryV2
@@ -49,6 +50,7 @@
 from vulnerabilities.models import Weakness
 from vulnerabilities.pipes.risk_score import compute_advisory_risk_score
 from vulnerabilities.pipes.univers_utils import get_exact_purls_v2
+from vulnerabilities.utils import compute_advisory_v2_diff
 
 
 def get_or_create_aliases(aliases: List) -> QuerySet:
@@ -295,6 +297,7 @@ def insert_advisory_v2(
     logger: Callable,
     datasource_id: str,
     precedence: int = 0,
+    calculate_history_diff: bool = True,
 ):
     from vulnerabilities.models import ImpactedPackage
     from vulnerabilities.models import PackageV2
@@ -338,12 +341,16 @@ def insert_advisory_v2(
     if not created:
         return advisory_obj
 
-    AdvisoryV2.objects.filter(
+    latest_qs = AdvisoryV2.objects.filter(
         avid=f"{datasource_id}/{advisory.advisory_id}",
         is_latest=True,
-    ).update(is_latest=False)
+    )
+
+    previous_latest_advisory = latest_qs.first()
+    latest_qs.update(is_latest=False)
+
     advisory_obj.is_latest = True
-    advisory_obj.save()
+    advisory_obj.save(update_fields=["is_latest"])
 
     aliases = get_or_create_advisory_aliases(aliases=advisory.aliases)
     references = get_or_create_advisory_references(references=advisory.references)
@@ -416,6 +423,31 @@ def insert_advisory_v2(
         )
         impact.introduced_by_package_commit_patches.add(*introduced_commit_v2)
         impact.fixed_by_package_commit_patches.add(*fixed_commit_v2)
+
+    if calculate_history_diff:
+        if previous_latest_advisory:
+            previous = AdvisoryV2.objects.prefetch_related(
+                "aliases",
+                "references",
+                "weaknesses",
+                "severities",
+                "patches",
+                "impacted_packages",
+            ).get(id=previous_latest_advisory.id)
+
+            current = AdvisoryV2.objects.prefetch_related(
+                "aliases",
+                "references",
+                "weaknesses",
+                "severities",
+                "patches",
+                "impacted_packages",
+            ).get(id=advisory_obj.id)
+
+            compute_advisory_v2_diff(previous, current)
+        else:
+            AdvisoryHistoryDiff.objects.create(advisory_after=advisory_obj, advisory_before=None)
+
     return advisory_obj