Merge remote-tracking branch 'origin/main' into save-risk

# Conflicts:
#	vulnerabilities/models.py
#	vulnerabilities/pipelines/compute_package_risk.py
#	vulnerabilities/risk.py

Author: ziadhany

CHANGELOG.rst

@@ -1,8 +1,23 @@
 Release notes
 =============
 
-Version (next)
------------------------
+Version v34.3.2
+----------------
+
+- HOTFIX: Add reference score to package endpoint #1655
+
+
+Version v34.3.1
+----------------
+
+- HOTFIX: Fix API bug #1654
+
+
+Version v34.3.0
+-----------------
+
+- Add bulk search in v2 #1649 
+- Refactor severity score model and fix incorrect suse scores #1636
 
 
 Version v34.2.0

docs/source/contributing.rst

@@ -89,7 +89,7 @@ Helpful Resources
 
 - Review our `comprehensive guide <https://scancode-toolkit.readthedocs.io/en/latest/contribute/index.html>`_
   for more details on how to add quality contributions to our codebase and documentation
-- Check this free resource on `how to contribute to an open source project on github <https://egghead.io/courses/how-to-contribute-to-an-open-source-project-on-github>`_
+- Check this free resource on `How to contribute to an open source project on github <https://egghead.io/lessons/javascript-identifying-how-to-contribute-to-an-open-source-project-on-github>`_
 - Follow `this wiki page <https://aboutcode.readthedocs.io/en/latest/contributing/writing_good_commit_messages.html>`_
   on how to write good commit messages
 - `Pro Git book <https://git-scm.com/book/en/v2>`_

docs/source/tutorial_add_importer_pipeline.rst

@@ -298,10 +298,14 @@ version management from `univers <https://github.com/aboutcode-org/univers>`_.
     **advisories_count** should never be directly added in steps.
 
 
+.. attention::
+
+   Implement ``on_failure`` to handle cleanup in case of pipeline failure.
+   Cleanup of downloaded archives or cloned repos is necessary to avoid potential resource leakage.
 
 .. note::
 
-   | Use ``make valid`` to format your code using black and isort automatically.
+   | Use ``make valid`` to format your new code using black and isort automatically.
    | Use ``make check`` to check for formatting errors.
 
 Register the Importer Pipeline

docs/source/tutorial_add_improver_pipeline.rst

@@ -187,6 +187,11 @@ methods.
             self.log(f"Successfully flagged {ghost_package_count:,d} ghost Packages")
 
 
+.. attention::
+
+   Implement ``on_failure`` to handle cleanup in case of pipeline failure.
+   Cleanup of downloaded archives or cloned repos is necessary to avoid potential resource leakage.
+
 .. note::
 
    | Use ``make valid`` to format your new code using black and isort automatically.

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = vulnerablecode
-version = 34.2.0
+version = 34.3.2
 license = Apache-2.0 AND CC-BY-SA-4.0
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390

vulnerabilities/api.py

@@ -54,13 +54,25 @@ def to_representation(self, instance):
 
 
 class VulnerabilityReferenceSerializer(serializers.ModelSerializer):
-    scores = VulnerabilitySeveritySerializer(many=True, source="vulnerabilityseverity_set")
+    scores = serializers.SerializerMethodField()
     reference_url = serializers.CharField(source="url")
 
     class Meta:
         model = VulnerabilityReference
         fields = ["reference_url", "reference_id", "reference_type", "scores", "url"]
 
+    def get_scores(self, instance):
+        severities_related_to_reference = [
+            severity
+            for severity in self.context.get("severities", [])
+            if severity.url == instance.url
+        ]
+
+        return VulnerabilitySeveritySerializer(
+            severities_related_to_reference,
+            many=True,
+        ).data
+
 
 class BaseResourceSerializer(serializers.HyperlinkedModelSerializer):
     """
@@ -143,14 +155,26 @@ class VulnSerializerRefsAndSummary(BaseResourceSerializer):
         many=True, source="filtered_fixed_packages", read_only=True
     )
 
-    references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set")
+    references = serializers.SerializerMethodField()
 
     aliases = serializers.SerializerMethodField()
 
     def get_aliases(self, obj):
         # Assuming `obj.aliases` is a queryset of `Alias` objects
         return [alias.alias for alias in obj.aliases.all()]
 
+    def get_references(self, vulnerability):
+        references = vulnerability.vulnerabilityreference_set.all()
+        severities = vulnerability.severities.all()
+
+        serialized_references = VulnerabilityReferenceSerializer(
+            references,
+            context={"severities": severities},
+            many=True,
+        ).data
+
+        return serialized_references
+
     class Meta:
         model = Vulnerability
         fields = ["url", "vulnerability_id", "summary", "references", "fixed_packages", "aliases"]
@@ -199,8 +223,7 @@ class VulnerabilitySerializer(BaseResourceSerializer):
         many=True, source="filtered_fixed_packages", read_only=True
     )
     affected_packages = MinimalPackageSerializer(many=True, read_only=True)
-
-    references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set")
+    references = serializers.SerializerMethodField()
     aliases = AliasSerializer(many=True, source="alias")
     exploits = ExploitSerializer(many=True, read_only=True)
     weaknesses = WeaknessSerializer(many=True)
@@ -214,10 +237,22 @@ def to_representation(self, instance):
 
         return data
 
+    def get_references(self, vulnerability):
+        references = vulnerability.vulnerabilityreference_set.all()
+        severities = vulnerability.severities.all()
+
+        serialized_references = VulnerabilityReferenceSerializer(
+            references,
+            context={"severities": severities},
+            many=True,
+        ).data
+
+        return serialized_references
+
     def get_severity_range_score(self, instance):
         severity_vectors = []
         severity_values = set()
-        for s in instance.severities:
+        for s in instance.severities.all():
             if s.scoring_system == EPSS.identifier:
                 continue
 
@@ -303,7 +338,7 @@ def get_vulnerabilities_for_a_package(self, package, fix) -> dict:
         otherwise return vulnerabilities fixed by the `package`.
         """
         fixed_packages = self.get_fixed_packages(package=package)
-        if fix:
+        if not fix:
             qs = package.affected_by_vulnerabilities.all()
         else:
             qs = package.fixing_vulnerabilities.all()

vulnerabilities/api_extension.py

@@ -84,11 +84,10 @@ class Meta:
 
 class V2VulnerabilitySeveritySerializer(ModelSerializer):
     score = CharField(source="value")
-    reference = V2VulnerabilityReferenceSerializer()
 
     class Meta:
         model = VulnerabilitySeverity
-        fields = ("score", "scoring_system", "scoring_elements", "published_at", "reference")
+        fields = ("url", "score", "scoring_system", "scoring_elements", "published_at")
 
 
 class V2WeaknessSerializer(ModelSerializer):
@@ -127,9 +126,9 @@ class V2VulnerabilitySerializer(ModelSerializer):
 
     aliases = SerializerMethodField("get_aliases")
     weaknesses = V2WeaknessSerializer(many=True, source="weaknesses_set")
-    scores = V2VulnerabilitySeveritySerializer(many=True, source="vulnerabilityseverity_set")
     references = V2VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set")
     exploits = V2ExploitSerializer(many=True, source="weaknesses")
+    severities = V2VulnerabilitySeveritySerializer(many=True)
 
     def get_aliases(self, vulnerability):
         return vulnerability.aliases.only("alias").values_list("alias", flat=True)
@@ -145,11 +144,11 @@ class Meta:
             "vulnerability_id",
             "aliases",
             "status",
-            "scores",
             "weaknesses",
             "summary",
             "exploits",
             "references",
+            "severities",
         )
 
 
@@ -358,7 +357,7 @@ def get_queryset(self):
             .get_queryset()
             .prefetch_related(
                 "weaknesses",
-                # "severities",
+                "severities",
                 # "exploits",
             )
         )