Frontend - analyzables page + user events modal (#2931)

* Frontend - Analyzables page (#2865)

* adjusted menu

* analyzables page draft + refactor

* adjusted table columns and serializer

* frontend tests

* tests

* prettier

* fix

* changes

* fix

* Analyzable overview page (#2883)

* analyzable overview page draft

* refactor

* visualizers draft

* Analyzable overview page

* prettier

* removed cache

* changes

* adjusted tests

* fixed decay date visualizer

* use enum

---------

Co-authored-by: drosetti <d.rosetti@certego.net>

* added data model tab (#2900)

* Added 'user reports' in the history page (#2895)

* added user-reports page

* fix

* changes

* tests

* changes

* prettier

* fixed decayed badge

* fix

* User events modal form (#2928)

* form draft

* adjusted backend

* adjusted analyzable history

* fix

* fixed evaluation modal

* renamed user reports to user events

* renamed analyzable to artifact in the gui

* adjusted frontend tests

* fix

* frontend tests

* fix frontend tests

* prettier

* fix

* fix

* fix

* refactor history tables

* fix

---------

Co-authored-by: drosetti <d.rosetti@certego.net>

Author: carellamartina

api_app/analyzables_manager/filters.py

@@ -0,0 +1,18 @@
+import rest_framework_filters as filters
+from django_filters.widgets import QueryArrayWidget
+
+from api_app.analyzables_manager.models import Analyzable
+
+
+class CharInFilter(filters.BaseInFilter, filters.CharFilter):
+    pass
+
+
+class AnalyzableFilter(filters.FilterSet):
+    name = CharInFilter(widget=QueryArrayWidget)
+
+    class Meta:
+        model = Analyzable
+        fields = {
+            "discovery_date": ["lte", "gte"],
+        }

api_app/analyzables_manager/models.py

@@ -7,12 +7,7 @@
 
 from api_app.analyzables_manager.queryset import AnalyzableQuerySet
 from api_app.choices import Classification
-from api_app.data_model_manager.models import (
-    BaseDataModel,
-    DomainDataModel,
-    FileDataModel,
-    IPDataModel,
-)
+from api_app.data_model_manager.models import BaseDataModel
 from api_app.data_model_manager.queryset import BaseDataModelQuerySet
 from api_app.defaults import file_directory_path
 from api_app.helpers import calculate_md5, calculate_sha1, calculate_sha256
@@ -85,20 +80,7 @@ def get_all_user_events_data_model(
         return self.get_data_model_class().objects.filter(query)
 
     def get_data_model_class(self) -> Type[BaseDataModel]:
-        if self.classification == Classification.IP.value:
-            return IPDataModel
-        elif self.classification in [
-            Classification.URL.value,
-            Classification.DOMAIN.value,
-        ]:
-            return DomainDataModel
-        elif self.classification in [
-            Classification.HASH.value,
-            Classification.FILE.value,
-        ]:
-            return FileDataModel
-        else:
-            raise NotImplementedError()
+        return self.CLASSIFICATIONS.get_data_model_class(self.classification)
 
     def _set_hashes(self, value: Union[str, bytes]):
         if isinstance(value, str):
@@ -111,6 +93,10 @@ def _set_hashes(self, value: Union[str, bytes]):
             self.sha1 = calculate_sha1(value)
 
     def clean(self):
+        if self.file:
+            self.classification = Classification.FILE.value
+        else:
+            self.classification = Classification.calculate_observable(self.name)
         if self.classification == Classification.FILE.value:
             from api_app.analyzers_manager.models import MimeTypes
 

api_app/analyzables_manager/queryset.py

@@ -10,14 +10,21 @@ class AnalyzableQuerySet(QuerySet):
     def visible_for_user(self, user):
 
         from api_app.models import Job
+        from api_app.user_events_manager.models import UserAnalyzableEvent
 
-        analyzables = (
+        analyzables_job = (
             Job.objects.visible_for_user(user)
             .values("analyzable")
             .distinct()
             .values_list("analyzable__pk", flat=True)
         )
-        return self.filter(pk__in=analyzables)
+        analyzables_ue = (
+            UserAnalyzableEvent.objects.visible_for_user(user)
+            .values("analyzable")
+            .distinct()
+            .values_list("analyzable__pk", flat=True)
+        )
+        return self.filter(pk__in=analyzables_job) | self.filter(pk__in=analyzables_ue)
 
     def create(self, *args, **kwargs):
         obj = self.model(**kwargs)

api_app/analyzables_manager/serializers.py

@@ -1,12 +1,58 @@
-from rest_framework.serializers import ModelSerializer
+from rest_framework import serializers as rfs
 
 from api_app.analyzables_manager.models import Analyzable
+from api_app.choices import Classification
+from api_app.models import Job
 from api_app.serializers.job import JobRelatedField
 
 
-class AnalyzableSerializer(ModelSerializer):
+class AnalyzableSerializer(rfs.ModelSerializer):
     jobs = JobRelatedField(many=True, read_only=True)
 
     class Meta:
         model = Analyzable
         fields = "__all__"
+        read_only_fields = [
+            "jobs",
+            "discovery_date",
+            "md5",
+            "classification",
+            "sha256",
+            "sha1",
+            "mimetype",
+        ]
+
+    def to_representation(self, instance):
+        analyzable = super().to_representation(instance)
+        job = (
+            Job.objects.filter(id__in=analyzable["jobs"])
+            .order_by("-finished_analysis_time")
+            .first()
+        )
+        user_event_data_model = (
+            instance.get_all_user_events_data_model().order_by("-date").first()
+        )
+
+        if job is None and user_event_data_model is None:
+            analyzable["last_data_model"] = None
+            return analyzable
+        elif job is not None and user_event_data_model is not None:
+            last_data_model = (
+                job.data_model
+                if job.data_model.date > user_event_data_model.date
+                else user_event_data_model
+            )
+        else:
+            last_data_model = (
+                job.data_model if job is not None else user_event_data_model
+            )
+
+        serializer_class = Classification.get_data_model_class(
+            classification=analyzable["classification"],
+        ).get_serializer()
+        analyzable["last_data_model"] = serializer_class(last_data_model).data
+        return analyzable
+
+    def create(self, validated_data):
+        instance, _ = self.Meta.model.objects.get_or_create(**validated_data)
+        return instance

api_app/analyzables_manager/views.py

@@ -1,14 +1,76 @@
+import logging
+from http import HTTPStatus
+
 from rest_framework import viewsets
+from rest_framework.decorators import action
+from rest_framework.exceptions import ValidationError
 from rest_framework.permissions import IsAuthenticated
+from rest_framework.response import Response
 
+from api_app.analyzables_manager.filters import AnalyzableFilter
+from api_app.analyzables_manager.models import Analyzable
 from api_app.analyzables_manager.serializers import AnalyzableSerializer
+from api_app.serializers.job import JobAnalyzableHistorySerializer
+from api_app.user_events_manager.serializers import (
+    UserAnalyzableEventSerializer,
+    UserDomainWildCardEventSerializer,
+    UserIPWildCardEventSerializer,
+)
+
+logger = logging.getLogger(__name__)
 
 
 class AnalyzableViewSet(viewsets.ReadOnlyModelViewSet):
 
     serializer_class = AnalyzableSerializer
     permission_classes = [IsAuthenticated]
+    queryset = Analyzable.objects.all()
+    filterset_class = AnalyzableFilter
 
     def get_queryset(self):
         user = self.request.user
         return super().get_queryset().visible_for_user(user)
+
+    @action(detail=True)
+    def history(self, request, pk=None):
+        user = request.user
+        try:
+            analyzable: Analyzable = self.get_queryset().get(pk=pk)
+        except Analyzable.DoesNotExist:
+            raise ValidationError({"detail": "Requested analyzable does not exist."})
+
+        jobs_queryset = analyzable.jobs.visible_for_user(user).order_by(
+            "-finished_analysis_time"
+        )
+        user_events_queryset = analyzable.user_events.visible_for_user(user).order_by(
+            "-date"
+        )
+        user_domain_wildcard_events_queryset = (
+            analyzable.user_domain_wildcard_events.visible_for_user(user).order_by(
+                "-date"
+            )
+        )
+        user_ip_wildcard_events_queryset = (
+            analyzable.user_ip_wildcard_events.visible_for_user(user).order_by("-date")
+        )
+
+        jobs = JobAnalyzableHistorySerializer(jobs_queryset, many=True).data
+        user_events = UserAnalyzableEventSerializer(
+            user_events_queryset, many=True
+        ).data
+        user_domain_wildcard_events = UserDomainWildCardEventSerializer(
+            user_domain_wildcard_events_queryset, many=True
+        ).data
+        user_ip_wildcard_events = UserIPWildCardEventSerializer(
+            user_ip_wildcard_events_queryset, many=True
+        ).data
+
+        return Response(
+            status=HTTPStatus.OK.value,
+            data={
+                "jobs": jobs,
+                "user_events": user_events,
+                "user_domain_wildcard_events": user_domain_wildcard_events,
+                "user_ip_wildcard_events": user_ip_wildcard_events,
+            },
+        )

api_app/choices.py

@@ -161,6 +161,29 @@ def calculate_observable(cls, value: str) -> str:
 
         return classification
 
+    @classmethod
+    def get_data_model_class(cls, classification: str) -> typing.Type:
+        from api_app.data_model_manager.models import (
+            DomainDataModel,
+            FileDataModel,
+            IPDataModel,
+        )
+
+        if classification == cls.IP.value:
+            return IPDataModel
+        elif classification in [
+            cls.URL.value,
+            cls.DOMAIN.value,
+        ]:
+            return DomainDataModel
+        elif classification in [
+            cls.HASH.value,
+            cls.FILE.value,
+        ]:
+            return FileDataModel
+        else:
+            raise NotImplementedError()
+
 
 class ScanMode(models.IntegerChoices):
     FORCE_NEW_ANALYSIS = 1

api_app/serializers/job.py

@@ -1288,3 +1288,24 @@ def to_representation(self, instance: Job):
     @staticmethod
     def get_playbook(instance: Job):
         return instance.playbook_to_execute.name if instance.playbook_to_execute else ""
+
+
+class JobAnalyzableHistorySerializer(rfs.ModelSerializer):
+    id = rfs.CharField(source="pk")
+    user = rfs.CharField(source="user.username", allow_null=False, read_only=True)
+    data_model = rfs.SerializerMethodField()
+    date = rfs.DateTimeField(
+        source="finished_analysis_time", read_only=True, allow_null=False
+    )
+    playbook = rfs.CharField(
+        source="playbook_to_execute.name", allow_null=True, read_only=True
+    )
+
+    class Meta:
+        model = Job
+        fields = ["playbook", "user", "date", "data_model", "id"]
+
+    def get_data_model(self, instance: Job):
+        if instance.data_model:
+            return instance.data_model.serialize()
+        return {}

api_app/urls.py

@@ -58,6 +58,7 @@
     path("", include("api_app.investigations_manager.urls")),
     path("data_model/", include("api_app.data_model_manager.urls")),
     path("user_event/", include("api_app.user_events_manager.urls")),
+    path("", include("api_app.analyzables_manager.urls")),
     # auth
     path("auth/", include("authentication.urls")),
     # certego_saas:

api_app/user_events_manager/filters.py

@@ -1,28 +1,104 @@
+import ipaddress
+
 import rest_framework_filters as filters
+from django.contrib.contenttypes.models import ContentType
+
+from api_app.data_model_manager.models import (
+    DomainDataModel,
+    FileDataModel,
+    IPDataModel,
+)
+from api_app.user_events_manager.models import UserAnalyzableEvent, UserEvent
 
 
 class UserEventFilterSet(filters.FilterSet):
 
     username = filters.CharFilter(lookup_expr="iexact", field_name="user__username")
     next_decay = filters.DateRangeFilter()
-    date = filters.DateRangeFilter()
+    id = filters.CharFilter(method="filter_for_id")
+
+    class Meta:
+        model = UserEvent
+        fields = {
+            "date": ["lte", "gte"],
+        }
+
+    @staticmethod
+    def filter_for_id(queryset, value, _id, *args, **kwargs):
+        try:
+            int_id = int(_id)
+        except ValueError:
+            # this is to manage bad data as input
+            return queryset
+        else:
+            return queryset.filter(id=int_id)
 
 
 class UserAnalyzableEventFilterSet(UserEventFilterSet):
     analyzable_name = filters.CharFilter(
         field_name="analyzable__name", lookup_expr="icontains"
     )
+    event_date__gte = filters.CharFilter(method="filter_for_event_date")
+    event_date__lte = filters.CharFilter(method="filter_for_event_date")
+
+    @staticmethod
+    def filter_for_event_date(queryset, value, _date, *args, **kwargs):
+        date_filters = {}
+        if value.endswith("__gte"):
+            date_filters["date__gte"] = _date
+        if value.endswith("__lte"):
+            date_filters["date__lte"] = _date
+
+        data_model_map = {
+            DomainDataModel: ContentType.objects.get_for_model(DomainDataModel),
+            IPDataModel: ContentType.objects.get_for_model(IPDataModel),
+            FileDataModel: ContentType.objects.get_for_model(FileDataModel),
+        }
+
+        user_events_ids = []
+        for model, content_type in data_model_map.items():
+            data_model_ids = model.objects.filter(**date_filters).values_list(
+                "id", flat=True
+            )
+            if data_model_ids:
+                ids = UserAnalyzableEvent.objects.filter(
+                    data_model_content_type=content_type,
+                    data_model_object_id__in=data_model_ids,
+                ).values_list("id", flat=True)
+                user_events_ids.extend(ids)
+
+        return queryset.filter(id__in=user_events_ids)
 
 
 class UserDomainWildCardEventFilterSet(UserEventFilterSet):
     query = filters.CharFilter(field_name="query", lookup_expr="icontains")
     analyzables = filters.BaseInFilter(field_name="analyzables__name")
+    event_date__gte = filters.CharFilter(
+        field_name="data_model__date", lookup_expr="gte"
+    )
+    event_date__lte = filters.CharFilter(
+        field_name="data_model__date", lookup_expr="lte"
+    )
 
 
 class UserIPWildCardEventFilterSet(UserEventFilterSet):
     ip = filters.CharFilter(method="filter_for_ip", lookup_expr="icontains")
+    network = filters.CharFilter(method="filter_for_network")
     analyzables = filters.BaseInFilter(field_name="analyzables__name")
+    event_date__gte = filters.CharFilter(
+        field_name="data_model__date", lookup_expr="gte"
+    )
+    event_date__lte = filters.CharFilter(
+        field_name="data_model__date", lookup_expr="lte"
+    )
+
+    @staticmethod
+    def filter_for_ip(queryset, value, _ip, *args, **kwargs):
+        return queryset.filter(start_ip__lte=_ip, end_ip__gte=_ip)
 
     @staticmethod
-    def filter_for_ip(queryset, value, user, *args, **kwargs):
-        return queryset.filter(start_ip__lte=value, end_ip__gte=value)
+    def filter_for_network(queryset, value, _network, *args, **kwargs):
+        network = ipaddress.IPv4Network(_network)
+        start_ip = str(network[0])
+        end_ip = str(network[-1])
+        return queryset.filter(start_ip=start_ip, end_ip=end_ip)