fail incomplete task(s) at startup #432 (#433)

Author: fqrious

docker-compose.yml

@@ -14,8 +14,8 @@ services:
             - .:/usr/src/app
         command: >
                 bash -c "
-                        python manage.py makemigrations &&
-                                python manage.py migrate
+                        python manage.py migrate &&
+                                python manage.py fail_incomplete_tasks
                         "
     django:
         extends: django_env

stixify/web/management/commands/fail_incomplete_tasks.py

@@ -0,0 +1,23 @@
+from django.core.cache import cache
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from stixify.web import models
+
+
+class Command(BaseCommand):
+    help = "Fail incomplete jobs and remove their upload locks."
+
+    def handle(self, *args, **options):
+        incomplete_jobs = models.Job.objects.filter(
+            state__in=(models.JobState.PENDING, models.JobState.PROCESSING)
+        )
+        job_ids = list(incomplete_jobs.values_list("id", flat=True))
+        updated = incomplete_jobs.update(
+            state=models.JobState.CANCELED,
+            error="canceled automatically during restart",
+            completion_time=timezone.now(),
+        )
+        cache.delete_many([f"arango_upload_lock:{job_id}" for job_id in job_ids])
+        cache.delete("arango_upload_active_count")
+        self.stdout.write(f"Failed {updated} incomplete job(s).")

stixify/web/migrations/0029_alter_job_state.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.12 on 2026-08-12 14:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('stixify_core', '0028_rename_reprocess_job_type'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='job',
+            name='state',
+            field=models.CharField(choices=[('pending', 'Pending'), ('processing', 'Processing'), ('failed', 'Failed'), ('completed', 'Completed'), ('canceled', 'Canceled')], default='pending', max_length=20),
+        ),
+    ]

stixify/web/models.py

@@ -297,6 +297,7 @@ class JobState(models.TextChoices):
     PROCESSING = "processing"
     FAILED = "failed"
     COMPLETED = "completed"
+    CANCELED = "canceled"
 
 class JobType(models.TextChoices):
     IMPORT_FILE = "import-file"

tests/src/commands/test_fail_incomplete_tasks.py

@@ -0,0 +1,39 @@
+import pytest
+from django.core.cache import cache
+from django.core.management import call_command
+
+from stixify.web import models
+
+
+@pytest.mark.django_db
+def test_fail_incomplete_tasks(stixify_file):
+    pending_job = models.Job.objects.create(file=stixify_file)
+    processing_job = models.Job.objects.create(
+        file=stixify_file, state=models.JobState.PROCESSING
+    )
+    completed_job = models.Job.objects.create(
+        file=stixify_file, state=models.JobState.COMPLETED
+    )
+    lock_keys = [
+        f"arango_upload_lock:{pending_job.id}",
+        # f"arango_upload_lock:{processing_job.id}", // test that cache delete_many works with missing keys
+    ]
+    cache.set_many({key: "locked" for key in lock_keys})
+    cache.set("arango_upload_active_count", 2)
+    cache.set("unrelated", "preserved")
+
+    call_command("fail_incomplete_tasks")
+
+    pending_job.refresh_from_db()
+    processing_job.refresh_from_db()
+    completed_job.refresh_from_db()
+    for job in (pending_job, processing_job):
+        assert job.state == models.JobState.CANCELED
+        assert job.error == "canceled automatically during restart"
+        assert job.completion_time is not None
+    assert completed_job.state == models.JobState.COMPLETED
+    assert completed_job.error is None
+    assert completed_job.completion_time is None
+    assert cache.get_many(lock_keys) == {}
+    assert cache.get("arango_upload_active_count") is None
+    assert cache.get("unrelated") == "preserved"