Merge pull request #45 from tpaulshippy/feature/roadmap-03-server-safety

Author: tpaulshippy

back/bots/admin.py

@@ -12,6 +12,7 @@
     Message,
     Profile,
     RevenueCatWebhookEvent,
+    SafetyEvent,
     UsageLimitHit,
     UserAccount,
 )
@@ -89,6 +90,14 @@ def get_readonly_fields(self, request, obj=None):
     def get_list_display(self, request):
         return ['flashcard_id', 'deck', 'front', 'order', 'created_at', 'updated_at'] + list(super().get_list_display(request))
 
+class SafetyEventAdmin(admin.ModelAdmin):
+    def get_readonly_fields(self, request, obj=None):
+        return ['created_at', 'event_id']
+
+    def get_list_display(self, request):
+        return ['created_at', 'stage', 'reason_code', 'user', 'chat', 'snippet_redacted'] + list(super().get_list_display(request))
+
+
 class UserAccountAdmin(admin.ModelAdmin):
     def get_list_display(self, request):
         return ['user_id', 'pin', 'subscription_level', 'timezone'] + list(super().get_list_display(request))
@@ -108,5 +117,6 @@ class UserAdmin(BaseUserAdmin):
 admin.site.register(RevenueCatWebhookEvent, RevenueCatWebhookEventAdmin)
 admin.site.register(Deck, DeckAdmin)
 admin.site.register(Flashcard, FlashcardAdmin)
+admin.site.register(SafetyEvent, SafetyEventAdmin)
 admin.site.unregister(User)
 admin.site.register(User, UserAdmin)

back/bots/management/__init__.py

@@ -0,0 +1,95 @@
+"""Seed idempotent e2e data for the server-side safety demo (roadmap 03).
+
+Creates the shared Detox login ('e2e-test-user' / 'testpassword123') plus:
+- a profile for the AsyncStorage injection,
+- "Safety Demo Bot": custom advanced-editor prompt with restrict flags ON —
+  proves the server regenerates the policy suffix even when the parent wrote
+  a custom system prompt (safe path: homework questions answer normally;
+  unsafe path: adult-topic messages get the fixed refusal),
+- "Open Flags Bot": restrict flags OFF — proves the global floor still
+  applies (crisis terms still blocked).
+
+The command resets a password that is published in this repository, so it
+refuses to run unless an E2E environment is explicitly enabled. Accidentally
+running it against staging or production would otherwise leave a trivially
+accessible user.
+
+Usage: E2E_SEEDING=1 python manage.py seed_e2e_server_safety
+"""
+
+import os
+
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand, CommandError
+
+from bots.models import Bot, Profile
+
+E2E_USERNAME = 'e2e-test-user'
+E2E_PASSWORD = 'testpassword123'
+
+SAFETY_DEMO_BOT_NAME = 'Safety Demo Bot'
+OPEN_FLAGS_BOT_NAME = 'Open Flags Bot'
+
+
+class Command(BaseCommand):
+    help = 'Seed idempotent e2e data for the server-side safety demos.'
+
+    def handle(self, *args, **options):
+        if os.environ.get('E2E_SEEDING') != '1':
+            raise CommandError(
+                'Refusing to create a known-password e2e user outside an E2E '
+                'environment. Re-run with E2E_SEEDING=1 to enable e2e seeding.'
+            )
+        user, created = User.objects.get_or_create(username=E2E_USERNAME)
+        if created or not user.check_password(E2E_PASSWORD):
+            user.set_password(E2E_PASSWORD)
+            user.save()
+        self.stdout.write(f"user: {user.username} ({'created' if created else 'exists'})")
+
+        profile, _ = Profile.objects.get_or_create(
+            user=user,
+            deleted_at=None,
+            defaults={'name': 'E2E Kid'},
+        )
+        if not profile.name:
+            # The post_save signal may have created it with a blank first name.
+            profile.name = 'E2E Kid'
+            profile.save()
+        self.stdout.write(f"profile: {profile.name} ({profile.profile_id})")
+
+        safety_bot, created = Bot.objects.update_or_create(
+            user=user,
+            name=SAFETY_DEMO_BOT_NAME,
+            defaults={
+                'system_prompt': (
+                    'You are a patient math tutor for a 14-year-old. Use '
+                    'Socratic questioning. Never just give the final answer.'
+                ),
+                'simple_editor': False,
+                'template_name': '',
+                'response_length': 150,
+                'restrict_language': True,
+                'restrict_adult_topics': True,
+                'enable_web_search': False,
+                'deleted_at': None,
+            },
+        )
+        self.stdout.write(f"bot: {safety_bot.name} ({'created' if created else 'updated'})")
+
+        open_bot, created = Bot.objects.update_or_create(
+            user=user,
+            name=OPEN_FLAGS_BOT_NAME,
+            defaults={
+                'system_prompt': 'You are a relaxed study buddy. All bot-level restrictions are off.',
+                'simple_editor': False,
+                'template_name': '',
+                'response_length': 200,
+                'restrict_language': False,
+                'restrict_adult_topics': False,
+                'enable_web_search': False,
+                'deleted_at': None,
+            },
+        )
+        self.stdout.write(f"bot: {open_bot.name} ({'created' if created else 'updated'})")
+
+        self.stdout.write(self.style.SUCCESS('e2e server-safety seed complete'))

back/bots/management/commands/__init__.py

@@ -0,0 +1,33 @@
+# Generated by Django 6.1 on 2026-08-25 18:14
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bots', '0038_alter_aimodel_options_alter_device_options'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SafetyEvent',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('event_id', models.UUIDField(default=uuid.uuid4, unique=True)),
+                ('stage', models.CharField(max_length=32)),
+                ('reason_code', models.CharField(max_length=64)),
+                ('snippet_redacted', models.CharField(blank=True, max_length=200)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('bot', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='bots.bot')),
+                ('chat', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='bots.chat')),
+                ('profile', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='bots.profile')),
+                ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

back/bots/management/commands/seed_e2e_server_safety.py

@@ -0,0 +1,16 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bots', '0039_safetyevent'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='message',
+            name='safety_blocked',
+            field=models.BooleanField(default=False),
+        ),
+    ]

back/bots/migrations/0039_safetyevent.py

@@ -6,6 +6,7 @@
 from .flashcard import Flashcard
 from .message import Message
 from .profile import Profile
+from .safety_event import SafetyEvent
 from .usage_limit_hit import UsageLimitHit
 from .user_account import RevenueCatWebhookEvent, UserAccount
 
@@ -19,6 +20,7 @@
     'Message',
     'Profile',
     'RevenueCatWebhookEvent',
+    'SafetyEvent',
     'UsageLimitHit',
     'UserAccount',
 ]

back/bots/migrations/0040_message_safety_blocked.py

@@ -4,11 +4,18 @@
 
 import boto3
 from django.conf import settings
-from django.db import models
+from django.db import models, transaction
 from langchain_aws import ChatBedrock
 from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
 
 from bots.services.chat_agent import ChatAgentService
+from bots.services.safety import (
+    SafetyPolicy,
+    build_system_prompt,
+    evaluate_text,
+    record_safety_event,
+    refusal_for_verdict,
+)
 
 from .ai_model import AiModel
 from .bot import Bot
@@ -60,37 +67,85 @@ def use_default_model(self, ai=None):
 
         self.ai = AiClientWrapper(model_id=default_model.model_id, client=ai)
 
-    def get_response(self, ai=None):
+    def get_response(self, ai=None, user_message=None):
+        # Input safety is evaluated BEFORE any model setup or quota check so
+        # a misconfigured model or an over-limit account cannot swallow the
+        # fixed crisis refusal or leave the unsafe message unmarked.
+        policy = SafetyPolicy.for_bot(self.bot)
+        subject = user_message or self.messages.filter(role='user').order_by('-id').first()
+        if subject is not None:
+            verdict = evaluate_text(subject.text, policy, source='INPUT')
+            if verdict.blocked:
+                # Short transaction only for the state change; external calls
+                # (moderation, Bedrock, Tavily) are never made while a row
+                # lock is held, so the DB connection is not held for tens of
+                # seconds. The message is marked so later turns exclude it via
+                # get_input().
+                with transaction.atomic():
+                    Chat.objects.select_for_update().get(pk=self.pk)
+                    if not subject.safety_blocked:
+                        subject.safety_blocked = True
+                        subject.save(update_fields=['safety_blocked', 'modified_at'])
+                    refusal = refusal_for_verdict(verdict)
+                    self.messages.create(
+                        text=refusal,
+                        role='assistant',
+                        order=self.messages.count(),
+                    )
+                    record_safety_event(
+                        stage='input',
+                        verdict=verdict,
+                        chat=self,
+                        snippet=subject.text,
+                    )
+                return refusal
+
+        if self.user.user_account.over_limit():
+            return "You have exceeded your daily limit. Please try again tomorrow or upgrade your subscription."
+
+        # AI client is instantiated only after input has passed the global
+        # floor, so a missing default model never blocks the crisis path.
         if self.bot and self.bot.ai_model:
             self.ai = AiClientWrapper(model_id=self.bot.ai_model.model_id, client=ai)
         else:
             self.use_default_model(ai)
-        
+
+        # Context is built AFTER the blocked check so any safety-blocked
+        # message (including this turn's) is excluded from model history.
+        # This work is done outside any DB transaction.
         message_list, contains_image = self.get_input()
 
         if contains_image and self.bot and self.bot.ai_model and 'image' not in self.bot.ai_model.supported_input_modalities:
             self.use_default_model(ai)
-        
-        if self.user.user_account.over_limit():
-            return "You have exceeded your daily limit. Please try again tomorrow or upgrade your subscription."
-        
-        response_text, usage_metadata = ChatAgentService(self, self.ai.client).respond(message_list)
-        
-        message_order = self.messages.count()
-        
-        input_tokens = usage_metadata.get('input_tokens', 0)
-        output_tokens = usage_metadata.get('output_tokens', 0)
-        
-        self.messages.create(
-            text=response_text, 
-            role='assistant', 
-            order=message_order,
-            input_tokens=input_tokens,
-            output_tokens=output_tokens
-        )
-        self.input_tokens += input_tokens
-        self.output_tokens += output_tokens
-        self.save()
+
+        response_text, usage_metadata = ChatAgentService(self, self.ai.client, policy=policy).respond(message_list)
+
+        # Post-model output filter: replace flagged completions before save.
+        output_verdict = evaluate_text(response_text, policy, source='OUTPUT')
+        flagged_output = None
+        if output_verdict.blocked:
+            flagged_output = response_text
+            response_text = refusal_for_verdict(output_verdict)
+
+        # Short transaction only for the final persist; the row lock is held
+        # briefly to claim the message order, not across the model call.
+        with transaction.atomic():
+            Chat.objects.select_for_update().get(pk=self.pk)
+            message_order = self.messages.count()
+            input_tokens = usage_metadata.get('input_tokens', 0)
+            output_tokens = usage_metadata.get('output_tokens', 0)
+            self.messages.create(
+                text=response_text,
+                role='assistant',
+                order=message_order,
+                input_tokens=input_tokens,
+                output_tokens=output_tokens,
+            )
+            self.input_tokens += input_tokens
+            self.output_tokens += output_tokens
+            self.save()
+            if output_verdict.blocked:
+                record_safety_event(stage='output', verdict=output_verdict, chat=self, snippet=flagged_output)
         return response_text
 
     def setup_human_message_content(self, message):
@@ -111,7 +166,13 @@ def has_image(self, message: HumanMessage):
 
     def get_input(self):
         contains_image = False
-        messages = self.messages.exclude(role='system').order_by('-id')[:10]
+        # Safety-blocked messages are excluded: denied content never becomes
+        # later model context even after the turn ends.
+        messages = (
+            self.messages.exclude(role='system')
+            .exclude(safety_blocked=True)
+            .order_by('-id')[:10]
+        )
         messages = sorted(messages, key=lambda message: message.id)
         message_list = []
 
@@ -131,9 +192,16 @@ def get_input(self):
         return message_list, contains_image
 
     def get_system_message(self):
-        if self.bot and self.bot.system_prompt:
-            return self.bot.system_prompt
-        return "You are chatting with a teen. Please keep the conversation appropriate and respectful. Your responses should be 200 words or less."
+        """Server-owned layered prompt: preamble + parent customization + policy suffix.
+
+        The flags are restated here every turn so a custom (advanced-editor)
+        system_prompt cannot strip the safety layers, and the client is never
+        the control plane for policy text.
+        """
+        policy = SafetyPolicy.for_bot(self.bot)
+        bot_prompt = self.bot.system_prompt if self.bot else None
+        response_length = self.bot.response_length if self.bot else None
+        return build_system_prompt(bot_prompt, policy, response_length)
 
     def get_image_data(self, filename):
         try:

back/bots/models/__init__.py

@@ -16,6 +16,10 @@ class Message(models.Model):
     created_at = models.DateTimeField(auto_now_add=True)
     modified_at = models.DateTimeField(auto_now=True)
     image_filename = models.CharField(max_length=255, blank=True, null=True)
+    # True when the server-side safety filter denied this message. Denied
+    # content is still visible in the chat history but must never re-enter
+    # the model context on later turns (see Chat.get_input()).
+    safety_blocked = models.BooleanField(default=False)
 
     def __str__(self):
         user_str = getattr(self.chat.user, 'email', 'unknown')

back/bots/models/chat.py

@@ -0,0 +1,28 @@
+import uuid
+
+from django.conf import settings
+from django.db import models
+
+from .bot import Bot
+from .chat import Chat
+from .profile import Profile
+
+
+class SafetyEvent(models.Model):
+    """Audit log of safety blocks. Admin-readable; parent API lands in 04."""
+
+    event_id = models.UUIDField(default=uuid.uuid4, unique=True)
+    user = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, on_delete=models.CASCADE)
+    profile = models.ForeignKey(Profile, null=True, on_delete=models.SET_NULL)
+    chat = models.ForeignKey(Chat, null=True, on_delete=models.SET_NULL)
+    bot = models.ForeignKey(Bot, null=True, on_delete=models.SET_NULL)
+    # input|output|web_query|web_result|tool_flashcard
+    stage = models.CharField(max_length=32)
+    # adult_topic|language|global_floor|web_blocked
+    reason_code = models.CharField(max_length=64)
+    # never store the full raw text when it matched a sexual/violent term
+    snippet_redacted = models.CharField(max_length=200, blank=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    def __str__(self):
+        return f"{self.created_at:%Y-%m-%d %H:%M} {self.stage}/{self.reason_code}"