From 62f7b5f8ff4f7b6bfd11c3cd8e1d5c1f953340c7 Mon Sep 17 00:00:00 2001 From: Shrushti-Sakat Date: Thu, 3 Sep 2026 17:59:20 +0530 Subject: [PATCH 1/2] Add Task 3: Customer Simulator Agent New standalone module (models, persona/scenario config, state engine, LLM turn generation, API endpoints, log export, demo script, 16 pytest tests). Task 2 (RAG pipeline, chat, documents, support) untouched except a 2-line router registration in main.py. --- RAG-Pipeline-backend/TASK3_DELIVERABLES.md | 41 ++ RAG-Pipeline-backend/app/api/simulator.py | 393 ++++++++++++++ RAG-Pipeline-backend/app/main.py | 2 + RAG-Pipeline-backend/app/models/simulator.py | 173 +++++++ .../app/services/persona_service.py | 113 ++++ .../app/services/scenario_service.py | 115 +++++ .../app/services/simulator_service.py | 197 +++++++ .../app/services/simulator_state.py | 214 ++++++++ .../scripts/create_simulator_table.py | 9 + .../scripts/demo_simulator_conversations.py | 292 +++++++++++ .../scripts/export_simulator_logs.py | 142 +++++ RAG-Pipeline-backend/tests/test_simulator.py | 483 ++++++++++++++++++ 12 files changed, 2174 insertions(+) create mode 100644 RAG-Pipeline-backend/TASK3_DELIVERABLES.md create mode 100644 RAG-Pipeline-backend/app/api/simulator.py create mode 100644 RAG-Pipeline-backend/app/models/simulator.py create mode 100644 RAG-Pipeline-backend/app/services/persona_service.py create mode 100644 RAG-Pipeline-backend/app/services/scenario_service.py create mode 100644 RAG-Pipeline-backend/app/services/simulator_service.py create mode 100644 RAG-Pipeline-backend/app/services/simulator_state.py create mode 100644 RAG-Pipeline-backend/scripts/create_simulator_table.py create mode 100644 RAG-Pipeline-backend/scripts/demo_simulator_conversations.py create mode 100644 RAG-Pipeline-backend/scripts/export_simulator_logs.py create mode 100644 RAG-Pipeline-backend/tests/test_simulator.py diff --git a/RAG-Pipeline-backend/TASK3_DELIVERABLES.md b/RAG-Pipeline-backend/TASK3_DELIVERABLES.md new file mode 100644 index 0000000..0378c96 --- /dev/null +++ b/RAG-Pipeline-backend/TASK3_DELIVERABLES.md @@ -0,0 +1,41 @@ +# TASK 3 Deliverables — Customer Simulator Agent + +This document maps the **Expected Output** requirements from `TASK3.docx` to the specific implementation files, functions, logs, and automated tests. + +--- + +## Deliverables Mapping + +| Expected Output Requirement | Satisfying Component / File / Command | +| :--- | :--- | +| **Customer Simulator Agent** | [`app/services/simulator_service.py`](file:///c:/Users/shrushti/Customer-Support-Assistant/RAG-Pipeline-backend/app/services/simulator_service.py) | +| **Persona and Scenario Configuration** | [`app/services/persona_service.py`](file:///c:/Users/shrushti/Customer-Support-Assistant/RAG-Pipeline-backend/app/services/persona_service.py) (6 personas) & [`app/services/scenario_service.py`](file:///c:/Users/shrushti/Customer-Support-Assistant/RAG-Pipeline-backend/app/services/scenario_service.py) (5 scenarios) | +| **Emotion / State Management** | [`app/services/simulator_state.py`](file:///c:/Users/shrushti/Customer-Support-Assistant/RAG-Pipeline-backend/app/services/simulator_state.py) (`initial_state`, `update_state`, `is_resolved`, `is_escalated`) | +| **Turn-by-Turn Response Generation** | `generate_customer_turn()` in [`app/services/simulator_service.py`](file:///c:/Users/shrushti/Customer-Support-Assistant/RAG-Pipeline-backend/app/services/simulator_service.py) | +| **API Integration** | [`app/api/simulator.py`](file:///c:/Users/shrushti/Customer-Support-Assistant/RAG-Pipeline-backend/app/api/simulator.py) (`POST /simulator/start`, `POST /simulator/message`, `GET /simulator/{session_id}/history`) | +| **Sample Conversation Logs** | [`logs/simulator/*.json`](file:///c:/Users/shrushti/Customer-Support-Assistant/RAG-Pipeline-backend/logs/simulator), generated via [`scripts/demo_simulator_conversations.py`](file:///c:/Users/shrushti/Customer-Support-Assistant/RAG-Pipeline-backend/scripts/demo_simulator_conversations.py) | +| **Test Cases Covering Customer Behaviors** | [`tests/test_simulator.py`](file:///c:/Users/shrushti/Customer-Support-Assistant/RAG-Pipeline-backend/tests/test_simulator.py) (16 automated tests via `pytest tests/test_simulator.py -v`) | + +--- + +## How to Run + +Execute the following commands from the `RAG-Pipeline-backend/` directory: + +### 1. Initialize Database Tables +Creates the simulator tables (`scenarios`, `sessions`, `conversations`, `messages`) in `app.db`: +```bash +python scripts/create_simulator_table.py +``` + +### 2. Run Demo Conversations & Generate Logs +Executes 5 end-to-end multi-turn conversations across all 5 scenarios and distinct personas, and exports the JSON logs into `logs/simulator/`: +```bash +python scripts/demo_simulator_conversations.py +``` + +### 3. Run the Automated Test Suite +Runs all 16 isolated unit and integration tests covering the state machine, persona/scenario libraries, LLM fallback handling, and FastAPI endpoints: +```bash +pytest tests/test_simulator.py -v +``` diff --git a/RAG-Pipeline-backend/app/api/simulator.py b/RAG-Pipeline-backend/app/api/simulator.py new file mode 100644 index 0000000..244898c --- /dev/null +++ b/RAG-Pipeline-backend/app/api/simulator.py @@ -0,0 +1,393 @@ +import json +from datetime import datetime +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel +from sqlalchemy.orm import Session as DBSession + +from app.models.database import SessionLocal +from app.models.simulator import Scenario, Session, Conversation, Message +from app.services.simulator_service import generate_customer_turn +from app.services.simulator_state import initial_state +from app.services.scenario_service import SCENARIOS, get_scenario_brief +from app.services.persona_service import get_persona_brief + + +router = APIRouter( + prefix="/simulator", + tags=["Customer Simulator"] +) + + +# -------------------------------------------------- +# Database dependency +# -------------------------------------------------- + +def get_db(): + + db = SessionLocal() + + try: + yield db + + finally: + db.close() + + +# -------------------------------------------------- +# Request models +# -------------------------------------------------- + +class SimulatorStartRequest(BaseModel): + + session_label: str + + persona: str + + scenario: str + + initial_emotion: str + + issue_severity: int + + patience_level: int + + expected_resolution: str + + +class SimulatorMessageRequest(BaseModel): + + session_id: int + + agent_response: str + + +# -------------------------------------------------- +# Endpoint 1: Start Simulation +# -------------------------------------------------- + +@router.post("/start") +def start_simulator_session( + + request: SimulatorStartRequest, + + db: DBSession = Depends(get_db) + +): + # Validate scenario + try: + get_scenario_brief(request.scenario) + except ValueError as e: + raise HTTPException( + status_code=400, + detail=str(e) + ) + + # Validate persona + try: + get_persona_brief(request.persona) + except ValueError as e: + raise HTTPException( + status_code=400, + detail=str(e) + ) + + scenario_key = request.scenario.strip().lower() + scenario_data = SCENARIOS[scenario_key] + + # Create Scenario row + scenario_row = Scenario( + title=request.session_label or f"Scenario - {scenario_key.title()}", + category=scenario_key, + difficulty="Medium", + objective=request.expected_resolution or scenario_data.get("resolution_condition"), + description=scenario_data.get("opening_complaint"), + is_active=True + ) + db.add(scenario_row) + db.flush() + + # Create Session row + session_row = Session( + scenario_id=scenario_row.scenario_id, + start_time=datetime.utcnow(), + status="In Progress" + ) + db.add(session_row) + db.flush() + + # Create Conversation row + conversation_row = Conversation( + session_id=session_row.session_id, + intent=scenario_key, + sentiment=request.initial_emotion, + resolution_status="Unresolved", + escalation_risk="Low", + created_at=datetime.utcnow() + ) + db.add(conversation_row) + db.flush() + + # Build initial state + start_state = initial_state( + persona=request.persona, + initial_emotion=request.initial_emotion, + issue_severity=request.issue_severity, + patience_level=request.patience_level + ) + + opening_message = scenario_data["opening_complaint"] + + # Customer's initial opening message + customer_msg = Message( + conversation_id=conversation_row.conversation_id, + sender_type="Customer", + message_text=opening_message, + timestamp=datetime.utcnow(), + message_type="Text" + ) + db.add(customer_msg) + + # System state message to track current state without schema alterations + system_state_msg = Message( + conversation_id=conversation_row.conversation_id, + sender_type="AI", + message_text=json.dumps({ + "persona": request.persona, + "scenario": scenario_key, + "state": start_state + }), + timestamp=datetime.utcnow(), + message_type="System" + ) + db.add(system_state_msg) + + db.commit() + + return { + "session_id": session_row.session_id, + "conversation_id": conversation_row.conversation_id, + "customer_message": opening_message, + "state": start_state, + "turn": 1 + } + + +# -------------------------------------------------- +# Endpoint 2: Next Customer Turn +# -------------------------------------------------- + +@router.post("/message") +def send_simulator_message( + + request: SimulatorMessageRequest, + + db: DBSession = Depends(get_db) + +): + # Lookup Session + session_row = ( + db.query(Session) + .filter(Session.session_id == request.session_id) + .first() + ) + + if not session_row: + raise HTTPException( + status_code=404, + detail="Simulator session not found" + ) + + # Lookup Conversation + conversation_row = ( + db.query(Conversation) + .filter(Conversation.session_id == session_row.session_id) + .first() + ) + + if not conversation_row: + raise HTTPException( + status_code=404, + detail="Conversation not found for session" + ) + + scenario_row = ( + db.query(Scenario) + .filter(Scenario.scenario_id == session_row.scenario_id) + .first() + ) + + # Fetch ordered messages + all_messages = ( + db.query(Message) + .filter(Message.conversation_id == conversation_row.conversation_id) + .order_by(Message.message_id.asc()) + .all() + ) + + # Reconstruct current state, persona, and scenario from latest System message + current_state = None + persona = "calm" + scenario_key = scenario_row.category if scenario_row else "refund" + + for m in reversed(all_messages): + if m.message_type == "System": + try: + payload = json.loads(m.message_text) + current_state = payload.get("state") + persona = payload.get("persona", persona) + scenario_key = payload.get("scenario", scenario_key) + break + except Exception: + pass + + if not current_state: + current_state = initial_state(persona, "neutral", 3, 3) + + # Filter dialogue history for prompt + dialogue_history = [ + { + "sender_type": m.sender_type, + "message_text": m.message_text + } + for m in all_messages + if m.message_type != "System" + ] + + # Persist agent's response + agent_msg = Message( + conversation_id=conversation_row.conversation_id, + sender_type="Support Agent", + message_text=request.agent_response, + timestamp=datetime.utcnow(), + message_type="Text" + ) + db.add(agent_msg) + db.flush() + + # Generate customer turn + turn_result = generate_customer_turn( + persona=persona, + scenario=scenario_key, + state=current_state, + conversation_history=dialogue_history, + agent_response=request.agent_response + ) + + customer_message = turn_result["customer_message"] + updated_state = turn_result["updated_state"] + is_res = turn_result["is_resolved"] + is_esc = turn_result["is_escalated"] + + # Persist customer message + customer_msg_row = Message( + conversation_id=conversation_row.conversation_id, + sender_type="Customer", + message_text=customer_message, + timestamp=datetime.utcnow(), + message_type="Text" + ) + db.add(customer_msg_row) + + # Persist updated state in a System message row + system_state_row = Message( + conversation_id=conversation_row.conversation_id, + sender_type="AI", + message_text=json.dumps({ + "persona": persona, + "scenario": scenario_key, + "state": updated_state + }), + timestamp=datetime.utcnow(), + message_type="System" + ) + db.add(system_state_row) + + # Update session and conversation status if resolved or escalated + if is_res: + session_row.status = "Completed" + session_row.end_time = datetime.utcnow() + conversation_row.resolution_status = "Resolved" + elif is_esc: + session_row.status = "Completed" + session_row.end_time = datetime.utcnow() + conversation_row.escalation_risk = "High" + + # Calculate turn count + customer_turns = sum( + 1 for m in dialogue_history if m["sender_type"] == "Customer" + ) + 1 + + db.commit() + + return { + "session_id": session_row.session_id, + "customer_message": customer_message, + "state": updated_state, + "turn": customer_turns, + "is_resolved": is_res, + "is_escalated": is_esc + } + + +# -------------------------------------------------- +# Endpoint 3: History +# -------------------------------------------------- + +@router.get("/{session_id}/history") +def get_simulator_history( + + session_id: int, + + db: DBSession = Depends(get_db) + +): + session_row = ( + db.query(Session) + .filter(Session.session_id == session_id) + .first() + ) + + if not session_row: + raise HTTPException( + status_code=404, + detail="Simulator session not found" + ) + + conversation_row = ( + db.query(Conversation) + .filter(Conversation.session_id == session_id) + .first() + ) + + if not conversation_row: + return { + "session_id": session_id, + "status": session_row.status, + "messages": [] + } + + # Retrieve only dialogue messages (excluding internal System state rows) + messages = ( + db.query(Message) + .filter( + Message.conversation_id == conversation_row.conversation_id, + Message.message_type != "System" + ) + .order_by(Message.message_id.asc()) + .all() + ) + + return { + "session_id": session_id, + "status": session_row.status, + "messages": [ + { + "message_id": message.message_id, + "sender_type": message.sender_type, + "message_text": message.message_text, + "message_type": message.message_type, + "timestamp": message.timestamp + } + for message in messages + ] + } diff --git a/RAG-Pipeline-backend/app/main.py b/RAG-Pipeline-backend/app/main.py index d9ee203..c30a300 100644 --- a/RAG-Pipeline-backend/app/main.py +++ b/RAG-Pipeline-backend/app/main.py @@ -6,6 +6,7 @@ from app.api.document_management import router as document_management_router from app.api.chat import router as chat_router from app.api.support import router as support_router +from app.api.simulator import router as simulator_router app = FastAPI( @@ -24,6 +25,7 @@ app.include_router(document_management_router) app.include_router(chat_router) app.include_router(support_router) +app.include_router(simulator_router) @app.get("/") diff --git a/RAG-Pipeline-backend/app/models/simulator.py b/RAG-Pipeline-backend/app/models/simulator.py new file mode 100644 index 0000000..9840b20 --- /dev/null +++ b/RAG-Pipeline-backend/app/models/simulator.py @@ -0,0 +1,173 @@ +from datetime import datetime +from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean, Float, ForeignKey + +from app.models.database import Base + + +class Scenario(Base): + + __tablename__ = "scenarios" + + scenario_id = Column( + Integer, + primary_key=True, + index=True + ) + + title = Column( + String, + nullable=False + ) + + category = Column( + String, + nullable=False + ) + + difficulty = Column( + String, + nullable=False + ) + + objective = Column( + Text, + nullable=True + ) + + description = Column( + Text, + nullable=True + ) + + is_active = Column( + Boolean, + default=True, + nullable=False + ) + + +class Session(Base): + + __tablename__ = "sessions" + + session_id = Column( + Integer, + primary_key=True, + index=True + ) + + agent_id = Column( + Integer, + nullable=True + ) + + scenario_id = Column( + Integer, + ForeignKey("scenarios.scenario_id"), + nullable=False + ) + + start_time = Column( + DateTime, + default=datetime.utcnow, + nullable=False + ) + + end_time = Column( + DateTime, + nullable=True + ) + + overall_score = Column( + Float, + nullable=True + ) + + status = Column( + String, + default="In Progress", + nullable=False + ) + + +class Conversation(Base): + + __tablename__ = "conversations" + + conversation_id = Column( + Integer, + primary_key=True, + index=True + ) + + session_id = Column( + Integer, + ForeignKey("sessions.session_id"), + unique=True, + nullable=False + ) + + intent = Column( + String, + nullable=True + ) + + sentiment = Column( + String, + nullable=True + ) + + resolution_status = Column( + String, + nullable=True + ) + + escalation_risk = Column( + String, + nullable=True + ) + + created_at = Column( + DateTime, + default=datetime.utcnow, + nullable=False + ) + + +class Message(Base): + + __tablename__ = "messages" + + message_id = Column( + Integer, + primary_key=True, + index=True + ) + + conversation_id = Column( + Integer, + ForeignKey("conversations.conversation_id"), + nullable=False + ) + + sender_type = Column( + String, + nullable=False + ) + + message_text = Column( + Text, + nullable=False + ) + + timestamp = Column( + DateTime, + default=datetime.utcnow, + nullable=False + ) + + message_type = Column( + String, + default="Text", + nullable=False + ) diff --git a/RAG-Pipeline-backend/app/services/persona_service.py b/RAG-Pipeline-backend/app/services/persona_service.py new file mode 100644 index 0000000..f125fdc --- /dev/null +++ b/RAG-Pipeline-backend/app/services/persona_service.py @@ -0,0 +1,113 @@ +"""Persona service providing customer personality profiles and tone configurations.""" + +PERSONAS = { + "calm": { + "tone": ( + "Speaks in measured, complete, and articulate sentences. " + "Uses neutral, balanced vocabulary, polite punctuation, and avoids exclamation " + "marks or emotional language. Remains objective and cooperative." + ), + "escalation_tendency": "low", + "sample_phrases": [ + "I would appreciate some clarification on this recent charge.", + "Let's see what steps we need to take to sort this out.", + "I understand these things happen, please let me know the next step." + ] + }, + "confused": { + "tone": ( + "Uses hesitant, questioning sentences with question marks and ellipses. " + "May repeat misunderstandings or mix up terminology. Expresses uncertainty " + "about instructions and frequently asks for reassurance." + ), + "escalation_tendency": "medium", + "sample_phrases": [ + "Wait, I don't understand... was I supposed to get an email confirmation?", + "I'm really not sure where to look for that code you mentioned.", + "Sorry, does this mean my order didn't go through, or is it already shipped?" + ] + }, + "frustrated": { + "tone": ( + "Speaks with an exasperated and tense tone. Uses pointed rhetorical questions, " + "dashes, and sighs. Frequently mentions wasted time, previous failed attempts, " + "or unmet expectations." + ), + "escalation_tendency": "medium", + "sample_phrases": [ + "I've already explained this twice to previous agents.", + "This is taking way longer than it should, and I really don't have time for this.", + "Why is it so difficult to get a straight answer regarding my account?" + ] + }, + "angry": { + "tone": ( + "Speaks in aggressive, short, clipped sentences with sharp language, exclamation " + "marks, and occasional capitalized words. Demands immediate actions, expresses " + "indignation, and threatens escalation or negative reviews." + ), + "escalation_tendency": "high", + "sample_phrases": [ + "This is completely unacceptable! Fix this right NOW.", + "I am NOT paying for a mistake YOUR system made.", + "Get me your manager immediately if you can't resolve this." + ] + }, + "impatient": { + "tone": ( + "Speaks in rushed, brief, and concise sentences. Uses prompt words like 'quickly', " + "'ASAP', and 'hurry'. Cuts straight to the point, dislikes long explanations or small " + "talk, and focuses strictly on resolution time." + ), + "escalation_tendency": "high", + "sample_phrases": [ + "I need this resolved ASAP, I have a meeting in 5 minutes.", + "Can we skip the pleasantries and just get to the point?", + "Just tell me how long this is going to take." + ] + }, + "polite": { + "tone": ( + "Speaks in warm, courteous, and respectful sentences. Uses pleasantries ('please', " + "'thank you', 'I hope you are having a nice day'), soft modal verbs ('could you', " + "'would it be possible'), and positive phrasing even when raising an issue." + ), + "escalation_tendency": "low", + "sample_phrases": [ + "Hello, thank you for your time today. Could you please help me check on this?", + "I would really appreciate any assistance you could provide with my account.", + "Thank you so much for looking into this for me, I really appreciate your patience." + ] + } +} + + +def get_persona_brief(persona_name: str) -> str: + """Returns a formatted plain-text paragraph describing the customer persona. + + Args: + persona_name: The name of the persona (calm, confused, frustrated, angry, + impatient, polite). + + Returns: + Formatted string ready for inclusion in LLM prompts. + + Raises: + ValueError: If persona_name is not one of the supported personas. + """ + key = persona_name.strip().lower() + if key not in PERSONAS: + valid_personas = ", ".join(sorted(PERSONAS.keys())) + raise ValueError( + f"Invalid persona '{persona_name}'. Must be one of: {valid_personas}" + ) + + persona = PERSONAS[key] + sample_phrases_text = " | ".join(f'"{phrase}"' for phrase in persona["sample_phrases"]) + + return ( + f"Customer Persona: {key.capitalize()}\n" + f"Tone: {persona['tone']}\n" + f"Escalation Tendency: {persona['escalation_tendency'].capitalize()}\n" + f"Sample Phrases: {sample_phrases_text}" + ) diff --git a/RAG-Pipeline-backend/app/services/scenario_service.py b/RAG-Pipeline-backend/app/services/scenario_service.py new file mode 100644 index 0000000..24d1fc9 --- /dev/null +++ b/RAG-Pipeline-backend/app/services/scenario_service.py @@ -0,0 +1,115 @@ +"""Scenario service providing support scenario templates and resolution conditions.""" + +SCENARIOS = { + "refund": { + "opening_complaint": ( + "Hi, I was charged $49.99 for a subscription renewal that I requested to cancel " + "last week. I need a full refund issued back to my card immediately." + ), + "key_facts": [ + "Order/Invoice ID: #INV-49201", + "Charged Amount: $49.99 on original payment card", + "Cancellation Request Date: 5 days prior to billing cycle renewal", + "Eligibility Window: Within 30-day refund grace policy" + ], + "resolution_condition": ( + "The agent confirms the customer's eligibility, initiates the full refund " + "transaction of $49.99, and clearly explains the 3-5 business day processing timeline." + ) + }, + "delayed_order": { + "opening_complaint": ( + "My order #ORD-78219 was scheduled for guaranteed delivery 3 days ago, but the " + "tracking hasn't updated and the package still hasn't arrived. Where is my item?" + ), + "key_facts": [ + "Order Number: #ORD-78219", + "Promised Delivery Window: 3 days past delivery SLA", + "Courier Status: Stuck in transit / carrier exception", + "Item Value: $120.00" + ], + "resolution_condition": ( + "The agent acknowledges the shipping delay, checks tracking status, offers either " + "an expedited replacement or shipping fee waiver/credit, and provides clear tracking updates." + ) + }, + "payment_failure": { + "opening_complaint": ( + "I'm trying to upgrade my team's subscription plan, but my credit card keeps getting " + "declined with error code ERR_PAYMENT_FAILED_04 even though my funds are sufficient." + ), + "key_facts": [ + "Error Code: ERR_PAYMENT_FAILED_04 (3D-Secure authentication timeout)", + "Target Plan: Annual Pro Tier ($299/yr)", + "Card Type: Visa ending in 4242", + "Failed Attempts: 3 consecutive transaction attempts" + ], + "resolution_condition": ( + "The agent identifies the payment gateway verification issue, guides the customer " + "through the alternate payment link or 3DS verification steps, and confirms successful transaction completion." + ) + }, + "account_issue": { + "opening_complaint": ( + "I've been locked out of my corporate account after losing access to my two-factor " + "authentication device, and I need access restored urgently." + ), + "key_facts": [ + "Account Email: user@company.com", + "Lockout Reason: Lost 2FA authenticator app on device change", + "Last Successful Login: 2 days ago", + "Verification Option: Registered backup security email on file" + ], + "resolution_condition": ( + "The agent follows security verification protocols, verifies the customer's identity " + "via the registered backup email magic link or one-time code, and safely restores account access." + ) + }, + "cancellation": { + "opening_complaint": ( + "I would like to cancel my current monthly subscription immediately and ensure " + "auto-renewal is turned off so I am not billed again." + ), + "key_facts": [ + "Subscription Tier: Monthly Business Plan ($29/mo)", + "Account ID: #ACC-88310", + "Next Billing Date: In 4 days", + "Usage Status: Active for 6 months" + ], + "resolution_condition": ( + "The agent acknowledges the cancellation request, informs the customer of retention " + "options or billing period end date without undue pressure, confirms cancellation, and ensures no future charges." + ) + } +} + + +def get_scenario_brief(scenario_type: str) -> str: + """Returns a formatted plain-text paragraph describing the support scenario. + + Args: + scenario_type: The type of scenario (refund, delayed_order, payment_failure, + account_issue, cancellation). + + Returns: + Formatted string ready for inclusion in LLM prompts. + + Raises: + ValueError: If scenario_type is not one of the supported scenarios. + """ + key = scenario_type.strip().lower() + if key not in SCENARIOS: + valid_scenarios = ", ".join(sorted(SCENARIOS.keys())) + raise ValueError( + f"Invalid scenario type '{scenario_type}'. Must be one of: {valid_scenarios}" + ) + + scenario = SCENARIOS[key] + key_facts_text = "\n".join(f"- {fact}" for fact in scenario["key_facts"]) + + return ( + f"Scenario Type: {key.replace('_', ' ').title()}\n" + f"Opening Complaint: \"{scenario['opening_complaint']}\"\n" + f"Key Facts:\n{key_facts_text}\n" + f"Resolution Condition: {scenario['resolution_condition']}" + ) diff --git a/RAG-Pipeline-backend/app/services/simulator_service.py b/RAG-Pipeline-backend/app/services/simulator_service.py new file mode 100644 index 0000000..8663cad --- /dev/null +++ b/RAG-Pipeline-backend/app/services/simulator_service.py @@ -0,0 +1,197 @@ +"""Simulator service orchestrating customer simulation turns, prompt assembly, and state updates.""" + +import re + +from app.services.rag_service import generate_with_gemini +from app.services.persona_service import get_persona_brief, PERSONAS +from app.services.scenario_service import get_scenario_brief +from app.services.simulator_state import ( + initial_state, + update_state, + is_resolved, + is_escalated, +) + + +def _format_conversation_history(conversation_history) -> str: + """Formats conversation history into human-readable customer/agent turns.""" + if not conversation_history: + return "No previous messages." + + if isinstance(conversation_history, str): + return conversation_history.strip() + + formatted_turns = [] + for item in conversation_history: + if isinstance(item, dict): + sender = ( + item.get("sender_type") + or item.get("sender") + or item.get("role") + or "Unknown" + ) + text = ( + item.get("message_text") + or item.get("text") + or item.get("content") + or item.get("message") + or "" + ) + + sender_lower = str(sender).lower() + if any(k in sender_lower for k in ["customer", "user"]): + role_label = "Customer" + elif any(k in sender_lower for k in ["agent", "support", "assistant"]): + role_label = "Support Agent" + elif "ai" in sender_lower or "system" in sender_lower: + role_label = "AI Suggestion" + else: + role_label = str(sender).title() + + formatted_turns.append(f"{role_label}: {text}") + elif isinstance(item, (list, tuple)) and len(item) >= 2: + formatted_turns.append(f"{item[0]}: {item[1]}") + else: + formatted_turns.append(str(item)) + + return "\n".join(formatted_turns) + + +def _format_state(state: dict) -> str: + """Formats customer emotional state into a clean text block.""" + if not state: + return "Unknown" + return ( + f"- Frustration: {state.get('frustration', 50)}/100\n" + f"- Trust: {state.get('trust', 50)}/100\n" + f"- Patience: {state.get('patience', 50)}/100\n" + f"- Satisfaction: {state.get('satisfaction', 50)}/100\n" + f"- Escalation Intent: {state.get('escalation_intent', 20)}/100" + ) + + +def _clean_customer_message(raw_text: str) -> str: + """Cleans up raw LLM output to extract just the customer message text.""" + if not raw_text: + return "" + + text = raw_text.strip() + + if text.startswith("```") and text.endswith("```"): + lines = text.splitlines() + text = "\n".join(lines[1:-1]).strip() + + text = re.sub(r"^(?:\[?\s*Customer\s*\]?\s*:\s*)", "", text, flags=re.IGNORECASE).strip() + + if (text.startswith('"') and text.endswith('"')) or (text.startswith("'") and text.endswith("'")): + text = text[1:-1].strip() + + return text + + +def build_customer_prompt( + persona: str, + scenario: str, + state: dict, + conversation_history, + agent_response: str +) -> str: + """Assembles a single prompt string for generating the customer's next turn. + + Args: + persona: Persona name (calm, confused, frustrated, angry, impatient, polite). + scenario: Scenario type (refund, delayed_order, payment_failure, account_issue, cancellation). + state: Current emotional state dictionary. + conversation_history: List or string representation of previous turns. + agent_response: The latest response provided by the support agent. + + Returns: + Structured prompt string ready for LLM consumption. + """ + persona_brief = get_persona_brief(persona) + scenario_brief = get_scenario_brief(scenario) + state_text = _format_state(state) + history_text = _format_conversation_history(conversation_history) + latest_agent_message = (agent_response or "").strip() or "(No response provided yet)" + + prompt = f"""You are simulating a customer in a customer support training exercise. + +=== CUSTOMER PROFILE === +{persona_brief} + +=== SCENARIO DETAILS === +{scenario_brief} + +=== CURRENT EMOTIONAL STATE === +{state_text} + +=== CONVERSATION HISTORY === +{history_text} + +=== LATEST SUPPORT AGENT MESSAGE === +Support Agent: {latest_agent_message} + +=== INSTRUCTIONS === +- Reply ONLY with the customer's next message in this conversation. +- Stay strictly in-character, adhering to the tone, sample phrases style, and escalation tendency of your persona. +- Reflect your current emotional state (frustration, trust, patience, satisfaction, escalation intent) naturally in how you speak. +- Maintain consistency with the scenario facts and prior conversation turns. +- DO NOT include prefixes like "Customer:", quotation marks, greetings if already deep in conversation, meta-commentary, explanations, or JSON formatting. +- Output ONLY the raw customer message text. +""" + return prompt.strip() + + +def generate_customer_turn( + persona: str, + scenario: str, + state: dict, + conversation_history, + agent_response: str +) -> dict: + """Generates the next customer turn in the simulation. + + Args: + persona: Persona name (calm, confused, frustrated, angry, impatient, polite). + scenario: Scenario type (refund, delayed_order, payment_failure, account_issue, cancellation). + state: Current customer state dict. + conversation_history: History of past conversation turns. + agent_response: Latest message from the support agent. + + Returns: + Dict with keys: + - customer_message: str + - updated_state: dict + - is_resolved: bool + - is_escalated: bool + """ + prompt = build_customer_prompt( + persona=persona, + scenario=scenario, + state=state, + conversation_history=conversation_history, + agent_response=agent_response, + ) + + try: + raw_response = generate_with_gemini(prompt) + customer_message = _clean_customer_message(raw_response) + if not customer_message: + raise ValueError("Empty response received from Gemini.") + except Exception as e: + print(f"Error generating customer response via Gemini: {e}") + persona_key = persona.strip().lower() + sample_phrases = PERSONAS.get(persona_key, {}).get( + "sample_phrases", + ["I see. Please let me know what we can do next."] + ) + customer_message = sample_phrases[0] if sample_phrases else "I see. Please help me resolve this." + + updated_state = update_state(state, agent_response, persona) + + return { + "customer_message": customer_message, + "updated_state": updated_state, + "is_resolved": is_resolved(updated_state), + "is_escalated": is_escalated(updated_state), + } diff --git a/RAG-Pipeline-backend/app/services/simulator_state.py b/RAG-Pipeline-backend/app/services/simulator_state.py new file mode 100644 index 0000000..5b00d47 --- /dev/null +++ b/RAG-Pipeline-backend/app/services/simulator_state.py @@ -0,0 +1,214 @@ +"""Simulator state engine for tracking customer emotional and resolution states.""" + +from app.services.persona_service import PERSONAS + +EMPATHY_SIGNALS = [ + "understand", + "sorry", + "apologize", + "apologies", + "let me help", + "happy to help", + "right away", + "refund", + "resolved", + "i can fix", + "i will fix", + "certainly", + "absolutely", + "thank you for your patience", + "i appreciate", + "let me check", + "let me look into", + "i'm on it", + "replacement", + "credit", +] + +DISMISSIVE_SIGNALS = [ + "can't help", + "cannot help", + "not possible", + "policy doesn't allow", + "against our policy", + "nothing i can do", + "not my problem", + "not our fault", + "contact someone else", + "call your bank", + "you should have", + "your fault", + "calm down", + "read the terms", + "deal with it", +] + +PERSONA_MULTIPLIERS = { + "low": 0.7, + "medium": 1.0, + "high": 1.5, +} + + +def _clamp(value: float) -> int: + """Clamps a numerical value to an integer within [0, 100].""" + return max(0, min(100, int(round(value)))) + + +def initial_state( + persona: str, + initial_emotion: str, + issue_severity: int, + patience_level: int +) -> dict: + """Builds an initial customer state dict from configuration parameters. + + Args: + persona: Persona name (calm, confused, frustrated, angry, impatient, polite). + initial_emotion: Starting emotional state label. + issue_severity: Severity score from 1 (minor) to 5 (critical). + patience_level: Patience score from 1 (very low) to 5 (very high). + + Returns: + A dictionary with integer values [0, 100] for frustration, trust, + patience, satisfaction, and escalation_intent. + """ + severity = max(1, min(5, issue_severity)) + patience_input = max(1, min(5, patience_level)) + + base_frustration = 20 + (severity - 1) * 15 + base_escalation = 10 + (severity - 1) * 12 + base_patience = patience_input * 20 + base_trust = 60 - (severity * 5) + base_satisfaction = 40 - (severity * 5) + + emotion_key = initial_emotion.strip().lower() + if "angry" in emotion_key: + base_frustration += 25 + base_escalation += 25 + base_patience -= 20 + base_trust -= 15 + base_satisfaction -= 15 + elif "frustrated" in emotion_key: + base_frustration += 15 + base_escalation += 15 + base_patience -= 10 + base_trust -= 10 + base_satisfaction -= 10 + elif "impatient" in emotion_key: + base_frustration += 10 + base_escalation += 15 + base_patience -= 25 + elif "confused" in emotion_key: + base_frustration += 5 + base_escalation += 5 + base_trust -= 5 + elif "calm" in emotion_key: + base_frustration -= 10 + base_escalation -= 10 + base_patience += 15 + base_trust += 10 + base_satisfaction += 10 + elif "polite" in emotion_key: + base_frustration -= 15 + base_escalation -= 15 + base_patience += 20 + base_trust += 15 + base_satisfaction += 15 + + persona_key = persona.strip().lower() + escalation_tendency = PERSONAS.get(persona_key, {}).get("escalation_tendency", "medium") + if escalation_tendency == "high": + base_frustration += 10 + base_escalation += 10 + base_patience -= 10 + elif escalation_tendency == "low": + base_frustration -= 5 + base_escalation -= 5 + base_patience += 10 + + return { + "frustration": _clamp(base_frustration), + "trust": _clamp(base_trust), + "patience": _clamp(base_patience), + "satisfaction": _clamp(base_satisfaction), + "escalation_intent": _clamp(base_escalation), + } + + +def update_state(current_state: dict, agent_response: str, persona: str) -> dict: + """Updates customer emotional state based on agent response and persona. + + Args: + current_state: Dict containing frustration, trust, patience, satisfaction, + and escalation_intent. + agent_response: Text message sent by the agent. + persona: Persona name to determine escalation tendency multiplier. + + Returns: + A new dict with updated integer values [0, 100]. + """ + persona_key = persona.strip().lower() + escalation_tendency = PERSONAS.get(persona_key, {}).get("escalation_tendency", "medium") + multiplier = PERSONA_MULTIPLIERS.get(escalation_tendency, 1.0) + + text = (agent_response or "").lower().strip() + words = text.split() + + empathy_matches = sum(1 for signal in EMPATHY_SIGNALS if signal in text) + dismissive_matches = sum(1 for signal in DISMISSIVE_SIGNALS if signal in text) + is_too_short = len(words) > 0 and len(words) < 4 + + delta_frustration = 0.0 + delta_trust = 0.0 + delta_patience = 0.0 + delta_satisfaction = 0.0 + delta_escalation = 0.0 + + if empathy_matches > 0: + factor = min(3, empathy_matches) + delta_frustration -= (12.0 * factor) + delta_trust += (10.0 * factor) + delta_patience += (6.0 * factor) + delta_satisfaction += (12.0 * factor) + delta_escalation -= (10.0 * factor) + + if dismissive_matches > 0: + factor = min(3, dismissive_matches) + delta_frustration += (18.0 * factor * multiplier) + delta_trust -= (14.0 * factor * multiplier) + delta_patience -= (16.0 * factor * multiplier) + delta_satisfaction -= (12.0 * factor * multiplier) + delta_escalation += (20.0 * factor * multiplier) + + if is_too_short and dismissive_matches == 0: + delta_frustration += (10.0 * multiplier) + delta_trust -= (8.0 * multiplier) + delta_patience -= (10.0 * multiplier) + delta_escalation += (12.0 * multiplier) + + if empathy_matches == 0 and dismissive_matches == 0 and not is_too_short: + if len(words) >= 4: + delta_frustration -= 3.0 + delta_trust += 3.0 + delta_patience -= 2.0 + delta_satisfaction += 3.0 + delta_escalation -= 2.0 + + return { + "frustration": _clamp(current_state.get("frustration", 50) + delta_frustration), + "trust": _clamp(current_state.get("trust", 50) + delta_trust), + "patience": _clamp(current_state.get("patience", 50) + delta_patience), + "satisfaction": _clamp(current_state.get("satisfaction", 50) + delta_satisfaction), + "escalation_intent": _clamp(current_state.get("escalation_intent", 20) + delta_escalation), + } + + +def is_resolved(state: dict) -> bool: + """Returns True if the customer state satisfies the resolution condition.""" + return state.get("satisfaction", 0) >= 75 and state.get("frustration", 100) <= 25 + + +def is_escalated(state: dict) -> bool: + """Returns True if the customer state indicates escalation.""" + return state.get("escalation_intent", 0) >= 85 diff --git a/RAG-Pipeline-backend/scripts/create_simulator_table.py b/RAG-Pipeline-backend/scripts/create_simulator_table.py new file mode 100644 index 0000000..e152bff --- /dev/null +++ b/RAG-Pipeline-backend/scripts/create_simulator_table.py @@ -0,0 +1,9 @@ +from app.models.database import Base, engine +from app.models.simulator import Scenario, Session, Conversation, Message + + +print("Creating simulator tables...") + +Base.metadata.create_all(bind=engine) + +print("Simulator tables created successfully.") diff --git a/RAG-Pipeline-backend/scripts/demo_simulator_conversations.py b/RAG-Pipeline-backend/scripts/demo_simulator_conversations.py new file mode 100644 index 0000000..a9e8207 --- /dev/null +++ b/RAG-Pipeline-backend/scripts/demo_simulator_conversations.py @@ -0,0 +1,292 @@ +"""Standalone runner to execute demo simulator conversations and export session logs. + +Demonstrates customer behavior across all 5 scenario types and 5 distinct personas, +generating genuine conversation logs in logs/simulator/ without requiring an HTTP server. +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath(".")) +os.environ.setdefault("GEMINI_API_KEY", "dummy_key_if_not_configured") + +from unittest.mock import MagicMock +for mod in ["pypdf", "sentence_transformers", "chromadb"]: + if mod not in sys.modules: + try: + __import__(mod) + except ImportError: + sys.modules[mod] = MagicMock() + +import json +from datetime import datetime + +from app.models.database import SessionLocal +from app.models.simulator import Scenario, Session, Conversation, Message +from app.services.simulator_service import generate_customer_turn +from app.services.simulator_state import initial_state +from app.services.scenario_service import SCENARIOS +from scripts.export_simulator_logs import export_all_completed_sessions + + +DEMO_CONVERSATIONS = [ + { + "title": "Demo 1: Angry Customer - Refund (Empathetic De-escalation)", + "scenario": "refund", + "persona": "angry", + "initial_emotion": "angry", + "issue_severity": 4, + "patience_level": 2, + "expected_resolution": "Full refund of $49.99 initiated within 3-5 business days", + "agent_turns": [ + "I sincerely apologize for the unexpected charge. I completely understand why you're upset, and I can see the duplicate subscription charge in our system right now.", + "I have authorized an immediate full refund of $49.99 back to your original payment card. You'll receive a confirmation email shortly, and the credit will appear on your statement in 3 to 5 business days.", + "You are very welcome. I'm truly sorry again for the frustration this caused. Is there anything else I can double-check for you today to make sure everything is sorted?" + ] + }, + { + "title": "Demo 2: Impatient Customer - Delayed Order (Dismissive Escalation)", + "scenario": "delayed_order", + "persona": "impatient", + "initial_emotion": "impatient", + "issue_severity": 4, + "patience_level": 1, + "expected_resolution": "Expedited courier replacement or full refund", + "agent_turns": [ + "We can't help with shipping delays once the courier has the box. That's against our policy.", + "Nothing I can do. It's not possible to expedite a package that is already in transit. You will just have to wait.", + "Call your bank or courier yourself if you're unhappy. There is nothing more I am authorized to do." + ] + }, + { + "title": "Demo 3: Confused Customer - Payment Failure (Guided Resolution)", + "scenario": "payment_failure", + "persona": "confused", + "initial_emotion": "confused", + "issue_severity": 3, + "patience_level": 3, + "expected_resolution": "Gateway 3DS verification explained and payment successfully verified", + "agent_turns": [ + "Hello! I understand how confusing payment errors can be. Let me look into that ERR_PAYMENT_FAILED_04 code for you right away.", + "It looks like your bank's 3D-Secure verification timed out during checkout. I have generated a direct secure verification link for you: https://pay.example.com/verify-3ds. Please click it to approve the transaction via your banking app.", + "I can confirm the payment has successfully gone through on our end! Your Annual Pro Plan is now fully activated. Thank you for your patience while we sorted that out." + ] + }, + { + "title": "Demo 4: Frustrated Customer - Account Lockout (Empathetic Verification)", + "scenario": "account_issue", + "persona": "frustrated", + "initial_emotion": "frustrated", + "issue_severity": 4, + "patience_level": 2, + "expected_resolution": "Backup email verification completed and account access restored", + "agent_turns": [ + "I'm very sorry for the lockout trouble! I know how frustrating it is to lose access to your account, especially with 2FA complications. Let me help you regain access right away.", + "To keep your account secure while bypassing the lost 2FA device, I've just sent a secure one-time verification magic link to your registered backup email address. Please check your inbox and click the link.", + "Great, I see the verification succeeded! I have reset your primary MFA requirement and unlocked your corporate account. You can now log in normally." + ] + }, + { + "title": "Demo 5: Calm Customer - Cancellation (Respectful Offboarding)", + "scenario": "cancellation", + "persona": "calm", + "initial_emotion": "calm", + "issue_severity": 2, + "patience_level": 4, + "expected_resolution": "Subscription cancelled cleanly with confirmation of access until billing period end", + "agent_turns": [ + "Hello! I understand you would like to cancel your monthly subscription. I'd be happy to assist you with that right away.", + "I have processed the cancellation of your Monthly Business Plan. Auto-renewal has been turned off, so you will not be charged again. Your access will remain active until the end of the current billing period in 4 days.", + "You are very welcome! If you ever decide to return, all your workspace settings and data will be saved. Have a wonderful day!" + ] + } +] + + +def run_demo(): + print("=" * 70) + print("CUSTOMER SIMULATOR - DEMO CONVERSATION RUNNER") + print("Generating genuine conversation logs across all 5 scenarios & personas") + print("=" * 70) + + db = SessionLocal() + completed_session_ids = [] + + try: + for idx, conv in enumerate(DEMO_CONVERSATIONS, 1): + scenario_key = conv["scenario"] + persona_key = conv["persona"] + scenario_data = SCENARIOS[scenario_key] + + print(f"\n[{idx}/5] {conv['title']}") + print("-" * 70) + + # 1. Create Scenario row + scenario_row = Scenario( + title=conv["title"], + category=scenario_key, + difficulty="Medium", + objective=conv["expected_resolution"], + description=scenario_data.get("opening_complaint"), + is_active=True + ) + db.add(scenario_row) + db.flush() + + # 2. Create Session row + session_row = Session( + scenario_id=scenario_row.scenario_id, + start_time=datetime.utcnow(), + status="In Progress" + ) + db.add(session_row) + db.flush() + + # 3. Create Conversation row + conversation_row = Conversation( + session_id=session_row.session_id, + intent=scenario_key, + sentiment=conv["initial_emotion"], + resolution_status="Unresolved", + escalation_risk="Low", + created_at=datetime.utcnow() + ) + db.add(conversation_row) + db.flush() + + # 4. Build initial state + current_state = initial_state( + persona=persona_key, + initial_emotion=conv["initial_emotion"], + issue_severity=conv["issue_severity"], + patience_level=conv["patience_level"] + ) + + # 5. Customer opening message + opening_message = scenario_data["opening_complaint"] + customer_msg = Message( + conversation_id=conversation_row.conversation_id, + sender_type="Customer", + message_text=opening_message, + timestamp=datetime.utcnow(), + message_type="Text" + ) + db.add(customer_msg) + + # 6. System state message + system_state_msg = Message( + conversation_id=conversation_row.conversation_id, + sender_type="AI", + message_text=json.dumps({ + "persona": persona_key, + "scenario": scenario_key, + "state": current_state + }), + timestamp=datetime.utcnow(), + message_type="System" + ) + db.add(system_state_msg) + db.commit() + + print(f"Turn 1 (Customer Opening):") + print(f" Customer: {opening_message}") + print(f" State: Frust={current_state['frustration']}, Trust={current_state['trust']}, Sat={current_state['satisfaction']}, Esc={current_state['escalation_intent']}") + + dialogue_history = [{ + "sender_type": "Customer", + "message_text": opening_message + }] + + turn_num = 1 + is_res = False + is_esc = False + + for agent_turn_text in conv["agent_turns"]: + turn_num += 1 + print(f"\nTurn {turn_num}:") + print(f" Support Agent: {agent_turn_text}") + + # Save agent message + agent_msg = Message( + conversation_id=conversation_row.conversation_id, + sender_type="Support Agent", + message_text=agent_turn_text, + timestamp=datetime.utcnow(), + message_type="Text" + ) + db.add(agent_msg) + db.flush() + + # Generate customer turn + turn_result = generate_customer_turn( + persona=persona_key, + scenario=scenario_key, + state=current_state, + conversation_history=dialogue_history, + agent_response=agent_turn_text + ) + + customer_reply = turn_result["customer_message"] + current_state = turn_result["updated_state"] + is_res = turn_result["is_resolved"] + is_esc = turn_result["is_escalated"] + + print(f" Customer: {customer_reply}") + print(f" State: Frust={current_state['frustration']}, Trust={current_state['trust']}, Sat={current_state['satisfaction']}, Esc={current_state['escalation_intent']} | Resolved={is_res}, Escalated={is_esc}") + + # Save customer message + customer_reply_msg = Message( + conversation_id=conversation_row.conversation_id, + sender_type="Customer", + message_text=customer_reply, + timestamp=datetime.utcnow(), + message_type="Text" + ) + db.add(customer_reply_msg) + + # Save system updated state + system_state_row = Message( + conversation_id=conversation_row.conversation_id, + sender_type="AI", + message_text=json.dumps({ + "persona": persona_key, + "scenario": scenario_key, + "state": current_state + }), + timestamp=datetime.utcnow(), + message_type="System" + ) + db.add(system_state_row) + + dialogue_history.append({"sender_type": "Support Agent", "message_text": agent_turn_text}) + dialogue_history.append({"sender_type": "Customer", "message_text": customer_reply}) + + if is_res or is_esc: + break + + # Mark completed session + session_row.status = "Completed" + session_row.end_time = datetime.utcnow() + if is_res: + conversation_row.resolution_status = "Resolved" + elif is_esc: + conversation_row.escalation_risk = "High" + + db.commit() + completed_session_ids.append(session_row.session_id) + print(f"\n -> Session {session_row.session_id} completed. (Status: {session_row.status}, Final Outcome: {'Resolved' if is_res else ('Escalated' if is_esc else 'Concluded')})") + + finally: + db.close() + + print("\n" + "=" * 70) + print("EXPORTING COMPLETED SESSION LOGS...") + print("=" * 70) + exported_files = export_all_completed_sessions(output_dir="logs/simulator") + print(f"\nSuccessfully generated and exported {len(exported_files)} conversation log files:") + for fpath in exported_files: + print(f" - {fpath}") + + +if __name__ == "__main__": + run_demo() diff --git a/RAG-Pipeline-backend/scripts/export_simulator_logs.py b/RAG-Pipeline-backend/scripts/export_simulator_logs.py new file mode 100644 index 0000000..af39362 --- /dev/null +++ b/RAG-Pipeline-backend/scripts/export_simulator_logs.py @@ -0,0 +1,142 @@ +import os +import json +from app.models.database import SessionLocal +from app.models.simulator import Session, Conversation, Message, Scenario + + +def export_session_log(session_id: int, output_dir: str = "logs/simulator") -> str: + """Exports a single simulator session log to a JSON file. + + Args: + session_id: The ID of the session to export. + output_dir: Directory path where the JSON file will be written. + + Returns: + The written file path. + + Raises: + ValueError: If the session does not exist. + """ + db = SessionLocal() + + try: + session_row = ( + db.query(Session) + .filter(Session.session_id == session_id) + .first() + ) + + if not session_row: + raise ValueError(f"Session with id {session_id} not found.") + + scenario_row = None + if session_row.scenario_id: + scenario_row = ( + db.query(Scenario) + .filter(Scenario.scenario_id == session_row.scenario_id) + .first() + ) + + conversation_row = ( + db.query(Conversation) + .filter(Conversation.session_id == session_row.session_id) + .first() + ) + + messages = [] + if conversation_row: + messages = ( + db.query(Message) + .filter(Message.conversation_id == conversation_row.conversation_id) + .order_by(Message.message_id.asc()) + .all() + ) + + turn_count = sum( + 1 for m in messages if m.sender_type == "Customer" + ) + + log_data = { + "session_id": session_row.session_id, + "scenario_title": scenario_row.title if scenario_row else None, + "scenario_category": scenario_row.category if scenario_row else None, + "status": session_row.status, + "start_time": ( + session_row.start_time.isoformat() + if session_row.start_time + else None + ), + "end_time": ( + session_row.end_time.isoformat() + if session_row.end_time + else None + ), + "turn_count": turn_count, + "messages": [ + { + "message_id": m.message_id, + "sender_type": m.sender_type, + "message_text": m.message_text, + "message_type": m.message_type, + "timestamp": ( + m.timestamp.isoformat() + if m.timestamp + else None + ), + } + for m in messages + ], + } + + os.makedirs(output_dir, exist_ok=True) + file_path = os.path.join(output_dir, f"session_{session_id}.json") + + with open(file_path, "w", encoding="utf-8") as f: + json.dump(log_data, f, indent=2, ensure_ascii=False) + + return file_path + + finally: + db.close() + + +def export_all_completed_sessions(output_dir: str = "logs/simulator") -> list[str]: + """Exports all completed simulator sessions to JSON files. + + Args: + output_dir: Directory path where JSON files will be written. + + Returns: + List of written file paths. + """ + db = SessionLocal() + + try: + completed_sessions = ( + db.query(Session) + .filter(Session.status == "Completed") + .order_by(Session.session_id.asc()) + .all() + ) + + exported_paths = [] + for s in completed_sessions: + path = export_session_log(s.session_id, output_dir=output_dir) + exported_paths.append(path) + + return exported_paths + + finally: + db.close() + + +if __name__ == "__main__": + print("Exporting completed simulator session logs...") + paths = export_all_completed_sessions() + + if not paths: + print("No completed simulator sessions found to export.") + else: + print(f"Successfully exported {len(paths)} session log(s):") + for p in paths: + print(f" - {p}") diff --git a/RAG-Pipeline-backend/tests/test_simulator.py b/RAG-Pipeline-backend/tests/test_simulator.py new file mode 100644 index 0000000..5f434e7 --- /dev/null +++ b/RAG-Pipeline-backend/tests/test_simulator.py @@ -0,0 +1,483 @@ +"""Automated test suite for Customer Simulator models, services, state engine, and API endpoints.""" + +import os +import sys +from unittest.mock import MagicMock +import pytest + +# Ensure GEMINI_API_KEY is configured so imports succeed without real credentials +os.environ.setdefault("GEMINI_API_KEY", "mock_key_for_testing") + +# Mock optional packages if not present in the current Python environment +for mod in ["google", "google.genai", "pypdf", "sentence_transformers", "chromadb"]: + if mod not in sys.modules: + try: + __import__(mod) + except ImportError: + sys.modules[mod] = MagicMock() + +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from app.main import app +from app.models.database import Base +from app.models.simulator import Scenario, Session, Conversation, Message +from app.api.simulator import get_db +from app.services.persona_service import get_persona_brief, PERSONAS +from app.services.scenario_service import get_scenario_brief, SCENARIOS +from app.services.simulator_state import ( + initial_state, + update_state, + is_resolved, + is_escalated, +) +from app.services.simulator_service import generate_customer_turn + + +# --------------------------------------------------------------------------- +# Test Database Isolation & Pytest Fixtures +# --------------------------------------------------------------------------- + +TEST_DB_FILE = "test_simulator.db" +TEST_DATABASE_URL = f"sqlite:///./{TEST_DB_FILE}" + +test_engine = create_engine( + TEST_DATABASE_URL, + connect_args={"check_same_thread": False} +) +TestingSessionLocal = sessionmaker( + autocommit=False, + autoflush=False, + bind=test_engine +) + + +@pytest.fixture(scope="module", autouse=True) +def setup_test_db(): + """Sets up an isolated SQLite test database and overrides FastAPI dependency.""" + if os.path.exists(TEST_DB_FILE): + try: + os.remove(TEST_DB_FILE) + except OSError: + pass + + # Create all simulator tables in the test database + Base.metadata.create_all(bind=test_engine) + + def override_get_db(): + db = TestingSessionLocal() + try: + yield db + finally: + db.close() + + app.dependency_overrides[get_db] = override_get_db + + yield + + app.dependency_overrides.clear() + test_engine.dispose() + if os.path.exists(TEST_DB_FILE): + try: + os.remove(TEST_DB_FILE) + except OSError: + pass + + +@pytest.fixture(scope="module") +def client(): + """FastAPI TestClient fixture.""" + return TestClient(app) + + +# --------------------------------------------------------------------------- +# Section 1: Unit Tests for simulator_state.py (No DB, No HTTP, No LLM) +# --------------------------------------------------------------------------- + +def test_empathy_signals_lower_frustration_and_raise_trust(): + """An agent response with empathy signals lowers frustration and raises trust/satisfaction.""" + initial = { + "frustration": 65, + "trust": 40, + "patience": 45, + "satisfaction": 30, + "escalation_intent": 55, + } + agent_response = "I understand, I'm sorry for the trouble, let me fix this right away." + updated = update_state(initial, agent_response, persona="calm") + + assert updated["frustration"] < initial["frustration"], ( + f"Frustration should decrease: was {initial['frustration']}, now {updated['frustration']}" + ) + assert updated["trust"] > initial["trust"], ( + f"Trust should increase: was {initial['trust']}, now {updated['trust']}" + ) + assert updated["satisfaction"] > initial["satisfaction"], ( + f"Satisfaction should increase: was {initial['satisfaction']}, now {updated['satisfaction']}" + ) + assert updated["patience"] >= initial["patience"], ( + f"Patience should increase: was {initial['patience']}, now {updated['patience']}" + ) + + +def test_dismissive_signals_increase_frustration_and_lower_trust(): + """A dismissive agent response raises frustration/escalation and lowers trust/patience.""" + initial = { + "frustration": 35, + "trust": 65, + "patience": 60, + "satisfaction": 50, + "escalation_intent": 25, + } + agent_response = "Not possible, that's against our policy. Nothing I can do." + updated = update_state(initial, agent_response, persona="calm") + + assert updated["frustration"] > initial["frustration"], ( + f"Frustration should increase: was {initial['frustration']}, now {updated['frustration']}" + ) + assert updated["escalation_intent"] > initial["escalation_intent"], ( + f"Escalation intent should increase: was {initial['escalation_intent']}, now {updated['escalation_intent']}" + ) + assert updated["trust"] < initial["trust"], ( + f"Trust should decrease: was {initial['trust']}, now {updated['trust']}" + ) + assert updated["patience"] < initial["patience"], ( + f"Patience should decrease: was {initial['patience']}, now {updated['patience']}" + ) + + +def test_persona_escalation_tendency_multiplier_difference(): + """A dismissive response produces a larger frustration delta for angry (high) than calm (low).""" + base_state = { + "frustration": 30, + "trust": 50, + "patience": 60, + "satisfaction": 40, + "escalation_intent": 20, + } + dismissive_msg = "Not possible, that's against our policy." + + angry_state = update_state(base_state, dismissive_msg, persona="angry") + calm_state = update_state(base_state, dismissive_msg, persona="calm") + + angry_delta = angry_state["frustration"] - base_state["frustration"] + calm_delta = calm_state["frustration"] - base_state["frustration"] + + assert angry_delta > calm_delta, ( + f"Angry frustration delta ({angry_delta}) should be greater than calm delta ({calm_delta})" + ) + + +def test_is_resolved_condition(): + """is_resolved returns True only when satisfaction >= 75 and frustration <= 25.""" + passing_state = {"satisfaction": 75, "frustration": 25, "trust": 80, "patience": 70, "escalation_intent": 10} + high_satisfaction_state = {"satisfaction": 85, "frustration": 15, "trust": 90, "patience": 80, "escalation_intent": 5} + failing_satisfaction = {"satisfaction": 74, "frustration": 25, "trust": 70, "patience": 60, "escalation_intent": 15} + failing_frustration = {"satisfaction": 80, "frustration": 26, "trust": 70, "patience": 60, "escalation_intent": 15} + + assert is_resolved(passing_state) is True, "Boundary state (75 sat, 25 frust) should be resolved" + assert is_resolved(high_satisfaction_state) is True, "High satisfaction state should be resolved" + assert is_resolved(failing_satisfaction) is False, "Satisfaction < 75 should NOT be resolved" + assert is_resolved(failing_frustration) is False, "Frustration > 25 should NOT be resolved" + + +def test_is_escalated_condition(): + """is_escalated returns True only when escalation_intent >= 85.""" + assert is_escalated({"escalation_intent": 85}) is True, "85 escalation_intent should be escalated" + assert is_escalated({"escalation_intent": 95}) is True, "95 escalation_intent should be escalated" + assert is_escalated({"escalation_intent": 84}) is False, "84 escalation_intent should NOT be escalated" + assert is_escalated({"escalation_intent": 40}) is False, "40 escalation_intent should NOT be escalated" + + +def test_initial_state_severity_comparison(): + """issue_severity=5 produces higher starting frustration and escalation than issue_severity=1.""" + state_sev1 = initial_state(persona="calm", initial_emotion="calm", issue_severity=1, patience_level=3) + state_sev5 = initial_state(persona="calm", initial_emotion="calm", issue_severity=5, patience_level=3) + + assert state_sev5["frustration"] > state_sev1["frustration"], ( + f"Severity 5 frustration ({state_sev5['frustration']}) must exceed severity 1 ({state_sev1['frustration']})" + ) + assert state_sev5["escalation_intent"] > state_sev1["escalation_intent"], ( + f"Severity 5 escalation ({state_sev5['escalation_intent']}) must exceed severity 1 ({state_sev1['escalation_intent']})" + ) + + +# --------------------------------------------------------------------------- +# Section 2: Unit Tests for persona_service.py / scenario_service.py +# --------------------------------------------------------------------------- + +def test_persona_and_scenario_brief_content(): + """Brief functions return non-empty strings containing expected key terms.""" + persona_brief = get_persona_brief("angry") + assert isinstance(persona_brief, str) and len(persona_brief) > 0, "Persona brief must not be empty" + assert "Angry" in persona_brief, "Persona brief should identify the Angry persona" + assert "Tone:" in persona_brief, "Persona brief should describe tone" + assert "Escalation Tendency:" in persona_brief, "Persona brief should include escalation tendency" + + scenario_brief = get_scenario_brief("refund") + assert isinstance(scenario_brief, str) and len(scenario_brief) > 0, "Scenario brief must not be empty" + assert "Refund" in scenario_brief, "Scenario brief should identify the scenario" + assert "Opening Complaint:" in scenario_brief, "Scenario brief must include the opening complaint" + assert "Key Facts:" in scenario_brief, "Scenario brief must list key facts" + + +def test_persona_and_scenario_invalid_keys_raise_value_error(): + """Invalid keys raise ValueError and list available valid options in the message.""" + with pytest.raises(ValueError) as exc_p: + get_persona_brief("joyful") + p_error = str(exc_p.value) + assert "Invalid persona 'joyful'" in p_error + for expected_persona in ["calm", "confused", "frustrated", "angry", "impatient", "polite"]: + assert expected_persona in p_error, f"Error message must list '{expected_persona}'" + + with pytest.raises(ValueError) as exc_s: + get_scenario_brief("broken_hardware") + s_error = str(exc_s.value) + assert "Invalid scenario type 'broken_hardware'" in s_error + for expected_scenario in ["refund", "delayed_order", "payment_failure", "account_issue", "cancellation"]: + assert expected_scenario in s_error, f"Error message must list '{expected_scenario}'" + + +# --------------------------------------------------------------------------- +# Section 3: Unit Tests for simulator_service.py (Mocked LLM) +# --------------------------------------------------------------------------- + +def test_generate_customer_turn_success(monkeypatch): + """generate_customer_turn returns cleaned LLM response and invokes update_state properly.""" + fake_reply = 'Customer: "Alright, please make sure the refund is completed quickly."' + monkeypatch.setattr( + "app.services.simulator_service.generate_with_gemini", + lambda prompt: fake_reply + ) + + state = initial_state("calm", "calm", 2, 4) + result = generate_customer_turn( + persona="calm", + scenario="refund", + state=state, + conversation_history=[], + agent_response="I understand your concern and I have issued your refund right away." + ) + + assert result["customer_message"] == "Alright, please make sure the refund is completed quickly.", ( + f"Customer message should be cleaned of quotes/prefix: {result['customer_message']}" + ) + assert result["updated_state"]["frustration"] < state["frustration"], "Frustration should have decreased" + assert "is_resolved" in result, "Turn result must include is_resolved" + assert "is_escalated" in result, "Turn result must include is_escalated" + + +def test_generate_customer_turn_fallback_on_exception(monkeypatch): + """When Gemini raises an exception, generate_customer_turn falls back to a persona sample phrase.""" + def fail_gemini(prompt): + raise RuntimeError("Quota exceeded or API connection failed") + + monkeypatch.setattr( + "app.services.simulator_service.generate_with_gemini", + fail_gemini + ) + + state = initial_state("angry", "angry", 4, 2) + result = generate_customer_turn( + persona="angry", + scenario="refund", + state=state, + conversation_history=[], + agent_response="We cannot process that request." + ) + + assert result["customer_message"] in PERSONAS["angry"]["sample_phrases"], ( + f"Customer message must fall back to an angry sample phrase, got: {result['customer_message']}" + ) + assert isinstance(result["updated_state"], dict), "Updated state must still be a dictionary" + assert "frustration" in result["updated_state"], "Updated state must track emotional metrics" + + +# --------------------------------------------------------------------------- +# Section 4: Integration Tests via FastAPI TestClient (3 Combinations + Errors) +# --------------------------------------------------------------------------- + +def test_scenario_angry_refund_e2e(client, monkeypatch): + """E2E Test 1: Angry persona in a refund scenario.""" + monkeypatch.setattr( + "app.services.simulator_service.generate_with_gemini", + lambda prompt: "I see the refund request, but when will the money actually be in my account?" + ) + + # 1. Start simulation + start_resp = client.post("/simulator/start", json={ + "session_label": "Test Angry Refund Session", + "persona": "angry", + "scenario": "refund", + "initial_emotion": "angry", + "issue_severity": 4, + "patience_level": 2, + "expected_resolution": "Full refund of $49.99 processed", + }) + assert start_resp.status_code == 200, f"Start failed: {start_resp.text}" + start_data = start_resp.json() + session_id = start_data["session_id"] + assert start_data["turn"] == 1 + assert "customer_message" in start_data + state_turn1 = start_data["state"] + + # 2. Empathetic agent response -> state improves + msg1_resp = client.post("/simulator/message", json={ + "session_id": session_id, + "agent_response": "I sincerely apologize for the frustration. I have submitted your full refund of $49.99 right away." + }) + assert msg1_resp.status_code == 200, f"Turn 2 failed: {msg1_resp.text}" + msg1_data = msg1_resp.json() + assert msg1_data["turn"] == 2 + state_turn2 = msg1_data["state"] + assert state_turn2["frustration"] < state_turn1["frustration"], "Frustration should drop after empathy" + assert state_turn2["trust"] > state_turn1["trust"], "Trust should rise after empathy" + + # 3. Dismissive agent response -> state worsens + msg2_resp = client.post("/simulator/message", json={ + "session_id": session_id, + "agent_response": "Policy doesn't allow any expedited processing. Nothing I can do, call your bank." + }) + assert msg2_resp.status_code == 200, f"Turn 3 failed: {msg2_resp.text}" + msg2_data = msg2_resp.json() + assert msg2_data["turn"] == 3 + state_turn3 = msg2_data["state"] + assert state_turn3["frustration"] > state_turn2["frustration"], "Frustration should rise after dismissive response" + assert state_turn3["escalation_intent"] > state_turn2["escalation_intent"], "Escalation should rise" + + +def test_scenario_polite_delayed_order_e2e(client, monkeypatch): + """E2E Test 2: Polite persona in a delayed_order scenario.""" + monkeypatch.setattr( + "app.services.simulator_service.generate_with_gemini", + lambda prompt: "Thank you so much for looking into the courier tracking for me." + ) + + start_resp = client.post("/simulator/start", json={ + "session_label": "Test Polite Delayed Order Session", + "persona": "polite", + "scenario": "delayed_order", + "initial_emotion": "calm", + "issue_severity": 2, + "patience_level": 5, + "expected_resolution": "Tracking update and waived shipping fee", + }) + assert start_resp.status_code == 200, f"Start failed: {start_resp.text}" + start_data = start_resp.json() + session_id = start_data["session_id"] + state_turn1 = start_data["state"] + + # Empathetic response + msg_resp = client.post("/simulator/message", json={ + "session_id": session_id, + "agent_response": "I understand how important this delivery is. I'm happy to help waive the shipping fee and track this right away." + }) + assert msg_resp.status_code == 200 + msg_data = msg_resp.json() + assert msg_data["session_id"] == session_id + assert msg_data["turn"] == 2 + assert "is_resolved" in msg_data + assert "is_escalated" in msg_data + assert msg_data["state"]["satisfaction"] > state_turn1["satisfaction"], "Satisfaction should increase" + + +def test_scenario_impatient_payment_failure_e2e(client, monkeypatch): + """E2E Test 3: Impatient persona in a payment_failure scenario.""" + monkeypatch.setattr( + "app.services.simulator_service.generate_with_gemini", + lambda prompt: "Just give me the alternate payment link right now so I can finish this." + ) + + start_resp = client.post("/simulator/start", json={ + "session_label": "Test Impatient Payment Failure Session", + "persona": "impatient", + "scenario": "payment_failure", + "initial_emotion": "impatient", + "issue_severity": 3, + "patience_level": 1, + "expected_resolution": "Alternate payment checkout link verified", + }) + assert start_resp.status_code == 200 + start_data = start_resp.json() + session_id = start_data["session_id"] + state_turn1 = start_data["state"] + + # Empathetic response with resolution keyword + msg_resp = client.post("/simulator/message", json={ + "session_id": session_id, + "agent_response": "I understand you are in a hurry. Let me help you with an alternate link right away to get this resolved." + }) + assert msg_resp.status_code == 200 + msg_data = msg_resp.json() + assert msg_data["turn"] == 2 + assert msg_data["state"]["frustration"] < state_turn1["frustration"] + + +def test_start_with_invalid_scenario_returns_400(client): + """POST /simulator/start with an invalid scenario returns 400 Bad Request.""" + resp = client.post("/simulator/start", json={ + "session_label": "Invalid Scenario Session", + "persona": "calm", + "scenario": "unsupported_scenario_type", + "initial_emotion": "calm", + "issue_severity": 2, + "patience_level": 3, + "expected_resolution": "None", + }) + assert resp.status_code == 400, f"Expected 400, got: {resp.status_code}" + error_detail = resp.json()["detail"] + assert "Invalid scenario type" in error_detail + + +def test_message_with_nonexistent_session_returns_404(client): + """POST /simulator/message with a nonexistent session_id returns 404 Not Found.""" + resp = client.post("/simulator/message", json={ + "session_id": 999999, + "agent_response": "Hello, how may I help you?", + }) + assert resp.status_code == 404, f"Expected 404, got: {resp.status_code}" + assert "Simulator session not found" in resp.json()["detail"] + + +def test_get_history_excludes_system_messages(client, monkeypatch): + """GET /simulator/{session_id}/history returns only dialogue messages, excluding System rows.""" + monkeypatch.setattr( + "app.services.simulator_service.generate_with_gemini", + lambda prompt: "I got the updated invoice, thank you." + ) + + # Start session (creates Customer opening message + System state message) + start_resp = client.post("/simulator/start", json={ + "session_label": "History Test Session", + "persona": "calm", + "scenario": "cancellation", + "initial_emotion": "calm", + "issue_severity": 1, + "patience_level": 4, + "expected_resolution": "Subscription cancelled cleanly", + }) + session_id = start_resp.json()["session_id"] + + # Send one agent message (creates Agent message + Customer reply + System state message) + client.post("/simulator/message", json={ + "session_id": session_id, + "agent_response": "I can help you cancel your subscription right away." + }) + + # Fetch history + hist_resp = client.get(f"/simulator/{session_id}/history") + assert hist_resp.status_code == 200 + hist_data = hist_resp.json() + + messages = hist_data["messages"] + # Total dialogue messages should be 3: Customer opening -> Support Agent -> Customer reply + assert len(messages) == 3, f"Expected 3 dialogue messages, found {len(messages)}" + + for msg in messages: + assert msg["message_type"] != "System", ( + f"System message should be excluded from history endpoint: {msg}" + ) + assert msg["sender_type"] in ["Customer", "Support Agent"], ( + f"Unexpected sender type in history: {msg['sender_type']}" + ) From 2c442b443ca035462e725ab08d43b34b1dbfbc88 Mon Sep 17 00:00:00 2001 From: Shrushti-Sakat Date: Thu, 3 Sep 2026 18:00:41 +0530 Subject: [PATCH 2/2] Exclude simulator logs and test sqlite database in .gitignore --- RAG-Pipeline-backend/.gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/RAG-Pipeline-backend/.gitignore b/RAG-Pipeline-backend/.gitignore index a2e634c..0fff434 100644 --- a/RAG-Pipeline-backend/.gitignore +++ b/RAG-Pipeline-backend/.gitignore @@ -1,4 +1,6 @@ .venv/ .env __pycache__/ -data/chroma_db/ \ No newline at end of file +data/chroma_db/ +logs/simulator/ +test_simulator.db \ No newline at end of file