Merge pull request #53 from green-api/feature/vad

Добавить VAD в библиотеку чат-бота и обновить зависимости в демо-чатботе

Author: kennyromanov

README.md

@@ -540,10 +540,13 @@ credentials:
 
 ```python
 import asyncio
-import logging
 
 from whatsapp_chatbot_python.calls import (
-    CallEvent, CallSession, CallStateMachine,
+    CallEvent,
+    CallOptions,
+    CallSession,
+    CallStateMachine,
+    VadOptions,
 )
 from whatsapp_chatbot_python.calls.service import WhatsAppCallService
 
@@ -560,10 +563,17 @@ service = WhatsAppCallService(
     openai_api_key=openai_api_key,
     realtime_model=realtime_model,
     realtime_voice=realtime_voice,
-    ring_timeout_seconds=30,
-    talk_timeout_seconds=120,
-    shutdown_timeout_seconds=10,
-    logger=logging.getLogger("calls"),
+    options=CallOptions(
+        ring_seconds=30,
+        talk_seconds=120,
+        shutdown_seconds=10,
+        vad=VadOptions(
+            threshold=0.70,
+            prefix_padding_ms=400,
+            silence_duration_ms=800,
+            noise_reduction="near_field",
+        ),
+    ),
 )
 
 async def call_once():
@@ -611,7 +621,7 @@ session = CallSession.incoming(
     call_id=call_id,
     language="en",
     incoming_deadline=monotonic() + remaining_seconds,
-    greeting_text="Hello! This is a WhatsApp demo call. What would you like to discuss?",
+    prompt="Greet the user: 'Hello! This is a WhatsApp demo call. What would you like to discuss?'",
 )
 fsm = CallStateMachine()
 fsm.apply(session, CallEvent.ENQUEUED)

docs/README.md

@@ -532,10 +532,13 @@ python -m pip install 'whatsapp-chatbot-python[voip]'
 
 ```python
 import asyncio
-import logging
 
 from whatsapp_chatbot_python.calls import (
-    CallEvent, CallSession, CallStateMachine,
+    CallEvent,
+    CallOptions,
+    CallSession,
+    CallStateMachine,
+    VadOptions,
 )
 from whatsapp_chatbot_python.calls.service import WhatsAppCallService
 
@@ -552,10 +555,17 @@ service = WhatsAppCallService(
     openai_api_key=openai_api_key,
     realtime_model=realtime_model,
     realtime_voice=realtime_voice,
-    ring_timeout_seconds=30,
-    talk_timeout_seconds=120,
-    shutdown_timeout_seconds=10,
-    logger=logging.getLogger("calls"),
+    options=CallOptions(
+        ring_seconds=30,
+        talk_seconds=120,
+        shutdown_seconds=10,
+        vad=VadOptions(
+            threshold=0.70,
+            prefix_padding_ms=400,
+            silence_duration_ms=800,
+            noise_reduction="near_field",
+        ),
+    ),
 )
 
 async def call_once():
@@ -603,7 +613,7 @@ session = CallSession.incoming(
     call_id=call_id,
     language="ru",
     incoming_deadline=monotonic() + remaining_seconds,
-    greeting_text="Здравствуйте! Это демонстрационный звонок WhatsApp. Что хотите обсудить?",
+    prompt="Поприветствуй пользователя: 'Здравствуйте! Это демонстрационный звонок WhatsApp. Что хотите обсудить?'",
 )
 fsm = CallStateMachine()
 fsm.apply(session, CallEvent.ENQUEUED)

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name="whatsapp-chatbot-python",
-    version="1.1.1",
+    version="1.2.0",
     description=(
         "This library helps you easily create"
         " a Python chatbot with WhatsApp API."

tests/calls/fakes.py

@@ -126,8 +126,9 @@ async def hangUpAsync(self):
 
 
 class FakeVoice:
-    def __init__(self, scenario, *args):
+    def __init__(self, scenario, *args, vad=None):
         self.scenario = scenario
+        self.vad = vad
         self.greetings = []
         self.closed = False
         self.started = False
@@ -147,9 +148,9 @@ async def start(self):
             raise RuntimeError("voice unavailable")
         self.started = True
 
-    async def greet_once(self, text):
+    async def prompt_once(self, prompt):
         if not self.greetings:
-            self.greetings.append(text)
+            self.greetings.append(prompt)
             self.scenario.actions.append("greet")
 
     def read_frame(self):

tests/calls/test_call_cleanup.py

@@ -26,8 +26,8 @@ def api_factory(*args, **kwargs):
             self.apis.append(api)
             return api
 
-        def voice_factory(*args):
-            voice = FakeVoice(self.scenario, *args)
+        def voice_factory(*args, **kwargs):
+            voice = FakeVoice(self.scenario, *args, **kwargs)
             self.voices.append(voice)
             return voice
 

tests/calls/test_call_service.py

@@ -8,7 +8,7 @@
 
 from whatsapp_chatbot_python.calls.models import (
     CallEndReason, CallEvent, CallOptions, CallSession, CallState, ProviderError,
-    RecordingOptions,
+    RecordingOptions, VadOptions,
 )
 from whatsapp_chatbot_python.calls.service import WhatsAppCallService
 from whatsapp_chatbot_python.calls.state_machine import CallStateMachine
@@ -28,8 +28,8 @@ def create_api(*args, **kwargs):
             self.apis.append(api)
             return api
 
-        def create_voice(*args):
-            voice = FakeVoice(self.scenario, *args)
+        def create_voice(*args, **kwargs):
+            voice = FakeVoice(self.scenario, *args, **kwargs)
             self.voices.append(voice)
             return voice
 
@@ -41,7 +41,7 @@ def create_voice(*args):
             patcher.start()
             self.addCleanup(patcher.stop)
 
-    def service(self, *, ring=0.2, talk=0.2):
+    def service(self, *, ring=0.2, talk=0.2, vad=None):
         return WhatsAppCallService(
             api_url="https://example.invalid", id_instance="1",
             api_token_instance="token", openai_api_key="explicit-key",
@@ -50,6 +50,7 @@ def service(self, *, ring=0.2, talk=0.2):
                 ring_seconds=ring, talk_seconds=talk, setup_seconds=0.1,
                 command_seconds=0.1, shutdown_seconds=0.1,
                 recording=RecordingOptions(enabled=False),
+                vad=vad or VadOptions(),
             ),
         )
 
@@ -59,10 +60,11 @@ def outgoing(self):
         self.fsm.apply(session, CallEvent.DEQUEUED)
         return session
 
-    def incoming(self, *, call_id="call-1", deadline=None):
+    def incoming(self, *, call_id="call-1", deadline=None, prompt=None):
         session = CallSession.incoming(
             sender_id="sender@c.us", call_id=call_id, language="ru",
             incoming_deadline=deadline if deadline is not None else monotonic() + 1,
+            prompt=prompt,
         )
         self.fsm.apply(session, CallEvent.ENQUEUED)
         self.fsm.apply(session, CallEvent.DEQUEUED)
@@ -94,13 +96,19 @@ async def test_outgoing_answer_greets_once_and_preserves_remote_reason(self):
 
     async def test_incoming_accepts_matching_offer_and_greets_after_answer(self):
         self.scenario.state = "inc-call"
-        session = self.incoming()
+        session = self.incoming(prompt="Introduce yourself briefly.")
         await self.execute(self.service(), session)
         self.assertEqual(session.state, CallState.REMOTE_ENDED)
         self.assertTrue(session.incoming_accept_completed)
         self.assertLess(self.scenario.actions.index("accept"), self.scenario.actions.index("greet"))
+        self.assertEqual(self.voices[0].greetings, ["Introduce yourself briefly."])
         self.assertTrue(self.apis[0].session_closed)
 
+    async def test_service_passes_vad_options_to_voice(self):
+        vad = VadOptions(threshold=0.85, noise_reduction="far_field")
+        await self.execute(self.service(vad=vad), self.outgoing())
+        self.assertIs(self.voices[0].vad, vad)
+
     async def test_incoming_call_id_takes_priority_over_pn_or_lid_address(self):
         self.scenario.state = "inc-call"
         self.scenario.wid = "sender@lid"

tests/calls/test_call_state_machine.py

@@ -17,6 +17,18 @@ def setUp(self) -> None:
         self.fsm = CallStateMachine()
         self.session = CallSession("sender@c.us", "sender@c.us", "ru")
 
+    def test_incoming_call_keeps_its_prompt(self) -> None:
+        session = CallSession.incoming(
+            sender_id="sender@c.us", call_id="call-1", language="adaptive",
+            incoming_deadline=monotonic() + 1,
+            prompt="Introduce yourself briefly.",
+        )
+        self.assertEqual(session.prompt, "Introduce yourself briefly.")
+
+    def test_empty_prompt_is_rejected(self) -> None:
+        with self.assertRaisesRegex(ValueError, "Call prompt must not be empty"):
+            CallSession("sender@c.us", "sender@c.us", "adaptive", prompt="  ")
+
     def apply(self, event: CallEvent) -> CallState:
         self.fsm.apply(self.session, event)
         return self.session.state

tests/calls/test_realtime_voice.py

@@ -7,7 +7,7 @@
 from unittest.mock import patch
 
 from whatsapp_chatbot_python.calls.models import (
-    AudioLoss, MediaFailed, PCM, PlaybackOptions, ProviderConfig,
+    AudioLoss, CallOptions, MediaFailed, PCM, PlaybackOptions, ProviderConfig, VadOptions,
 )
 from whatsapp_chatbot_python.calls.openai import RealtimeConversation
 from whatsapp_chatbot_python.calls.playback_state_machine import PlaybackController
@@ -146,25 +146,92 @@ async def audio(self, payload=FRAME, response="response-1", item="item-1"):
             delta=base64.b64encode(payload).decode("ascii"),
         ))
 
-    async def test_session_uses_explicit_key_and_greets_only_once(self):
+    async def test_explicit_prompt_is_sent_unchanged_only_once(self):
         self.assertEqual(self.client.api_key, "explicit-key")
         settings = self.client.connection.actions[0][1]["session"]
         self.assertEqual(settings["audio"]["output"]["voice"], "marin")
         self.assertEqual(settings["audio"]["input"]["format"]["rate"], PCM.rate)
-        await self.voice.greet_once("Привет")
-        await self.voice.greet_once("Ещё раз")
+        self.assertEqual(settings["audio"]["input"]["noise_reduction"],
+                         {"type": "near_field"})
+        self.assertEqual(settings["audio"]["input"]["turn_detection"], {
+            "type": "server_vad",
+            "threshold": 0.70,
+            "prefix_padding_ms": 400,
+            "silence_duration_ms": 800,
+            "create_response": True,
+            "interrupt_response": True,
+        })
+        await self.voice.prompt_once("Introduce yourself and ask how you can help.")
+        await self.voice.prompt_once("This prompt must be ignored.")
         self.assertEqual([action for action, _ in self.client.connection.actions].count("create"), 1)
         self.assertEqual(
             self.client.connection.actions[-1][1]["response"]["instructions"],
-            'Greet the caller: "Привет" in Russian.',
+            "Introduce yourself and ask how you can help.",
         )
 
+    async def test_custom_vad_settings_reach_realtime(self):
+        vad = VadOptions(
+            threshold=0.85, prefix_padding_ms=250,
+            silence_duration_ms=1200, noise_reduction="far_field",
+        )
+        config = ProviderConfig("https://example.invalid", "1", "token", "key", "model", "marin")
+        async with RealtimeConversation(
+            config, PlaybackOptions(), self._record_event, "ru", 0.1, vad=vad,
+        ) as voice:
+            await voice.start()
+            audio_input = voice._client.connection.actions[0][1]["session"]["audio"]["input"]
+
+        self.assertEqual(audio_input["noise_reduction"], {"type": "far_field"})
+        self.assertEqual(audio_input["turn_detection"], {
+            "type": "server_vad",
+            "threshold": 0.85,
+            "prefix_padding_ms": 250,
+            "silence_duration_ms": 1200,
+            "create_response": True,
+            "interrupt_response": True,
+        })
+
+    async def test_none_disables_noise_reduction(self):
+        self.voice._vad = VadOptions(noise_reduction=None)
+        self.assertIsNone(self.voice._session_settings()["audio"]["input"]["noise_reduction"])
+
     async def test_default_greeting_instruction_uses_language_name(self):
-        await self.voice.greet_once(None)
+        await self.voice.prompt_once(None)
         self.assertEqual(
             self.client.connection.actions[-1][1]["response"]["instructions"],
-            "Greet the caller briefly in Russian.",
+            "Greet the user briefly in Russian.",
+        )
+
+    async def test_default_greeting_is_not_used_as_session_instruction(self):
+        self.assertNotIn("instructions", self.voice._session_settings())
+
+    async def test_explicit_conversation_instructions_remain_session_wide(self):
+        config = ProviderConfig(
+            "https://example.invalid", "1", "token", "key", "model", "marin",
+            "You are a helpful Russian-speaking phone assistant.",
         )
+        async with RealtimeConversation(
+            config, PlaybackOptions(), self._record_event, "ru", 0.1,
+        ) as voice:
+            await voice.start()
+
+        settings = self.client.connection.actions[0][1]["session"]
+        self.assertEqual(
+            settings["instructions"],
+            "You are a helpful Russian-speaking phone assistant.",
+        )
+
+    async def test_adaptive_language_uses_switching_fallbacks(self):
+        self.voice._language = "adaptive"
+        settings = self.voice._session_settings()
+        required_phrases = ("another language", "switch to", "in English")
+        self.assertNotIn("instructions", settings)
+
+        await self.voice.prompt_once(None)
+        greeting_instructions = self.client.connection.actions[-1][1]["response"]["instructions"].lower()
+        for phrase in required_phrases:
+            with self.subTest(target="greeting", phrase=phrase):
+                self.assertIn(phrase.lower(), greeting_instructions)
 
     async def test_incoming_pcm_is_base64_encoded(self):
         await self.voice.write_audio(FRAME)
@@ -182,6 +249,16 @@ async def test_barge_in_truncates_to_audio_actually_heard(self):
         self.assertEqual(actions[-1][1], dict(item_id="item-1", content_index=0,
                                               audio_end_ms=PCM.frame_ms))
 
+    async def test_provider_cancel_error_after_speech_started_is_ignored(self):
+        await self.dispatch(dict(type="response.created", response=SimpleNamespace(id="response-1")))
+        await self.dispatch(dict(type="input_audio_buffer.speech_started"))
+        cancel = next(data for name, data in self.client.connection.actions if name == "cancel")
+        await self.dispatch(dict(type="error", error=SimpleNamespace(
+            event_id=cancel["event_id"], code="response_cancel_failed", message="Already cancelled",
+        )))
+        self.assertEqual(self.failures, [])
+
+
     async def test_replaced_track_drops_old_audio_and_new_response_plays(self):
         await self.dispatch(dict(type="response.created", response=SimpleNamespace(id="response-1")))
         await self.audio(FRAME * 2)
@@ -292,3 +369,21 @@ async def test_finished_cancelled_response_releases_tracking(self):
         await self.dispatch(dict(type="response.done", response=SimpleNamespace(
             id="response-1", status="cancelled")))
         self.assertNotIn("response-1", self.voice._cancelled)
+
+
+class VadOptionsTest(unittest.TestCase):
+    def test_invalid_vad_settings_are_rejected_before_call_start(self):
+        invalid_settings = (
+            {"threshold": -0.1}, {"threshold": 1.1},
+            {"threshold": float("nan")}, {"threshold": float("inf")},
+            {"threshold": True}, {"threshold": "0.7"},
+            {"prefix_padding_ms": -1}, {"prefix_padding_ms": True},
+            {"prefix_padding_ms": 0.5}, {"silence_duration_ms": 0},
+            {"silence_duration_ms": True}, {"noise_reduction": "unknown"},
+        )
+        for settings in invalid_settings:
+            with self.subTest(settings=settings), self.assertRaises(ValueError):
+                VadOptions(**settings)
+
+        with self.assertRaisesRegex(ValueError, "VAD options must be VadOptions"):
+            CallOptions(vad=None)

whatsapp_chatbot_python/calls/__init__.py

@@ -24,6 +24,7 @@
     RecordingOptions,
     StateTransition,
     TransitionCallback,
+    VadOptions,
     ACTIVE_STATES,
     TERMINAL_STATES,
     WAITING_STATES,
@@ -37,5 +38,5 @@
     "CallExecutionResult", "CallExecutor", "CallOptions", "CallSession", "CallServiceUnavailable", "CallState",
     "CallStateMachine", "IncomingCallExecutor", "InputAudioOptions", "InputOverflowPolicy", "PlaybackOptions",
     "PlaybackOverflowPolicy", "ProviderError", "RecordingOptions", "StateTransition", "TransitionCallback",
-    "WhatsAppCallService",
+    "VadOptions", "WhatsAppCallService",
 ]