gemini user sim and audio quality

.env.example

@@ -244,22 +244,24 @@ EVA_MODEL_LIST='[
 # --- User simulator provider ---
 #i Caller provider. ElevenLabs is the backward-compatible default.
 #d enum
-#e elevenlabs,openai_realtime
+#e elevenlabs,openai_realtime,gemini_live
 EVA_USER_SIMULATOR__PROVIDER=elevenlabs
 
-#i OpenAI Realtime caller model. Used only when provider=openai_realtime.
+#i Realtime caller model. Used when provider=openai_realtime (e.g. gpt-realtime-1.5) or
+#i provider=gemini_live (e.g. gemini-2.5-flash-native-audio-preview-09-2025).
+#i gemini_live requires one of GEMINI_API_KEY, GOOGLE_API_KEY, GOOGLE_CLOUD_PROJECT (Vertex AI), or GOOGLE_APPLICATION_CREDENTIALS.
 #d string
-#x EVA_USER_SIMULATOR__PROVIDER=openai_realtime
+#x EVA_USER_SIMULATOR__PROVIDER=openai_realtime,gemini_live
 #v EVA_USER_SIMULATOR__MODEL=gpt-realtime-1.5
 
-#i OpenAI voice for the female caller persona.
+#i Voice for the female caller persona. Provider-specific (e.g. marin for openai_realtime, Aoede for gemini_live).
 #d string
-#x EVA_USER_SIMULATOR__PROVIDER=openai_realtime
+#x EVA_USER_SIMULATOR__PROVIDER=openai_realtime,gemini_live
 #v EVA_USER_SIMULATOR__FEMALE_VOICE=marin
 
-#i OpenAI voice for the male caller persona.
+#i Voice for the male caller persona. Provider-specific (e.g. cedar for openai_realtime, Charon for gemini_live).
 #d string
-#x EVA_USER_SIMULATOR__PROVIDER=openai_realtime
+#x EVA_USER_SIMULATOR__PROVIDER=openai_realtime,gemini_live
 #v EVA_USER_SIMULATOR__MALE_VOICE=cedar
 
 # --- Language (mutually exclusive with Accent and Behavior) ---

src/eva/assistant/audio_bridge.py

@@ -35,20 +35,42 @@ def mulaw_8k_to_pcm16_16k(mulaw_bytes: bytes) -> bytes:
     return pcm_16k
 
 
+def pcm16_16k_to_mulaw_8k(pcm_bytes: bytes) -> bytes:
+    """Convert 16kHz 16-bit PCM to 8kHz mu-law.
+
+    Uses soxr VHQ resampling for proper anti-aliasing during the 2:1 downsampling.
+    audioop.ratecv lacks an anti-aliasing filter, causing frequencies 4-8kHz (sibilants)
+    to alias back into the speech band as electrical-sounding distortion.
+    """
+    audio_data = np.frombuffer(pcm_bytes, dtype=np.int16)
+    resampled = soxr.resample(audio_data, 16000, 8000, quality="VHQ")
+    expected_samples = round(len(audio_data) * 8000 / 16000)
+    if len(resampled) < expected_samples:
+        resampled = np.pad(resampled, (0, expected_samples - len(resampled)))
+    elif len(resampled) > expected_samples:
+        resampled = resampled[:expected_samples]
+    pcm_8k = np.clip(resampled, -32768, 32767).astype(np.int16).tobytes()
+    return audioop.lin2ulaw(pcm_8k, 2)
+
+
 def mulaw_8k_to_pcm16_24k(mulaw_bytes: bytes) -> bytes:
-    """Convert 8kHz mu-law audio to 24kHz 16-bit PCM."""
+    """Convert 8kHz mu-law audio to 24kHz 16-bit PCM.
+
+    Uses soxr VHQ resampling for clean 3:1 upsampling. audioop.ratecv restarts
+    its interpolation filter on every call (stateless) which creates audible
+    discontinuities at chunk boundaries for the 3:1 ratio.
+    """
     # Decode mu-law to 16-bit PCM at 8kHz
     pcm_8k = audioop.ulaw2lin(mulaw_bytes, 2)
-    # Upsample from 8kHz to 24kHz
-    pcm_24k, _ = audioop.ratecv(pcm_8k, 2, 1, 8000, 24000, None)
-    # audioop.ratecv can produce ±2 samples; clamp to exact 3× input length
-    # so that the inverse conversion recovers the original sample count.
-    expected_bytes = len(pcm_8k) * 3
-    if len(pcm_24k) < expected_bytes:
-        pcm_24k = pcm_24k + b"\x00" * (expected_bytes - len(pcm_24k))
-    elif len(pcm_24k) > expected_bytes:
-        pcm_24k = pcm_24k[:expected_bytes]
-    return pcm_24k
+    # Upsample from 8kHz to 24kHz using high-quality resampler
+    audio_data = np.frombuffer(pcm_8k, dtype=np.int16)
+    resampled = soxr.resample(audio_data, 8000, 24000, quality="VHQ")
+    expected_samples = round(len(audio_data) * 24000 / 8000)
+    if len(resampled) < expected_samples:
+        resampled = np.pad(resampled, (0, expected_samples - len(resampled)))
+    elif len(resampled) > expected_samples:
+        resampled = resampled[:expected_samples]
+    return np.clip(resampled, -32768, 32767).astype(np.int16).tobytes()
 
 
 def pcm16_24k_to_mulaw_8k(pcm_bytes: bytes) -> bytes:
@@ -67,7 +89,7 @@ def pcm16_24k_to_mulaw_8k(pcm_bytes: bytes) -> bytes:
         resampled = np.pad(resampled, (0, expected_samples - len(resampled)))
     elif len(resampled) > expected_samples:
         resampled = resampled[:expected_samples]
-    pcm_8k = resampled.astype(np.int16).tobytes()
+    pcm_8k = np.clip(resampled, -32768, 32767).astype(np.int16).tobytes()
     # Encode to mu-law
     return audioop.lin2ulaw(pcm_8k, 2)
 

src/eva/models/config.py

@@ -371,8 +371,17 @@ class OpenAIRealtimeSimulatorConfig(BaseModel):
     male_voice: str = Field("cedar", description="Voice used for male caller personas.")
 
 
+class GeminiLiveSimulatorConfig(BaseModel):
+    """Gemini Live-specific settings for the user simulator."""
+
+    provider: Literal["gemini_live"] = "gemini_live"
+    model: str = Field("gemini-2.5-flash-native-audio-preview-09-2025", description="Gemini Live model.")
+    female_voice: str = Field("Aoede", description="Voice used for female caller personas.")
+    male_voice: str = Field("Charon", description="Voice used for male caller personas.")
+
+
 UserSimulatorConfig = Annotated[
-    ElevenLabsSimulatorConfig | OpenAIRealtimeSimulatorConfig,
+    ElevenLabsSimulatorConfig | OpenAIRealtimeSimulatorConfig | GeminiLiveSimulatorConfig,
     Field(discriminator="provider"),
 ]
 
@@ -608,13 +617,13 @@ def _check_companion_services(self) -> "RunConfig":
         config is unused and conflicting env vars are harmless.
         """
         if (
-            isinstance(self.user_simulator, OpenAIRealtimeSimulatorConfig)
+            not isinstance(self.user_simulator, ElevenLabsSimulatorConfig)
             and self.perturbation is not None
             and self.perturbation.accent is not None
         ):
             raise ValueError(
-                "Accent perturbations require the ElevenLabs user simulator; "
-                "OpenAI Realtime supports behavior, noise, and connection perturbations."
+                f"Accent perturbations require the ElevenLabs user simulator; "
+                f"{self.user_simulator.provider} supports behavior, noise, and connection perturbations."
             )
 
         if self.max_rerun_attempts == 0 or self.aggregate_only:
@@ -717,12 +726,32 @@ def _check_language_personas(self) -> "RunConfig":
     @model_validator(mode="after")
     def _check_openai_realtime_simulator(self) -> "RunConfig":
         """When openai_realtime user simulator is selected, OPENAI_API_KEY must be present."""
+        if self.max_rerun_attempts == 0 or self.aggregate_only:
+            return self
         if not isinstance(self.user_simulator, OpenAIRealtimeSimulatorConfig):
             return self
         if not os.environ.get("OPENAI_API_KEY"):
             raise ValueError("EVA_USER_SIMULATOR__PROVIDER=openai_realtime requires OPENAI_API_KEY to be set.")
         return self
 
+    @model_validator(mode="after")
+    def _check_gemini_live_simulator(self) -> "RunConfig":
+        """When gemini_live user simulator is selected, Gemini credentials must be present."""
+        if self.max_rerun_attempts == 0 or self.aggregate_only:
+            return self
+        if not isinstance(self.user_simulator, GeminiLiveSimulatorConfig):
+            return self
+        has_credentials = any(
+            os.environ.get(key)
+            for key in ("GEMINI_API_KEY", "GOOGLE_API_KEY", "GOOGLE_CLOUD_PROJECT", "GOOGLE_APPLICATION_CREDENTIALS")
+        )
+        if not has_credentials:
+            raise ValueError(
+                "EVA_USER_SIMULATOR__PROVIDER=gemini_live requires one of GEMINI_API_KEY, GOOGLE_API_KEY, "
+                "GOOGLE_CLOUD_PROJECT (Vertex AI), or GOOGLE_APPLICATION_CREDENTIALS to be set."
+            )
+        return self
+
     @model_validator(mode="before")
     @classmethod
     def _handle_all_keyword(cls, data: Any):

src/eva/user_simulator/audio_bridge.py

@@ -14,15 +14,10 @@
 from collections.abc import Callable
 
 import websockets
-from websockets.protocol import State as WebSocketState
-
-try:
-    import audioop
-except ImportError:
-    import audioop_lts as audioop
-
 from elevenlabs.conversational_ai.conversation import AudioInterface
+from websockets.protocol import State as WebSocketState
 
+from eva.assistant.audio_bridge import pcm16_16k_to_mulaw_8k
 from eva.user_simulator.perturbation import AudioPerturbator
 from eva.utils.logging import get_logger
 
@@ -294,19 +289,7 @@ def _convert_pcm_to_mulaw(pcm_data: bytes) -> bytes:
             mulaw encoded audio data at 8kHz (for assistant)
         """
         try:
-            # Downsample from 16kHz to 8kHz
-            pcm_8khz, _ = audioop.ratecv(
-                pcm_data,
-                PCM_SAMPLE_WIDTH,  # 16-bit PCM
-                1,  # mono
-                ELEVENLABS_OUTPUT_RATE,  # from 16kHz
-                ASSISTANT_SAMPLE_RATE,  # to 8kHz
-                None,
-            )
-
-            # Convert 16-bit PCM to μ-law
-            mulaw_data = audioop.lin2ulaw(pcm_8khz, PCM_SAMPLE_WIDTH)
-            return mulaw_data
+            return pcm16_16k_to_mulaw_8k(pcm_data)
         except Exception as e:
             logger.warning(f"Error converting PCM to mulaw: {e}")
             return b""

src/eva/user_simulator/base.py

@@ -21,6 +21,21 @@
 
 _BEHAVIORS_PATH = Path(__file__).parent.parent.parent.parent / "configs" / "user_behaviors.yaml"
 
+# Shared across realtime-style callers (OpenAI Realtime, Gemini Live) that expose
+# an ``end_call`` function tool to let the simulated caller hang up.
+END_CALL_DESCRIPTION = """Use this to end the phone call and hang up.
+
+Call this function when it is time to end the call and one of the following is true:
+1. The agent has confirmed your request is resolved, all steps are completed, and you have said goodbye.
+2. The agent has initiated a transfer to a live agent.
+3. The agent has been unable to make progress for at least 5 consecutive turns.
+4. The agent says goodbye or indicates the conversation is over.
+5. The agent indicates that the remainder of your request cannot be fulfilled.
+6. The assistant reports an unrecoverable processing error.
+
+Never call this tool in the same turn that you provide the agent with data, an identifier,
+an approval to proceed, a transfer request, or any other information. Say a brief goodbye first."""
+
 
 @lru_cache(maxsize=1)
 def load_behavior_prompts() -> dict:

src/eva/user_simulator/factory.py

@@ -4,7 +4,12 @@
 
 from typing import Any
 
-from eva.models.config import ElevenLabsSimulatorConfig, OpenAIRealtimeSimulatorConfig, UserSimulatorConfig
+from eva.models.config import (
+    ElevenLabsSimulatorConfig,
+    GeminiLiveSimulatorConfig,
+    OpenAIRealtimeSimulatorConfig,
+    UserSimulatorConfig,
+)
 from eva.user_simulator.base import AbstractUserSimulator
 
 
@@ -21,4 +26,8 @@ def create_user_simulator(
         from eva.user_simulator.openai_realtime import OpenAIRealtimeUserSimulator
 
         return OpenAIRealtimeUserSimulator(simulator_config=simulator_config, **kwargs)
+    if isinstance(simulator_config, GeminiLiveSimulatorConfig):
+        from eva.user_simulator.gemini_live import GeminiLiveUserSimulator
+
+        return GeminiLiveUserSimulator(simulator_config=simulator_config, **kwargs)
     raise ValueError(f"Unknown user simulator provider: {simulator_config.provider!r}")