diff --git a/.env.example b/.env.example index 9f4edfaf..ab6ada69 100644 --- a/.env.example +++ b/.env.example @@ -97,22 +97,29 @@ EVA_MODEL__TTS=cartesia #x pipeline_mode=LLM,AudioLLM EVA_MODEL__TTS_PARAMS='{"api_key": "your_cartesia_api_key", "model": "sonic"}' +#i TTS gender for the voice pipeline. For use in gendered languages to ensure the LLM generates speech +#i which matches the gender of the TTS voice. +#d enum +#e M,F,none +#x pipeline_mode=LLM,AudioLLM +#v EVA_MODEL__ASSISTANT_GENDER=none + # --- LLM mode: cascade latency optimizations --- #i Prompt a model-generated lead-in before a tool call: 'off' or 'auto'. #d enum #e off,auto -#x pipeline_mode=LLM -# EVA_MODEL__PRE_TOOL_SPEECH=off +#x pipeline_mode=LLM,AudioLLM +#v EVA_MODEL__PRE_TOOL_SPEECH=off #i Stream Chat Completions output to TTS sentence-by-sentence; Responses API falls back with a warning. #d bool -#x pipeline_mode=LLM -# EVA_MODEL__LLM_STREAMING=false +#x pipeline_mode=LLM,AudioLLM +#v EVA_MODEL__LLM_STREAMING=false #i Forward provider parallel_tool_calls when tools are present; leave unset for defaults. #d bool -#x pipeline_mode=LLM -# EVA_MODEL__PARALLEL_TOOL_CALLS=false +#x pipeline_mode=LLM,AudioLLM +#v EVA_MODEL__PARALLEL_TOOL_CALLS=false # --- S2S mode --- #i Speech-to-speech model name. diff --git a/configs/prompts/simulation.yaml b/configs/prompts/simulation.yaml index 2c846a5c..4648cb62 100644 --- a/configs/prompts/simulation.yaml +++ b/configs/prompts/simulation.yaml @@ -69,9 +69,13 @@ agent: pre_tool_speech: | ## Responsiveness - Before calling a tool, say one brief sentence first. For lookups, use a short lead-in (e.g. "Let me pull that up."). Before any action that is irreversible or has real-world consequences, state what you'll do, get confirmation, then call the tool. + Before calling a tool, say one brief sentence first. For lookups, use a short lead-in (e.g. "Let me pull that up."). Before any action that is irreversible or has real-world consequences, state what you'll do, get confirmation, then call the tool. If calling multiple tools at once, only one lead-in is needed. + gender_instruction: | + ## Speaking Gender + You are speaking as a {gender_word} voice. In languages with grammatical gender, consistently use {gender_word} forms (verb conjugations, adjectives, self-references) for yourself throughout the conversation. + audio_llm_agent: system_prompt: | You are an AI voice assistant on a live phone call. diff --git a/src/eva/assistant/agentic/audio_llm_system.py b/src/eva/assistant/agentic/audio_llm_system.py index d272e8dd..694d3626 100644 --- a/src/eva/assistant/agentic/audio_llm_system.py +++ b/src/eva/assistant/agentic/audio_llm_system.py @@ -42,6 +42,7 @@ def __init__( pre_tool_speech: str = "off", llm_streaming: bool = False, full_audio_context: bool = False, + assistant_gender: str | None = None, ): super().__init__( current_date_time=current_date_time, @@ -52,6 +53,7 @@ def __init__( output_dir=output_dir, pre_tool_speech=pre_tool_speech, llm_streaming=llm_streaming, + assistant_gender=assistant_gender, ) self.alm_client: BaseALMClient = alm_client # When True, every user turn is sent as audio (full audio context). When False @@ -69,6 +71,12 @@ def __init__( # Reuse the shared pre-tool lead-in prompt appended to the system prompt. if self.pre_tool_speech == "auto": self.system_prompt += "\n\n" + self.prompt_manager.get_prompt("agent.pre_tool_speech") + # Reuse the shared gender instruction appended to the system prompt. + if self.assistant_gender is not None: + gender_word = "male" if self.assistant_gender == "M" else "female" + self.system_prompt += "\n\n" + self.prompt_manager.get_prompt( + "agent.gender_instruction", gender_word=gender_word + ) # Per-turn audio history: list of (audio_bytes, sample_rate) # One entry per user turn, aligned 1:1 with the user messages in the conversation history diff --git a/src/eva/assistant/agentic/system.py b/src/eva/assistant/agentic/system.py index e61530f3..fa268ce2 100644 --- a/src/eva/assistant/agentic/system.py +++ b/src/eva/assistant/agentic/system.py @@ -106,6 +106,7 @@ def __init__( output_dir: Path | None = None, # Output directory for performance stats pre_tool_speech: str = "off", llm_streaming: bool = False, + assistant_gender: str | None = None, ): """Initialize the agentic system. @@ -118,6 +119,8 @@ def __init__( output_dir: Optional output directory for saving performance stats pre_tool_speech: Lead-in mode ('off'|'auto') llm_streaming: Stream LLM output sentence-by-sentence + assistant_gender: Speaking gender for the assistant ('M'|'F'); None adds no + gender instruction (for gender-neutral languages like English) """ self.agent = agent self.tool_handler = tool_handler @@ -127,6 +130,7 @@ def __init__( self.current_date_time = current_date_time self.pre_tool_speech = pre_tool_speech self.llm_streaming = llm_streaming + self.assistant_gender = assistant_gender self._warned_responses_streaming_fallback = False self.prompt_manager = PromptManager() @@ -143,6 +147,11 @@ def __init__( ) if self.pre_tool_speech == "auto": self.system_prompt += "\n\n" + self.prompt_manager.get_prompt("agent.pre_tool_speech") + if self.assistant_gender is not None: + gender_word = "male" if self.assistant_gender == "M" else "female" + self.system_prompt += "\n\n" + self.prompt_manager.get_prompt( + "agent.gender_instruction", gender_word=gender_word + ) # Build tools for the LLM self.tools = agent.build_tools_for_agent() diff --git a/src/eva/assistant/pipecat_server.py b/src/eva/assistant/pipecat_server.py index dc8fa542..f9ebc6ee 100644 --- a/src/eva/assistant/pipecat_server.py +++ b/src/eva/assistant/pipecat_server.py @@ -391,6 +391,7 @@ async def _realtime_tool_handler(params) -> None: pre_tool_speech=self.pipeline_config.pre_tool_speech, llm_streaming=self.pipeline_config.llm_streaming, full_audio_context=self.pipeline_config.audio_llm_params.get("full_audio_context", False), + assistant_gender=self.pipeline_config.assistant_gender, ) audio_llm_processor.on_assistant_response = lambda msg: self._save_transcript_message_from_turn( role="assistant", content=msg, timestamp=self._current_iso_timestamp() @@ -444,6 +445,7 @@ async def on_user_transcription(text: str, timestamp: str, turn_id: int | None) output_dir=self.output_dir, pre_tool_speech=self.pipeline_config.pre_tool_speech, llm_streaming=self.pipeline_config.llm_streaming, + assistant_gender=self.pipeline_config.assistant_gender, ) async def on_assistant_response(msg: str) -> None: diff --git a/src/eva/assistant/pipeline/agent_processor.py b/src/eva/assistant/pipeline/agent_processor.py index a8f9cec5..6099aef7 100644 --- a/src/eva/assistant/pipeline/agent_processor.py +++ b/src/eva/assistant/pipeline/agent_processor.py @@ -65,6 +65,7 @@ def __init__( output_dir=None, pre_tool_speech: str = "off", llm_streaming: bool = False, + assistant_gender: str | None = None, **kwargs, ) -> None: """Initialize the agent processor. @@ -78,6 +79,8 @@ def __init__( output_dir: Optional output directory for saving performance stats pre_tool_speech: Lead-in mode ('off'|'auto') llm_streaming: Stream LLM output sentence-by-sentence + assistant_gender: Speaking gender for the assistant ('M'|'F'); None adds no + gender instruction **kwargs: Additional keyword arguments passed to FrameProcessor The turn-end fallback timer is hosted by ``UserObserver`` (on the pipeline spine), which @@ -101,6 +104,7 @@ def __init__( output_dir=output_dir, pre_tool_speech=pre_tool_speech, llm_streaming=llm_streaming, + assistant_gender=assistant_gender, ) # State tracking diff --git a/src/eva/assistant/pipeline/audio_llm_processor.py b/src/eva/assistant/pipeline/audio_llm_processor.py index 8bb05648..5024612d 100644 --- a/src/eva/assistant/pipeline/audio_llm_processor.py +++ b/src/eva/assistant/pipeline/audio_llm_processor.py @@ -198,6 +198,7 @@ def __init__( pre_tool_speech: str = "off", llm_streaming: bool = False, full_audio_context: bool = False, + assistant_gender: str | None = None, **kwargs, ) -> None: super().__init__(**kwargs) @@ -216,6 +217,7 @@ def __init__( pre_tool_speech=pre_tool_speech, llm_streaming=llm_streaming, full_audio_context=full_audio_context, + assistant_gender=assistant_gender, ) # State tracking (mirrors BenchmarkAgentProcessor) diff --git a/src/eva/models/config.py b/src/eva/models/config.py index 54d10834..26cfdc3c 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -211,6 +211,20 @@ def _normalize_pre_tool_speech(cls, value: str) -> str: None, description="Forward parallel_tool_calls when tools are present; None leaves provider defaults.", ) + assistant_gender: Literal["M", "F"] | None = Field( + None, + description=( + "Assistant speaking gender ('M' or 'F'), appended as a small system prompt " + "instruction so gendered languages (e.g. Hindi) produce grammatically consistent " + "speech. CASCADE/AUDIO_LLM only — S2S providers manage their own voice/gender. " + "None (default) adds no instruction, which is perfect for gender-neutral languages like English." + ), + ) + + @field_validator("assistant_gender", mode="before") + @classmethod + def _normalize_assistant_gender(cls, value: Any) -> Any: + return value.upper() if isinstance(value, str) else value @property def pipeline_type(self) -> "PipelineType": @@ -308,6 +322,11 @@ def _validate_latency_optimizations(self) -> "ModelConfig": "parallel_tool_calls applies only to the CASCADE pipeline; it will be ignored " f"for pipeline_type={self.pipeline_type}." ) + if self.assistant_gender is not None and self.pipeline_type == PipelineType.S2S: + logger.warning( + "assistant_gender applies only to CASCADE/AUDIO_LLM pipelines; it will be " + "ignored for pipeline_type=s2s (S2S providers manage their own voice/gender)." + ) return self