diff --git a/.env.example b/.env.example index 65edd67..9920cc4 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,10 @@ # Common settings AGENT_CORE_LLM_TIMEOUT_SECONDS=120 # AGENT_CORE_LLM_MAX_OUTPUT_TOKENS=4096 +# AGENT_CORE_MODEL_BACKEND=native +# AGENT_CORE_AGENT_KERNEL_BACKEND=native +# LangSmith export is disabled by default even if LANGSMITH_TRACING=true. +# AGENT_CORE_LANGCHAIN_TRACING_ENABLED=false # OpenAI LLM_PROVIDER=openai @@ -10,6 +14,7 @@ AGENT_CORE_MEMORY_MODEL=gpt-4.1-mini # Optional dedicated provider for memory/internal synthesis. # If omitted, memory synthesis uses the main provider with AGENT_CORE_MEMORY_MODEL. +# AGENT_CORE_MEMORY_MODEL_BACKEND=native # # Example: main assistant on Azure Anthropic, memory synthesis on Azure OpenAI. # AGENT_CORE_MEMORY_LLM_PROVIDER=azure_openai @@ -25,6 +30,7 @@ AGENT_CORE_MEMORY_MODEL=gpt-4.1-mini # Azure OpenAI # LLM_PROVIDER=azure_openai +# AGENT_CORE_MODEL_BACKEND=langchain # AZURE_OPENAI_ENDPOINT=https://.openai.azure.com # AZURE_OPENAI_API_KEY= # AGENT_CORE_MODEL= diff --git a/CHANGELOG.md b/CHANGELOG.md index 55c0143..daeb290 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,22 @@ ## Unreleased +- Added opt-in LangGraph kernels for direct, investigate, and deep-investigate + conversation turns behind `CoreSettings.agent_kernel_backend`, using typed + internal state, versioned pending graph cursors, and shared native/LangGraph + operations while preserving agent-core persistence, budgets, artifacts, + traces, and memory contracts. Added deterministic contract parity and opt-in + paired real-model kernel evals for tools, pending resume, structured output, + investigation synthesis, critique, tokens, latency, persistence, and traces. +- Added an opt-in LangChain model backend for Azure OpenAI behind the existing + `BaseLLMProvider` contract, with shared request normalization, adaptive retry, + token usage, tool-call, request-id, and primary/memory provider semantics. +- Fixed the provider compatibility quickstart to consume typed + `LLMCompletionResult` values for plain-text and JSON Schema checks. +- Added persisted `model_backend` telemetry, explicit opt-in LangSmith tracing, + broader Azure provider contract coverage, and an opt-in paid `live_llm` test + matrix for native and LangChain model invocation. + - Replaced overflow-driven conversation summaries and separately synthesized task state with append-only `ExchangeMemory` and `TurnMemory` journals plus a deterministic, rebuildable `SessionView`. diff --git a/README.md b/README.md index 4330084..83e61d0 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,93 @@ Or run a small REPL: .venv/bin/python examples/quickstart.py --interactive ``` +### Optional LangChain model backend for Azure OpenAI + +Azure OpenAI can use LangChain for model invocation while the agent loop, +tools, policies, checkpoints, memory lifecycle, and public provider contract +remain owned by agent-core. The native SDK backend remains the default. + +```bash +LLM_PROVIDER=azure_openai +AGENT_CORE_MODEL_BACKEND=langchain +AZURE_OPENAI_ENDPOINT=https://.openai.azure.com +AZURE_OPENAI_API_KEY=... +AZURE_OPENAI_API_VERSION=2025-01-01-preview +AGENT_CORE_MODEL= +AGENT_CORE_MEMORY_MODEL= + +.venv/bin/python examples/quickstart.py --compat-check +``` + +`AGENT_CORE_MEMORY_MODEL_BACKEND` can override the backend for a dedicated +memory provider. If it is omitted, memory synthesis uses the primary backend; +`native` and `langchain` can therefore be compared without changing the agent +or memory contracts. The LangChain backend currently supports Azure OpenAI +only. + +LangSmith tracing is forcibly disabled around LangChain model calls by default, +including when the process inherits `LANGSMITH_TRACING=true`. A host that has +reviewed its data-handling requirements can explicitly opt in with +`AGENT_CORE_LANGCHAIN_TRACING_ENABLED=true`. + +Provider telemetry keeps `provider="azure_openai"` stable and records the +implementation separately as `model_backend="native"` or `"langchain"` in +completion records, run results, structured-task checkpoints, and conversation +trace response events. + +### Optional LangGraph conversation-agent kernel + +The conversation orchestrator can use LangGraph for the internal control flow +of `direct`, `investigate`, and `deep_investigate` turns. This switch is +independent from the model backend: native or LangChain model invocation can be +used with either agent kernel. + +```bash +AGENT_CORE_AGENT_KERNEL_BACKEND=langgraph +``` + +The default remains `native`. LangGraph owns the direct model/tool loop plus the +planning, assistant, tool, reflection/decision, critique, and terminal routes +for investigation modes. The existing controller operations still implement +the domain semantics shared by both kernels. Session persistence, pending-tool +payloads, resume idempotence, budgets, artifacts, traces, and memory commits +remain owned by agent-core. A versioned graph cursor is stored in pending +payloads, while the LangGraph graphs are intentionally compiled without their +own durable checkpointer to avoid two competing state stores. + +The same explicit LangSmith privacy boundary applies to graph execution: +tracing stays disabled unless `AGENT_CORE_LANGCHAIN_TRACING_ENABLED=true`, even +if the process inherits `LANGSMITH_TRACING=true`. + +See [docs/langgraph_migration.md](docs/langgraph_migration.md) for the state and +transition map, current ownership boundaries, and the remaining full-graph +migration work. + +Paid Azure integration tests are excluded from normal test runs. To execute the +same behavioral matrix against both backends: + +```bash +AGENT_CORE_RUN_LIVE_LLM_TESTS=1 \ +AGENT_CORE_LIVE_LLM_MODEL=gpt-5.4-mini \ +AZURE_OPENAI_ENDPOINT=https://.openai.azure.com \ +AZURE_OPENAI_API_KEY=... \ +.venv/bin/python -m pytest -m live_llm -q -s +``` + +Use `AGENT_CORE_LIVE_LLM_BACKENDS=native` or `langchain` to test only one +implementation. The live suite covers reasoning and usage metadata, strict JSON +Schema output, a complete tool-result roundtrip, and `StructuredTaskRunner`. +When the LangChain model backend is selected, it also compares native and +LangGraph direct-agent kernels through the full conversation tool loop, +validates pending/resume for both kernels, and runs paired investigation and +deep-investigation scenarios. The extended matrix covers competing-tool +selection, structured final output, initial planning, reflection, decision, +final critique, and final synthesis. Lines prefixed with +`LIVE_KERNEL_EVAL` report per-kernel wall/provider latency, model-call count, +token usage, selected tools, persistence projections, and trace-event paths. +Latency and token deltas are observations rather than pass/fail thresholds; +repeat the paid suite before drawing performance conclusions from them. + See [examples/README.md](examples/README.md) for the pending tool result and resume example. diff --git a/agent_core/agent_graph/__init__.py b/agent_core/agent_graph/__init__.py new file mode 100644 index 0000000..f36e1b1 --- /dev/null +++ b/agent_core/agent_graph/__init__.py @@ -0,0 +1,7 @@ +"""Internal agent-kernel implementations. + +This package is deliberately not re-exported from :mod:`agent_core`. Its +types describe the implementation state of the conversation adapter, not a +public persistence or extension contract. +""" + diff --git a/agent_core/agent_graph/direct.py b/agent_core/agent_graph/direct.py new file mode 100644 index 0000000..3f98d1a --- /dev/null +++ b/agent_core/agent_graph/direct.py @@ -0,0 +1,464 @@ +from __future__ import annotations + +from typing import Any, Literal, Protocol, cast + +from agent_core.agent_graph.state import ( + AgentGraphState, + AgentGraphUpdate, + build_graph_checkpoint, + normalize_agent_kernel_backend, +) +from agent_core.execution_context import ExecutionContext +from agent_core.llm.base import LLMCompletionResult, LLMMessage +from agent_core.llm.errors import LLMProviderError +from agent_core.logging_utils import get_logger, safe_preview +from agent_core.run_trace import RunTrace +from agent_core.settings import CoreSettings +from agent_core.tool_artifacts import active_tool_artifact_runtime +from agent_core.turn_steps import ToolExecutionStepResult +from agent_core.types import AgentTurnResult, ToolExecutionStatus + +logger = get_logger("core.agent_graph.direct") + +AgentKernelBackend = Literal["native", "langgraph"] +AfterModelRoute = Literal["execute_tools", "complete_response", "end"] +AfterToolsRoute = Literal["call_model", "complete_budget", "end"] + + +class DirectTurnOperations(Protocol): + """Existing agent-core effects used by either control-flow backend.""" + + settings: CoreSettings + + def _estimate_prompt_tokens(self, *, messages: list[LLMMessage]) -> int: ... + + def _record_trace_event( + self, + trace: RunTrace | None, + *, + event_type: str, + summary: str, + iteration: int | None = None, + payload: dict[str, Any] | None = None, + related_tool_call_id: str | None = None, + ) -> None: ... + + def _call_model_once(self, *, messages: list[LLMMessage]) -> LLMCompletionResult: ... + + def _handle_provider_failure( + self, + *, + error: LLMProviderError, + user_input: str, + turn_index: int, + ) -> AgentTurnResult: ... + + def _persist_conversation_turn( + self, + *, + turn_index: int, + user_input: str, + assistant_content: str, + provider_failure: bool = False, + ) -> None: ... + + def _refresh_memory_after_turn(self, *, turn_index: int) -> None: ... + + def _execute_tool_calls_once( + self, + *, + user_input: str, + session_id: str, + context: ExecutionContext, + messages: list[LLMMessage], + turn_index: int, + exchange_index: int, + tool_calls_used: int, + assistant_message: LLMMessage, + max_tool_calls: int, + start_tool_call_index: int = 0, + existing_tool_messages: list[LLMMessage] | None = None, + existing_tool_statuses: list[ToolExecutionStatus] | None = None, + existing_tool_names: list[str] | None = None, + reuse_exchange_index: bool = False, + pending_metadata_extra: dict[str, Any] | None = None, + trace: RunTrace | None = None, + ) -> ToolExecutionStepResult: ... + + +class DirectTurnNodes: + """Behavior shared by the native loop and LangGraph transitions.""" + + def __init__(self, operations: DirectTurnOperations) -> None: + self.operations = operations + + def initial_state( + self, + *, + user_input: str, + session_id: str, + context: ExecutionContext, + messages: list[LLMMessage], + turn_index: int, + tool_calls_used: int, + exchange_index: int, + trace: RunTrace | None, + resume_tool_step: ToolExecutionStepResult | None = None, + ) -> AgentGraphState: + return AgentGraphState( + user_input=user_input, + session_id=session_id, + context=context, + messages=list(messages), + turn_index=turn_index, + tool_calls_used=tool_calls_used, + exchange_index=exchange_index, + trace=trace, + start_prompt_tokens=self.operations._estimate_prompt_tokens(messages=messages), + tool_loop_reserve_tokens=max(1, self.operations.settings.max_active_context_tokens), + prompt_reserve_warning_emitted=False, + model_call_index=0, + assistant_message=None, + tool_step=resume_tool_step, + result=resume_tool_step.pending_result if resume_tool_step is not None else None, + entrypoint="after_tools" if resume_tool_step is not None else "model", + ) + + @staticmethod + def route_entry(state: AgentGraphState) -> AfterToolsRoute: + if state["entrypoint"] == "model": + return "call_model" + return DirectTurnNodes.route_after_tools(state) + + def call_model(self, state: AgentGraphState) -> AgentGraphUpdate: + messages = list(state["messages"]) + model_call_index = state["model_call_index"] + 1 + prompt_tokens = self.operations._estimate_prompt_tokens(messages=messages) + prompt_growth_tokens = max(0, prompt_tokens - state["start_prompt_tokens"]) + warning_emitted = state["prompt_reserve_warning_emitted"] + logger.debug( + "Calling LLM", + extra={ + "model": self.operations.settings.model, + "message_count": len(messages), + "estimated_prompt_tokens": prompt_tokens, + "start_turn_prompt_tokens": state["start_prompt_tokens"], + "tool_loop_reserve_tokens": state["tool_loop_reserve_tokens"], + "prompt_growth_tokens": prompt_growth_tokens, + }, + ) + if ( + state["tool_calls_used"] > 0 + and prompt_growth_tokens >= state["tool_loop_reserve_tokens"] + and not warning_emitted + ): + logger.warning( + "Tool loop consumed the start-turn prompt reserve", + extra={ + "estimated_prompt_tokens": prompt_tokens, + "start_turn_prompt_tokens": state["start_prompt_tokens"], + "prompt_growth_tokens": prompt_growth_tokens, + "tool_loop_reserve_tokens": state["tool_loop_reserve_tokens"], + "tool_calls_used": state["tool_calls_used"], + }, + ) + warning_emitted = True + + self.operations._record_trace_event( + state["trace"], + event_type="llm_call_started", + summary="LLM call started", + iteration=model_call_index, + payload={ + "message_count": len(messages), + "estimated_prompt_tokens": prompt_tokens, + "tool_calls_used": state["tool_calls_used"], + "exchange_index": state["exchange_index"], + }, + ) + try: + llm_response = self.operations._call_model_once(messages=messages) + except LLMProviderError as exc: + self.operations._record_trace_event( + state["trace"], + event_type="llm_provider_failure", + summary="LLM provider failure handled", + iteration=model_call_index, + payload={ + "kind": exc.kind, + "detail_preview": safe_preview(exc.detail or exc.user_message, limit=200), + }, + ) + return { + "model_call_index": model_call_index, + "prompt_reserve_warning_emitted": warning_emitted, + "assistant_message": None, + "tool_step": None, + "result": self.operations._handle_provider_failure( + error=exc, + user_input=state["user_input"], + turn_index=state["turn_index"], + ), + } + + logger.debug( + "Received LLM response", + extra={ + "content_length": len(llm_response.content), + "tool_call_count": len(llm_response.tool_calls), + "provider": llm_response.provider, + "model_backend": llm_response.model_backend, + "model": llm_response.model, + "provider_attempts": llm_response.provider_attempts, + }, + ) + assistant_message = LLMMessage( + role="assistant", + content=llm_response.content, + tool_calls=list(llm_response.tool_calls), + ) + messages.append(assistant_message) + self.operations._record_trace_event( + state["trace"], + event_type="assistant_response_received", + summary="Assistant response received", + iteration=model_call_index, + payload={ + "content_length": len(llm_response.content), + "tool_call_count": len(llm_response.tool_calls), + "tool_calls": [ + {"id": tool_call.id, "name": tool_call.name} for tool_call in llm_response.tool_calls + ], + "provider": llm_response.provider, + "model_backend": llm_response.model_backend, + "model": llm_response.model, + "provider_request_id": llm_response.provider_request_id, + "provider_attempts": llm_response.provider_attempts, + "usage": llm_response.usage.to_dict() if llm_response.usage is not None else None, + }, + ) + return { + "messages": messages, + "model_call_index": model_call_index, + "prompt_reserve_warning_emitted": warning_emitted, + "assistant_message": assistant_message, + "tool_step": None, + "result": None, + } + + @staticmethod + def route_after_model(state: AgentGraphState) -> AfterModelRoute: + if state["result"] is not None: + return "end" + assistant_message = state["assistant_message"] + if assistant_message is None: + raise RuntimeError("Direct agent graph has no assistant message after a successful model call") + return "execute_tools" if assistant_message.tool_calls else "complete_response" + + def complete_response(self, state: AgentGraphState) -> AgentGraphUpdate: + assistant_message = state["assistant_message"] + if assistant_message is None: + raise RuntimeError("Direct agent graph cannot complete without an assistant message") + self.operations._persist_conversation_turn( + turn_index=state["turn_index"], + user_input=state["user_input"], + assistant_content=assistant_message.content, + ) + self.operations._refresh_memory_after_turn(turn_index=state["turn_index"]) + logger.info("Completing run_turn without additional tool calls") + return {"result": AgentTurnResult(status="completed", content=assistant_message.content)} + + def execute_tools(self, state: AgentGraphState) -> AgentGraphUpdate: + assistant_message = state["assistant_message"] + if assistant_message is None: + raise RuntimeError("Direct agent graph cannot execute tools without an assistant message") + tool_step = self.operations._execute_tool_calls_once( + user_input=state["user_input"], + session_id=state["session_id"], + context=state["context"], + messages=list(state["messages"]), + turn_index=state["turn_index"], + exchange_index=state["exchange_index"], + tool_calls_used=state["tool_calls_used"], + assistant_message=assistant_message, + max_tool_calls=self.operations.settings.max_tool_calls_per_turn, + pending_metadata_extra={ + "agent_graph_checkpoint": build_graph_checkpoint( + graph="direct", + backend=normalize_agent_kernel_backend(self.operations.settings.agent_kernel_backend), + resume_node="resume_tool_exchange", + ) + }, + trace=state["trace"], + ) + return { + "messages": tool_step.messages, + "exchange_index": tool_step.exchange_index, + "tool_calls_used": tool_step.tool_calls_used, + "tool_step": tool_step, + "result": tool_step.pending_result, + } + + @staticmethod + def route_after_tools(state: AgentGraphState) -> AfterToolsRoute: + tool_step = state["tool_step"] + if tool_step is None: + raise RuntimeError("Direct agent graph has no tool step after tool execution") + if tool_step.pending_result is not None: + return "end" + if tool_step.budget_exhausted: + return "complete_budget" + return "call_model" + + def complete_budget(self, state: AgentGraphState) -> AgentGraphUpdate: + message = "Maximum number of tool calls reached for this turn." + logger.error(message) + self.operations._record_trace_event( + state["trace"], + event_type="tool_budget_exhausted", + summary=message, + iteration=state["model_call_index"], + payload={"tool_calls_used": state["tool_calls_used"]}, + ) + self.operations._persist_conversation_turn( + turn_index=state["turn_index"], + user_input=state["user_input"], + assistant_content=message, + ) + self.operations._refresh_memory_after_turn(turn_index=state["turn_index"]) + return {"result": AgentTurnResult(status="completed", content=message)} + + +class DirectTurnKernel(Protocol): + backend: AgentKernelBackend + + def run(self, initial_state: AgentGraphState) -> AgentTurnResult: ... + + +class NativeDirectTurnKernel: + backend: AgentKernelBackend = "native" + + def __init__(self, nodes: DirectTurnNodes) -> None: + self.nodes = nodes + + def run(self, initial_state: AgentGraphState) -> AgentTurnResult: + state = initial_state + entry_route = self.nodes.route_entry(state) + if entry_route == "end": + result = state["result"] + if result is None: + raise RuntimeError("Native direct agent kernel resumed without a result") + return result + if entry_route == "complete_budget": + state.update(self.nodes.complete_budget(state)) + result = state["result"] + if result is None: + raise RuntimeError("Native direct agent kernel budget completion produced no result") + return result + + while True: + state.update(self.nodes.call_model(state)) + model_route = self.nodes.route_after_model(state) + if model_route == "end": + break + if model_route == "complete_response": + state.update(self.nodes.complete_response(state)) + break + + state.update(self.nodes.execute_tools(state)) + tools_route = self.nodes.route_after_tools(state) + if tools_route == "end": + break + if tools_route == "complete_budget": + state.update(self.nodes.complete_budget(state)) + break + + result = state["result"] + if result is None: + raise RuntimeError("Native direct agent kernel completed without a result") + return result + + +class LangGraphDirectTurnKernel: + backend: AgentKernelBackend = "langgraph" + + def __init__(self, nodes: DirectTurnNodes) -> None: + from langgraph.graph import END, START, StateGraph + + self.nodes = nodes + builder = StateGraph(AgentGraphState) + builder.add_node("call_model", nodes.call_model) + builder.add_node("complete_response", nodes.complete_response) + builder.add_node("execute_tools", nodes.execute_tools) + builder.add_node("complete_budget", nodes.complete_budget) + builder.add_conditional_edges( + START, + nodes.route_entry, + { + "call_model": "call_model", + "complete_budget": "complete_budget", + "end": END, + }, + ) + builder.add_conditional_edges( + "call_model", + nodes.route_after_model, + { + "execute_tools": "execute_tools", + "complete_response": "complete_response", + "end": END, + }, + ) + builder.add_conditional_edges( + "execute_tools", + nodes.route_after_tools, + { + "call_model": "call_model", + "complete_budget": "complete_budget", + "end": END, + }, + ) + builder.add_edge("complete_response", END) + builder.add_edge("complete_budget", END) + self.graph = builder.compile() + + def run(self, initial_state: AgentGraphState) -> AgentTurnResult: + import langsmith as ls + + # A model/tool exchange consumes two graph supersteps. Keep the limit + # proportional to the configured tool budget instead of LangGraph's + # low generic default. + artifact_runtime = active_tool_artifact_runtime() + max_internal_tool_calls = artifact_runtime.policy.max_reads_per_run if artifact_runtime is not None else 0 + recursion_limit = max( + 25, + ((self.nodes.operations.settings.max_tool_calls_per_turn + max_internal_tool_calls) * 2) + 8, + ) + # Graph execution can contain the complete transcript in its state. + # Reuse the explicit LangSmith opt-in from the LangChain model backend + # instead of inheriting process-wide LANGSMITH_TRACING implicitly. + with ls.tracing_context(enabled=self.nodes.operations.settings.langchain_tracing_enabled): + final_state = cast( + AgentGraphState, + self.graph.invoke(initial_state, {"recursion_limit": recursion_limit}), + ) + result = final_state["result"] + if result is None: + raise RuntimeError("LangGraph direct agent kernel completed without a result") + return result + + +def build_direct_turn_kernel( + *, + backend: str, + operations: DirectTurnOperations, +) -> tuple[DirectTurnNodes, DirectTurnKernel]: + normalized = normalize_agent_kernel_backend(backend) + nodes = DirectTurnNodes(operations) + if normalized == "native": + return nodes, NativeDirectTurnKernel(nodes) + if normalized == "langgraph": + return nodes, LangGraphDirectTurnKernel(nodes) + raise ValueError( + f"Unsupported agent kernel backend: {backend!r}. Expected 'native' or 'langgraph'." + ) diff --git a/agent_core/agent_graph/investigation.py b/agent_core/agent_graph/investigation.py new file mode 100644 index 0000000..8917327 --- /dev/null +++ b/agent_core/agent_graph/investigation.py @@ -0,0 +1,647 @@ +from __future__ import annotations + +from dataclasses import asdict +from typing import Any, Literal, Protocol, cast + +from agent_core.agent_graph.state import ( + InvestigationGraphState, + InvestigationGraphUpdate, + build_graph_checkpoint, + normalize_agent_kernel_backend, +) +from agent_core.execution_context import ExecutionContext +from agent_core.investigation_state import InvestigationState +from agent_core.llm.base import LLMCallOptions, LLMCompletionResult, LLMMessage +from agent_core.llm.errors import LLMProviderError +from agent_core.logging_utils import get_logger, safe_preview +from agent_core.run_options import RunOptions +from agent_core.settings import CoreSettings +from agent_core.tool_artifacts import active_tool_artifact_runtime +from agent_core.turn_steps import PendingResumeState, ToolExecutionStepResult +from agent_core.types import AgentTurnResult + +logger = get_logger("core.agent_graph.investigation") + +EntryRoute = Literal["initialize_plan", "reflect_decide"] +InitializeRoute = Literal["assistant_step", "end"] +AssistantRoute = Literal["execute_tools", "handle_final_draft", "complete_max_tools", "end"] +ToolRoute = Literal["reflect_decide", "complete_max_tools", "end"] +ContinueRoute = Literal["assistant_step", "complete_max_iterations", "end"] + + +def _llm_failure_stop_reason(error: LLMProviderError) -> str: + if error.kind == "budget_exhausted": + return "llm_budget_exhausted" + if error.kind == "context_overflow": + return "llm_context_overflow" + return "provider_failure" + + +class InvestigationOperations(Protocol): + settings: CoreSettings + + def _record_event( + self, + *, + event_type: str, + summary: str, + iteration: int | None = None, + payload: dict[str, Any] | None = None, + related_tool_call_id: str | None = None, + ) -> None: ... + + def _synthesize_initial_plan( + self, + *, + user_input: str, + state: InvestigationState, + options: RunOptions, + ) -> InvestigationState: ... + + def _messages_with_iteration_state( + self, + *, + messages: list[LLMMessage], + state: InvestigationState, + iteration: int, + ) -> list[LLMMessage]: ... + + def _call_options(self, *, options: RunOptions, target: str) -> LLMCallOptions: ... + + def call_model_once( + self, + *, + messages: list[LLMMessage], + options: LLMCallOptions | None = None, + ) -> LLMCompletionResult: ... + + def handle_provider_failure( + self, + *, + error: LLMProviderError, + user_input: str, + turn_index: int, + ) -> AgentTurnResult: ... + + def _attach_metadata( + self, + result: AgentTurnResult, + *, + options: RunOptions, + iterations_used: int, + tool_calls_used: int, + stop_reason: str, + state: InvestigationState, + ) -> AgentTurnResult: ... + + def execute_tool_calls_once( + self, + *, + user_input: str, + session_id: str, + context: ExecutionContext, + messages: list[LLMMessage], + turn_index: int, + exchange_index: int, + tool_calls_used: int, + assistant_message: LLMMessage, + max_tool_calls: int, + pending_metadata_extra: dict[str, Any] | None = None, + ) -> ToolExecutionStepResult: ... + + def _reflect_and_decide_after_tools( + self, + *, + user_input: str, + turn_index: int, + options: RunOptions, + state: InvestigationState, + messages: list[LLMMessage], + tool_step: ToolExecutionStepResult, + iterations_used: int, + no_progress_iterations: int, + ) -> tuple[AgentTurnResult | None, int]: ... + + def _evaluate_final_draft( + self, + *, + user_input: str, + messages: list[LLMMessage], + turn_index: int, + options: RunOptions, + state: InvestigationState, + final_draft: str, + iterations_used: int, + tool_calls_used: int, + ) -> AgentTurnResult | None: ... + + def _complete_with_budget_answer( + self, + *, + user_input: str, + turn_index: int, + options: RunOptions, + state: InvestigationState, + messages: list[LLMMessage], + iterations_used: int, + tool_calls_used: int, + stop_reason: str, + ) -> AgentTurnResult: ... + + +class InvestigationTurnNodes: + """Multi-node investigate/deep-investigate control flow.""" + + def __init__(self, operations: InvestigationOperations) -> None: + self.operations = operations + + def initial_state( + self, + *, + user_input: str, + session_id: str, + context: ExecutionContext, + messages: list[LLMMessage], + turn_index: int, + options: RunOptions, + investigation_state: InvestigationState | None = None, + iterations_used: int = 0, + tool_calls_used: int = 0, + exchange_index: int = 0, + no_progress_iterations: int = 0, + resume_tool_step: ToolExecutionStepResult | None = None, + ) -> InvestigationGraphState: + return InvestigationGraphState( + user_input=user_input, + session_id=session_id, + context=context, + messages=list(messages), + turn_index=turn_index, + options=options, + investigation_state=investigation_state or InvestigationState.create_template(objective=user_input), + iterations_used=iterations_used, + tool_calls_used=tool_calls_used, + exchange_index=exchange_index, + no_progress_iterations=no_progress_iterations, + assistant_message=None, + tool_step=resume_tool_step, + final_draft=None, + result=resume_tool_step.pending_result if resume_tool_step is not None else None, + entrypoint="after_tools" if resume_tool_step is not None else "initialize", + ) + + @staticmethod + def route_entry(state: InvestigationGraphState) -> EntryRoute: + return "reflect_decide" if state["entrypoint"] == "after_tools" else "initialize_plan" + + def initialize_plan(self, state: InvestigationGraphState) -> InvestigationGraphUpdate: + options = state["options"] + investigation_state = state["investigation_state"] + if not options.require_initial_plan: + return {"result": None} + + self.operations._record_event( + event_type="initial_plan_started", + summary="Initial investigation plan synthesis started", + payload={"mode": options.mode}, + ) + try: + investigation_state = self.operations._synthesize_initial_plan( + user_input=state["user_input"], + state=investigation_state, + options=options, + ) + except LLMProviderError as exc: + self.operations._record_event( + event_type="llm_provider_failure", + summary="Initial plan provider failure handled", + payload={"kind": exc.kind}, + ) + failure_result = self.operations.handle_provider_failure( + error=exc, + user_input=state["user_input"], + turn_index=state["turn_index"], + ) + return { + "result": self.operations._attach_metadata( + failure_result, + options=options, + iterations_used=0, + tool_calls_used=0, + stop_reason=_llm_failure_stop_reason(exc), + state=investigation_state, + ) + } + except ValueError as exc: + if not options.recover_internal_synthesis_errors: + raise + logger.warning( + "Initial investigation plan synthesis failed; continuing with template state", + extra={"error_preview": safe_preview(str(exc), limit=200)}, + ) + investigation_state.metadata["initial_plan_synthesis_error"] = safe_preview(str(exc), limit=200) + self.operations._record_event( + event_type="structured_synthesis_recovered", + summary="Initial plan synthesis failed; continuing with template investigation state", + payload={ + "target": "investigation_initial_plan", + "error_preview": investigation_state.metadata["initial_plan_synthesis_error"], + }, + ) + else: + self.operations._record_event( + event_type="initial_plan_created", + summary="Initial investigation plan created", + payload={"investigation_state": investigation_state.compact_summary()}, + ) + return {"investigation_state": investigation_state, "result": None} + + @staticmethod + def route_after_initialize(state: InvestigationGraphState) -> InitializeRoute: + return "end" if state["result"] is not None else "assistant_step" + + def assistant_step(self, state: InvestigationGraphState) -> InvestigationGraphUpdate: + options = state["options"] + iterations_used = state["iterations_used"] + 1 + messages = list(state["messages"]) + investigation_state = state["investigation_state"] + self.operations._record_event( + event_type="investigation_iteration_started", + summary="Investigation iteration started", + iteration=iterations_used, + payload={ + "tool_calls_used": state["tool_calls_used"], + "max_iterations": options.max_iterations, + "max_tool_calls": options.max_tool_calls, + }, + ) + try: + assistant_messages = self.operations._messages_with_iteration_state( + messages=messages, + state=investigation_state, + iteration=iterations_used, + ) + llm_response = self.operations.call_model_once( + messages=assistant_messages, + options=self.operations._call_options(options=options, target="assistant_step"), + ) + except LLMProviderError as exc: + self.operations._record_event( + event_type="llm_provider_failure", + summary="Assistant step provider failure handled", + iteration=iterations_used, + payload={"kind": exc.kind}, + ) + failure_result = self.operations.handle_provider_failure( + error=exc, + user_input=state["user_input"], + turn_index=state["turn_index"], + ) + return { + "iterations_used": iterations_used, + "result": self.operations._attach_metadata( + failure_result, + options=options, + iterations_used=iterations_used, + tool_calls_used=state["tool_calls_used"], + stop_reason=_llm_failure_stop_reason(exc), + state=investigation_state, + ), + } + + assistant_message = LLMMessage( + role="assistant", + content=llm_response.content, + tool_calls=list(llm_response.tool_calls), + ) + messages.append(assistant_message) + self.operations._record_event( + event_type="assistant_step_completed", + summary="Assistant investigation step completed", + iteration=iterations_used, + payload={ + "content_length": len(llm_response.content), + "tool_call_count": len(llm_response.tool_calls), + "tool_calls": [ + {"id": tool_call.id, "name": tool_call.name} for tool_call in llm_response.tool_calls + ], + }, + ) + final_draft = None + if not llm_response.tool_calls: + final_draft = llm_response.content + self.operations._record_event( + event_type="final_draft_received", + summary="Assistant produced a final draft", + iteration=iterations_used, + payload={"content_length": len(llm_response.content)}, + ) + return { + "messages": messages, + "iterations_used": iterations_used, + "assistant_message": assistant_message, + "tool_step": None, + "final_draft": final_draft, + "result": None, + } + + @staticmethod + def route_after_assistant(state: InvestigationGraphState) -> AssistantRoute: + if state["result"] is not None: + return "end" + assistant_message = state["assistant_message"] + if assistant_message is None: + raise RuntimeError("Investigation graph has no assistant message") + if not assistant_message.tool_calls: + return "handle_final_draft" + artifact_runtime = active_tool_artifact_runtime() + only_internal_calls = ( + artifact_runtime is not None + and all(artifact_runtime.is_internal_tool(tool_call.name) for tool_call in assistant_message.tool_calls) + ) + if state["tool_calls_used"] >= state["options"].max_tool_calls and not only_internal_calls: + return "complete_max_tools" + return "execute_tools" + + def execute_tools(self, state: InvestigationGraphState) -> InvestigationGraphUpdate: + assistant_message = state["assistant_message"] + if assistant_message is None: + raise RuntimeError("Investigation graph cannot execute tools without an assistant message") + options = state["options"] + investigation_state = state["investigation_state"] + tool_step = self.operations.execute_tool_calls_once( + user_input=state["user_input"], + session_id=state["session_id"], + context=state["context"], + messages=list(state["messages"]), + turn_index=state["turn_index"], + exchange_index=state["exchange_index"], + tool_calls_used=state["tool_calls_used"], + assistant_message=assistant_message, + max_tool_calls=options.max_tool_calls, + pending_metadata_extra={ + "mode": options.mode, + "run_options": asdict(options), + "investigation_state": investigation_state.to_dict(), + "iterations_used": state["iterations_used"], + "no_progress_iterations": state["no_progress_iterations"], + "agent_graph_checkpoint": build_graph_checkpoint( + graph="investigation", + backend=normalize_agent_kernel_backend(self.operations.settings.agent_kernel_backend), + resume_node="resume_tool_exchange", + ), + }, + ) + self.operations._record_event( + event_type="tool_step_completed", + summary="Investigation tool step completed", + iteration=state["iterations_used"], + payload={ + "tool_names": list(tool_step.tool_names), + "tool_statuses": list(tool_step.tool_statuses), + "tool_calls_used": tool_step.tool_calls_used, + "budget_exhausted": tool_step.budget_exhausted, + "pending": tool_step.pending_result is not None, + }, + ) + result = tool_step.pending_result + if result is not None: + result = self.operations._attach_metadata( + result, + options=options, + iterations_used=state["iterations_used"], + tool_calls_used=tool_step.tool_calls_used, + stop_reason="pending_tool_result", + state=investigation_state, + ) + return { + "messages": tool_step.messages, + "exchange_index": tool_step.exchange_index, + "tool_calls_used": tool_step.tool_calls_used, + "tool_step": tool_step, + "result": result, + } + + @staticmethod + def route_after_tools(state: InvestigationGraphState) -> ToolRoute: + tool_step = state["tool_step"] + if tool_step is None: + raise RuntimeError("Investigation graph has no completed tool step") + if state["result"] is not None: + return "end" + if tool_step.budget_exhausted: + return "complete_max_tools" + return "reflect_decide" + + def reflect_decide(self, state: InvestigationGraphState) -> InvestigationGraphUpdate: + tool_step = state["tool_step"] + if tool_step is None: + raise RuntimeError("Investigation graph cannot reflect without a tool step") + result, no_progress_iterations = self.operations._reflect_and_decide_after_tools( + user_input=state["user_input"], + turn_index=state["turn_index"], + options=state["options"], + state=state["investigation_state"], + messages=state["messages"], + tool_step=tool_step, + iterations_used=state["iterations_used"], + no_progress_iterations=state["no_progress_iterations"], + ) + return { + "no_progress_iterations": no_progress_iterations, + "tool_calls_used": tool_step.tool_calls_used, + "exchange_index": tool_step.exchange_index, + "result": result, + } + + @staticmethod + def route_after_continue(state: InvestigationGraphState) -> ContinueRoute: + if state["result"] is not None: + return "end" + if state["iterations_used"] >= state["options"].max_iterations: + return "complete_max_iterations" + return "assistant_step" + + def handle_final_draft(self, state: InvestigationGraphState) -> InvestigationGraphUpdate: + final_draft = state["final_draft"] + if final_draft is None: + raise RuntimeError("Investigation graph cannot finalize without a draft") + result = self.operations._evaluate_final_draft( + user_input=state["user_input"], + messages=state["messages"], + turn_index=state["turn_index"], + options=state["options"], + state=state["investigation_state"], + final_draft=final_draft, + iterations_used=state["iterations_used"], + tool_calls_used=state["tool_calls_used"], + ) + return {"result": result, "final_draft": None} + + def complete_max_tools(self, state: InvestigationGraphState) -> InvestigationGraphUpdate: + return { + "result": self.operations._complete_with_budget_answer( + user_input=state["user_input"], + turn_index=state["turn_index"], + options=state["options"], + state=state["investigation_state"], + messages=state["messages"], + iterations_used=state["iterations_used"], + tool_calls_used=state["tool_calls_used"], + stop_reason="max_tool_calls", + ) + } + + def complete_max_iterations(self, state: InvestigationGraphState) -> InvestigationGraphUpdate: + return { + "result": self.operations._complete_with_budget_answer( + user_input=state["user_input"], + turn_index=state["turn_index"], + options=state["options"], + state=state["investigation_state"], + messages=state["messages"], + iterations_used=state["iterations_used"], + tool_calls_used=state["tool_calls_used"], + stop_reason="max_iterations", + ) + } + + +class LangGraphInvestigationKernel: + backend = "langgraph" + + def __init__(self, operations: InvestigationOperations) -> None: + from langgraph.graph import END, START, StateGraph + + self.nodes = InvestigationTurnNodes(operations) + builder = StateGraph(InvestigationGraphState) + builder.add_node("initialize_plan", self.nodes.initialize_plan) + builder.add_node("assistant_step", self.nodes.assistant_step) + builder.add_node("execute_tools", self.nodes.execute_tools) + builder.add_node("reflect_decide", self.nodes.reflect_decide) + builder.add_node("handle_final_draft", self.nodes.handle_final_draft) + builder.add_node("complete_max_tools", self.nodes.complete_max_tools) + builder.add_node("complete_max_iterations", self.nodes.complete_max_iterations) + builder.add_conditional_edges( + START, + self.nodes.route_entry, + {"initialize_plan": "initialize_plan", "reflect_decide": "reflect_decide"}, + ) + builder.add_conditional_edges( + "initialize_plan", + self.nodes.route_after_initialize, + {"assistant_step": "assistant_step", "end": END}, + ) + builder.add_conditional_edges( + "assistant_step", + self.nodes.route_after_assistant, + { + "execute_tools": "execute_tools", + "handle_final_draft": "handle_final_draft", + "complete_max_tools": "complete_max_tools", + "end": END, + }, + ) + builder.add_conditional_edges( + "execute_tools", + self.nodes.route_after_tools, + { + "reflect_decide": "reflect_decide", + "complete_max_tools": "complete_max_tools", + "end": END, + }, + ) + builder.add_conditional_edges( + "reflect_decide", + self.nodes.route_after_continue, + { + "assistant_step": "assistant_step", + "complete_max_iterations": "complete_max_iterations", + "end": END, + }, + ) + builder.add_conditional_edges( + "handle_final_draft", + self.nodes.route_after_continue, + { + "assistant_step": "assistant_step", + "complete_max_iterations": "complete_max_iterations", + "end": END, + }, + ) + builder.add_edge("complete_max_tools", END) + builder.add_edge("complete_max_iterations", END) + self.graph = builder.compile() + + def run( + self, + *, + user_input: str, + session_id: str, + context: ExecutionContext, + messages: list[LLMMessage], + turn_index: int, + options: RunOptions, + ) -> AgentTurnResult: + return self._invoke( + self.nodes.initial_state( + user_input=user_input, + session_id=session_id, + context=context, + messages=messages, + turn_index=turn_index, + options=options, + ) + ) + + def resume_after_pending( + self, + *, + pending: PendingResumeState, + session_id: str, + context: ExecutionContext, + options: RunOptions, + state: InvestigationState, + iterations_used: int, + no_progress_iterations: int, + tool_step: ToolExecutionStepResult | None = None, + ) -> AgentTurnResult: + completed_tool_step = tool_step or ToolExecutionStepResult( + messages=pending.messages, + tool_messages=pending.tool_messages, + exchange_index=pending.exchange_index, + tool_calls_used=pending.tool_calls_used, + tool_statuses=pending.tool_statuses or [pending.tool_status], + tool_names=pending.tool_names or [str(pending.pending_payload.get("tool_name") or "unknown")], + ) + return self._invoke( + self.nodes.initial_state( + user_input=pending.user_input, + session_id=session_id, + context=context, + messages=completed_tool_step.messages, + turn_index=pending.turn_index, + options=options, + investigation_state=state, + iterations_used=iterations_used, + tool_calls_used=completed_tool_step.tool_calls_used, + exchange_index=completed_tool_step.exchange_index, + no_progress_iterations=no_progress_iterations, + resume_tool_step=completed_tool_step, + ) + ) + + def _invoke(self, initial_state: InvestigationGraphState) -> AgentTurnResult: + import langsmith as ls + + options = initial_state["options"] + recursion_limit = max(50, (options.max_iterations * 8) + 16) + with ls.tracing_context(enabled=self.nodes.operations.settings.langchain_tracing_enabled): + final_state = cast( + InvestigationGraphState, + self.graph.invoke(initial_state, {"recursion_limit": recursion_limit}), + ) + result = final_state["result"] + if result is None: + raise RuntimeError("LangGraph investigation kernel completed without a result") + return result diff --git a/agent_core/agent_graph/state.py b/agent_core/agent_graph/state.py new file mode 100644 index 0000000..8d9ca0a --- /dev/null +++ b/agent_core/agent_graph/state.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from typing import Literal, TypedDict + +from agent_core.execution_context import ExecutionContext +from agent_core.investigation_state import InvestigationState +from agent_core.llm.base import LLMMessage +from agent_core.run_options import RunOptions +from agent_core.run_trace import RunTrace +from agent_core.turn_steps import ToolExecutionStepResult +from agent_core.types import AgentTurnResult + + +class AgentGraphState(TypedDict): + """Typed, ephemeral state for the direct conversation graph. + + Durable conversation state remains owned by ``SessionManager`` and + ``RunStore``. This state exists only for one in-process graph invocation. + """ + + user_input: str + session_id: str + context: ExecutionContext + messages: list[LLMMessage] + turn_index: int + tool_calls_used: int + exchange_index: int + trace: RunTrace | None + start_prompt_tokens: int + tool_loop_reserve_tokens: int + prompt_reserve_warning_emitted: bool + model_call_index: int + assistant_message: LLMMessage | None + tool_step: ToolExecutionStepResult | None + result: AgentTurnResult | None + entrypoint: Literal["model", "after_tools"] + + +class AgentGraphUpdate(TypedDict, total=False): + """Partial update emitted by one direct-graph node.""" + + messages: list[LLMMessage] + tool_calls_used: int + exchange_index: int + prompt_reserve_warning_emitted: bool + model_call_index: int + assistant_message: LLMMessage | None + tool_step: ToolExecutionStepResult | None + result: AgentTurnResult | None + + +class InvestigationGraphState(TypedDict): + """Ephemeral state for investigate and deep-investigate graphs.""" + + user_input: str + session_id: str + context: ExecutionContext + messages: list[LLMMessage] + turn_index: int + options: RunOptions + investigation_state: InvestigationState + iterations_used: int + tool_calls_used: int + exchange_index: int + no_progress_iterations: int + assistant_message: LLMMessage | None + tool_step: ToolExecutionStepResult | None + final_draft: str | None + result: AgentTurnResult | None + entrypoint: Literal["initialize", "after_tools"] + + +class InvestigationGraphUpdate(TypedDict, total=False): + messages: list[LLMMessage] + investigation_state: InvestigationState + iterations_used: int + tool_calls_used: int + exchange_index: int + no_progress_iterations: int + assistant_message: LLMMessage | None + tool_step: ToolExecutionStepResult | None + final_draft: str | None + result: AgentTurnResult | None + + +def normalize_agent_kernel_backend(value: str) -> str: + return value.strip().lower().replace("-", "_") + + +def build_graph_checkpoint( + *, + graph: Literal["direct", "investigation"], + backend: str, + resume_node: Literal["resume_tool_exchange"], +) -> dict[str, str]: + """Return the durable graph cursor embedded in agent-core pending state.""" + + return { + "schema_version": "1", + "graph": graph, + "backend": backend, + "resume_node": resume_node, + } diff --git a/agent_core/investigation_controller.py b/agent_core/investigation_controller.py index 63576b1..bf408f4 100644 --- a/agent_core/investigation_controller.py +++ b/agent_core/investigation_controller.py @@ -4,6 +4,7 @@ from dataclasses import asdict from typing import Any, Protocol +from agent_core.agent_graph.state import build_graph_checkpoint, normalize_agent_kernel_backend from agent_core.execution_context import ExecutionContext from agent_core.investigation_models import FinalCritique, InvestigationDecision, StepReflection from agent_core.investigation_prompts import ( @@ -209,6 +210,17 @@ def run( turn_index: int, options: RunOptions, ) -> AgentTurnResult: + from agent_core.agent_graph.investigation import LangGraphInvestigationKernel + if normalize_agent_kernel_backend(self.settings.agent_kernel_backend) == "langgraph": + return LangGraphInvestigationKernel(self).run( + user_input=user_input, + session_id=session_id, + context=context, + messages=messages, + turn_index=turn_index, + options=options, + ) + state = InvestigationState.create_template(objective=user_input) if options.require_initial_plan: self._record_event( @@ -282,6 +294,19 @@ def resume_after_pending( no_progress_iterations: int, tool_step: ToolExecutionStepResult | None = None, ) -> AgentTurnResult: + from agent_core.agent_graph.investigation import LangGraphInvestigationKernel + if normalize_agent_kernel_backend(self.settings.agent_kernel_backend) == "langgraph": + return LangGraphInvestigationKernel(self).resume_after_pending( + pending=pending, + session_id=session_id, + context=context, + options=options, + state=state, + iterations_used=iterations_used, + no_progress_iterations=no_progress_iterations, + tool_step=tool_step, + ) + completed_tool_step = tool_step or ToolExecutionStepResult( messages=pending.messages, tool_messages=pending.tool_messages, @@ -446,6 +471,11 @@ def _run_loop( "investigation_state": state.to_dict(), "iterations_used": iterations_used, "no_progress_iterations": no_progress_iterations, + "agent_graph_checkpoint": build_graph_checkpoint( + graph="investigation", + backend=normalize_agent_kernel_backend(self.settings.agent_kernel_backend), + resume_node="resume_tool_exchange", + ), }, ) messages = tool_step.messages @@ -812,6 +842,44 @@ def _handle_final_draft( exchange_index: int, no_progress_iterations: int, ) -> AgentTurnResult: + result = self._evaluate_final_draft( + user_input=user_input, + messages=messages, + turn_index=turn_index, + options=options, + state=state, + final_draft=final_draft, + iterations_used=iterations_used, + tool_calls_used=tool_calls_used, + ) + if result is not None: + return result + return self._run_loop( + user_input=user_input, + session_id=session_id, + context=context, + messages=messages, + turn_index=turn_index, + options=options, + state=state, + iterations_used=iterations_used, + tool_calls_used=tool_calls_used, + exchange_index=exchange_index, + no_progress_iterations=no_progress_iterations, + ) + + def _evaluate_final_draft( + self, + *, + user_input: str, + messages: list[LLMMessage], + turn_index: int, + options: RunOptions, + state: InvestigationState, + final_draft: str, + iterations_used: int, + tool_calls_used: int, + ) -> AgentTurnResult | None: if not options.require_final_critique: return self._complete_turn( user_input=user_input, @@ -914,19 +982,7 @@ def _handle_final_draft( "Final critique rejected the draft; continuing investigation", extra={"unsupported_claim_count": len(critique.unsupported_claims)}, ) - return self._run_loop( - user_input=user_input, - session_id=session_id, - context=context, - messages=messages, - turn_index=turn_index, - options=options, - state=state, - iterations_used=iterations_used, - tool_calls_used=tool_calls_used, - exchange_index=exchange_index, - no_progress_iterations=no_progress_iterations, - ) + return None def _complete_turn( self, diff --git a/agent_core/llm/azure_anthropic_provider.py b/agent_core/llm/azure_anthropic_provider.py index d10754f..55ad828 100644 --- a/agent_core/llm/azure_anthropic_provider.py +++ b/agent_core/llm/azure_anthropic_provider.py @@ -186,6 +186,7 @@ def complete_text( content=content, usage=token_usage_from_anthropic_response(response), provider="azure_anthropic", + model_backend="native", model=model, provider_request_id=provider_request_id(response), duration_seconds=round(time.monotonic() - started_at, 3), @@ -218,6 +219,7 @@ def complete_with_tools( tool_calls=tool_calls, usage=token_usage_from_anthropic_response(response), provider="azure_anthropic", + model_backend="native", model=model, provider_request_id=provider_request_id(response), duration_seconds=round(time.monotonic() - started_at, 3), diff --git a/agent_core/llm/azure_openai_provider.py b/agent_core/llm/azure_openai_provider.py index a16e1c1..6125198 100644 --- a/agent_core/llm/azure_openai_provider.py +++ b/agent_core/llm/azure_openai_provider.py @@ -104,6 +104,7 @@ def complete_text( content=message.content or "", usage=token_usage_from_openai_response(response), provider="azure_openai", + model_backend="native", model=model, provider_request_id=provider_request_id(response), duration_seconds=round(time.monotonic() - started_at, 3), @@ -161,6 +162,7 @@ def complete_with_tools( tool_calls=tool_calls, usage=token_usage_from_openai_response(response), provider="azure_openai", + model_backend="native", model=model, provider_request_id=provider_request_id(response), duration_seconds=round(time.monotonic() - started_at, 3), diff --git a/agent_core/llm/base.py b/agent_core/llm/base.py index 2166ccc..0cfd6d0 100644 --- a/agent_core/llm/base.py +++ b/agent_core/llm/base.py @@ -103,6 +103,7 @@ class LLMCompletionResult: provider_request_id: str | None = None duration_seconds: float | None = None provider_attempts: int = 1 + model_backend: str | None = None @dataclass(slots=True) @@ -156,6 +157,7 @@ class LLMCallRecord: provider_request_id: str | None = None duration_seconds: float | None = None provider_attempts: int = 1 + model_backend: str | None = None def to_dict(self) -> dict[str, Any]: usage = self.usage @@ -164,6 +166,7 @@ def to_dict(self) -> dict[str, Any]: "call_index": self.call_index, "purpose": self.purpose, "provider": self.provider, + "model_backend": self.model_backend, "model": self.model, "input_tokens": usage.input_tokens if usage is not None else None, "output_tokens": usage.output_tokens if usage is not None else None, @@ -189,6 +192,7 @@ def from_dict(cls, payload: object) -> LLMCallRecord | None: call_index=call_index, purpose=str(payload.get("purpose") or "unspecified"), provider=_optional_text(payload.get("provider")), + model_backend=_optional_text(payload.get("model_backend")), model=_optional_text(payload.get("model")), usage=LLMTokenUsage.from_dict(payload.get("usage") or payload), provider_request_id=_optional_text(payload.get("provider_request_id")), @@ -209,6 +213,7 @@ def from_completion( call_index=call_index, purpose=purpose, provider=completion.provider, + model_backend=completion.model_backend, model=completion.model, usage=completion.usage, provider_request_id=completion.provider_request_id, diff --git a/agent_core/llm/langchain_azure_openai_provider.py b/agent_core/llm/langchain_azure_openai_provider.py new file mode 100644 index 0000000..8486f3e --- /dev/null +++ b/agent_core/llm/langchain_azure_openai_provider.py @@ -0,0 +1,435 @@ +from __future__ import annotations + +import json +import time +from collections.abc import Callable +from typing import Any, cast + +import langsmith as ls +from langchain_core.messages import AIMessage, ChatMessage, HumanMessage, SystemMessage, ToolMessage +from langchain_openai import AzureChatOpenAI +from openai import ( + APIConnectionError, + APIStatusError, + APITimeoutError, + AuthenticationError, + BadRequestError, + OpenAIError, + RateLimitError, +) + +from agent_core.llm.base import ( + LLMCallOptions, + LLMCompletionResult, + LLMMessage, + LLMTokenUsage, + LLMToolCall, + LLMToolDefinition, + publish_llm_completion, +) +from agent_core.llm.errors import LLMProviderError +from agent_core.llm.openai_compat import invoke_openai_request_with_adaptive_retry +from agent_core.llm.openai_request_policy import OpenAIChatRequestNormalizer, OpenAIModelCapabilityResolver +from agent_core.logging_utils import get_logger + +logger = get_logger(__name__) + +ChatModelFactory = Callable[[str], Any] + + +def _call_purpose(options: LLMCallOptions | None, *, default: str) -> str: + value = options.metadata.get("llm_call_purpose") if options is not None else None + return value.strip() if isinstance(value, str) and value.strip() else default + + +class LangChainAzureOpenAIProvider: + """Azure OpenAI adapter implemented with LangChain's model integration. + + The public agent-core contract deliberately remains independent from + LangChain. This adapter is therefore replaceable and can run beside the + existing native SDK provider during the migration. + """ + + def __init__( + self, + *, + azure_endpoint: str | None = None, + api_key: str | None = None, + api_version: str | None = None, + capability_resolver: OpenAIModelCapabilityResolver | None = None, + timeout_seconds: float = 120.0, + chat_model_factory: ChatModelFactory | None = None, + tracing_enabled: bool = False, + ) -> None: + self.azure_endpoint_configured = bool(azure_endpoint) + self.api_key_configured = bool(api_key) + self.api_version = api_version or "v1" + self.azure_endpoint = azure_endpoint + self.api_key = api_key + self.timeout_seconds = max(1.0, float(timeout_seconds)) + self.tracing_enabled = bool(tracing_enabled) + self.capability_resolver = capability_resolver or OpenAIModelCapabilityResolver() + self.request_normalizer = OpenAIChatRequestNormalizer(self.capability_resolver) + self._chat_model_factory = chat_model_factory or self._build_chat_model + self._chat_models: dict[str, Any] = {} + + logger.debug( + "LangChain Azure OpenAI provider initialized", + extra={ + "azure_endpoint_configured": self.azure_endpoint_configured, + "api_key_configured": self.api_key_configured, + "api_version": self.api_version, + "timeout_seconds": self.timeout_seconds, + "tracing_enabled": self.tracing_enabled, + }, + ) + + def complete_text( + self, + *, + messages: list[LLMMessage], + model: str, + temperature: float, + options: LLMCallOptions | None = None, + ) -> LLMCompletionResult: + return self._complete( + messages=messages, + tools=None, + model=model, + temperature=temperature, + options=options, + purpose="text", + ) + + def complete_with_tools( + self, + *, + messages: list[LLMMessage], + tools: list[LLMToolDefinition], + model: str, + temperature: float, + options: LLMCallOptions | None = None, + ) -> LLMCompletionResult: + return self._complete( + messages=messages, + tools=tools, + model=model, + temperature=temperature, + options=options, + purpose="tool_loop", + ) + + def _complete( + self, + *, + messages: list[LLMMessage], + tools: list[LLMToolDefinition] | None, + model: str, + temperature: float, + options: LLMCallOptions | None, + purpose: str, + ) -> LLMCompletionResult: + started_at = time.monotonic() + response, provider_attempts = self._invoke( + messages=messages, + tools=tools, + model=model, + temperature=temperature, + options=options, + ) + tool_calls = self._tool_calls_from_message(response) if tools is not None else [] + result = LLMCompletionResult( + content=self._message_text(response), + tool_calls=tool_calls, + usage=self._token_usage_from_message(response), + provider="azure_openai", + model_backend="langchain", + model=model, + provider_request_id=self._provider_request_id(response), + duration_seconds=round(time.monotonic() - started_at, 3), + provider_attempts=provider_attempts, + ) + logger.debug( + "Received LangChain Azure OpenAI completion response", + extra={"content_length": len(result.content), "tool_call_count": len(result.tool_calls)}, + ) + return publish_llm_completion(result, purpose=_call_purpose(options, default=purpose)) + + def _invoke( + self, + *, + messages: list[LLMMessage], + tools: list[LLMToolDefinition] | None, + model: str, + temperature: float, + options: LLMCallOptions | None, + ) -> tuple[AIMessage, int]: + self._validate_configuration() + request: dict[str, Any] = { + "model": model, + "messages": [self._to_langchain_message(message) for message in messages], + "temperature": temperature, + } + if tools: + request["tools"] = [self._to_openai_tool(tool) for tool in tools] + request["tool_choice"] = "auto" + request["parallel_tool_calls"] = True + if options is not None: + if options.response_format: + request["response_format"] = options.response_format + if options.max_output_tokens is not None: + request["max_tokens"] = options.max_output_tokens + if options.reasoning_effort: + request["reasoning_effort"] = options.reasoning_effort + + normalization = self.request_normalizer.normalize(request) + request = normalization.request + for change in normalization.changes: + logger.debug( + "Adjusted LangChain Azure OpenAI request", + extra={"model": model, "change": change}, + ) + + logger.info( + "Sending LangChain Azure OpenAI chat completion request", + extra={ + "model": model, + "message_count": len(messages), + "tool_count": len(tools or []), + "api_version": self.api_version, + "timeout_seconds": self.timeout_seconds, + }, + ) + provider_attempts = 0 + + def count_attempt() -> None: + nonlocal provider_attempts + provider_attempts += 1 + + try: + response = invoke_openai_request_with_adaptive_retry( + invoke=self._invoke_request, + request=request, + provider_name="Azure OpenAI via LangChain", + logger=logger, + capability_resolver=self.capability_resolver, + response_format_fallback=options.response_format_fallback if options is not None else None, + on_attempt=count_attempt, + ) + except AuthenticationError as exc: + logger.exception("LangChain Azure OpenAI authentication failed") + raise LLMProviderError( + kind="configuration_error", + user_message="Azure OpenAI rejected the credentials. Check AZURE_OPENAI_API_KEY and endpoint access.", + detail=str(exc), + ) from exc + except (APIConnectionError, APITimeoutError) as exc: + logger.exception("LangChain Azure OpenAI request failed due to connectivity or timeout") + raise LLMProviderError( + kind="request_error", + user_message="The assistant could not reach Azure OpenAI. Check network access and try again.", + detail=str(exc), + ) from exc + except RateLimitError as exc: + logger.exception("LangChain Azure OpenAI request was rate limited") + raise LLMProviderError( + kind="rate_limit_error", + user_message="Azure OpenAI rate-limited the request. Wait briefly and try again.", + detail=str(exc), + ) from exc + except (BadRequestError, APIStatusError) as exc: + logger.exception("LangChain Azure OpenAI request was rejected by the API") + raise LLMProviderError( + kind="request_error", + user_message="Azure OpenAI rejected the request. Review the deployment name, API version, and payload.", + detail=str(exc), + ) from exc + except OpenAIError as exc: + logger.exception("Unexpected LangChain Azure OpenAI SDK error") + raise LLMProviderError( + kind="unexpected_error", + user_message="Azure OpenAI failed unexpectedly. Try again after checking the provider configuration.", + detail=str(exc), + ) from exc + except LLMProviderError: + raise + except Exception as exc: + logger.exception("Unexpected LangChain Azure OpenAI provider error") + raise LLMProviderError( + kind="unexpected_error", + user_message="The assistant encountered an unexpected Azure provider failure.", + detail=str(exc), + ) from exc + + if not isinstance(response, AIMessage): + raise LLMProviderError( + kind="response_error", + user_message="Azure OpenAI returned an unusable response.", + detail=f"LangChain returned {type(response).__name__}, expected AIMessage", + ) + return response, provider_attempts + + def _invoke_request(self, request: dict[str, Any]) -> AIMessage: + deployment = str(request.pop("model")) + messages = request.pop("messages") + tools = request.pop("tools", None) + tool_choice = request.pop("tool_choice", None) + parallel_tool_calls = request.pop("parallel_tool_calls", None) + runnable = self._get_chat_model(deployment) + if tools: + binding_options: dict[str, Any] = {"tool_choice": tool_choice} + if parallel_tool_calls is not None: + binding_options["parallel_tool_calls"] = parallel_tool_calls + runnable = runnable.bind_tools(tools, **binding_options) + # Model prompts and tool results can contain sensitive application data. + # Disable LangSmith export unless the host explicitly opts in, even if + # LANGSMITH_TRACING=true is inherited from the process environment. + with ls.tracing_context(enabled=self.tracing_enabled): + return cast(AIMessage, runnable.invoke(messages, **request)) + + def _validate_configuration(self) -> None: + if not self.azure_endpoint_configured: + raise LLMProviderError( + kind="configuration_error", + user_message="The Azure OpenAI endpoint is not configured. Set AZURE_OPENAI_ENDPOINT.", + detail="Missing AZURE_OPENAI_ENDPOINT for LangChainAzureOpenAIProvider", + ) + if not self.api_key_configured: + raise LLMProviderError( + kind="configuration_error", + user_message="The Azure OpenAI API key is not configured. Set AZURE_OPENAI_API_KEY.", + detail="Missing AZURE_OPENAI_API_KEY for LangChainAzureOpenAIProvider", + ) + + def _get_chat_model(self, deployment: str) -> Any: + if deployment not in self._chat_models: + self._chat_models[deployment] = self._chat_model_factory(deployment) + return self._chat_models[deployment] + + def _build_chat_model(self, deployment: str) -> AzureChatOpenAI: + assert self.azure_endpoint is not None + assert self.api_key is not None + return AzureChatOpenAI( + azure_endpoint=self.azure_endpoint, + api_key=cast(Any, self.api_key), + api_version=self.api_version, + azure_deployment=deployment, + timeout=self.timeout_seconds, + # agent-core owns retries and attempt telemetry for parity with the native adapter. + max_retries=0, + cache=False, + ) + + @staticmethod + def _to_langchain_message(message: LLMMessage) -> Any: + if message.role == "system": + return SystemMessage(content=message.content) + if message.role == "user": + return HumanMessage(content=message.content) + if message.role == "tool": + if message.tool_call_id: + return ToolMessage(content=message.content, tool_call_id=message.tool_call_id) + return ChatMessage(role="tool", content=message.content) + additional_kwargs: dict[str, Any] = {} + if message.tool_calls: + additional_kwargs["tool_calls"] = [tool_call.to_history_dict() for tool_call in message.tool_calls] + return AIMessage(content=message.content, additional_kwargs=additional_kwargs) + + @staticmethod + def _to_openai_tool(tool: LLMToolDefinition) -> dict[str, Any]: + return { + "type": "function", + "function": { + "name": tool.name, + "description": tool.description, + "parameters": tool.parameters, + }, + } + + @staticmethod + def _message_text(message: AIMessage) -> str: + if isinstance(message.content, str): + return message.content + return message.text + + @classmethod + def _tool_calls_from_message(cls, message: AIMessage) -> list[LLMToolCall]: + raw_tool_calls = message.additional_kwargs.get("tool_calls") + if isinstance(raw_tool_calls, list): + converted = [ + tool_call + for item in raw_tool_calls + if isinstance(item, dict) + for tool_call in [LLMToolCall.from_history_dict(item)] + if tool_call is not None + ] + if converted: + return converted + + converted = [] + for item in [*message.tool_calls, *message.invalid_tool_calls]: + name = item.get("name") + call_id = item.get("id") + arguments = item.get("args") + if not isinstance(name, str) or not isinstance(call_id, str): + continue + if not isinstance(arguments, str): + arguments = json.dumps(arguments if isinstance(arguments, dict) else {}, separators=(",", ":")) + converted.append(LLMToolCall(id=call_id, name=name, arguments_json=arguments)) + return converted + + @staticmethod + def _token_usage_from_message(message: AIMessage) -> LLMTokenUsage | None: + usage = message.usage_metadata + if not isinstance(usage, dict): + return None + input_tokens = _non_negative_int(usage.get("input_tokens")) + output_tokens = _non_negative_int(usage.get("output_tokens")) + if input_tokens is None or output_tokens is None: + return None + input_details = usage.get("input_token_details") + output_details = usage.get("output_token_details") + return LLMTokenUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=_non_negative_int(usage.get("total_tokens")) or input_tokens + output_tokens, + cached_input_tokens=_detail_token_count(input_details, "cache_read"), + cache_creation_input_tokens=_detail_token_count(input_details, "cache_creation"), + reasoning_output_tokens=_detail_token_count(output_details, "reasoning"), + ) + + @staticmethod + def _provider_request_id(message: AIMessage) -> str | None: + for key in ("id", "request_id", "response_id"): + value = message.response_metadata.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _non_negative_int(value: object) -> int | None: + if isinstance(value, bool): + return None + try: + parsed = int(value) # type: ignore[call-overload] + except (TypeError, ValueError): + return None + return parsed if parsed >= 0 else None + + +def _detail_token_count(details: object, key: str) -> int | None: + if not isinstance(details, dict): + return None + direct = _non_negative_int(details.get(key)) + if direct is not None: + return direct + return next( + ( + parsed + for detail_key, value in details.items() + if detail_key.endswith(f"_{key}") + for parsed in [_non_negative_int(value)] + if parsed is not None + ), + None, + ) diff --git a/agent_core/llm/openai_compat.py b/agent_core/llm/openai_compat.py index 876faa0..ff89c2d 100644 --- a/agent_core/llm/openai_compat.py +++ b/agent_core/llm/openai_compat.py @@ -3,6 +3,7 @@ import os import random import time +from collections.abc import Callable from dataclasses import dataclass from email.utils import parsedate_to_datetime from typing import Any @@ -67,6 +68,39 @@ def create_chat_completion_with_adaptive_retry( random_fn: Any = random.random, on_attempt: Any = None, ) -> Any: + return invoke_openai_request_with_adaptive_retry( + invoke=lambda current_request: completions.create(**current_request), + request=request, + provider_name=provider_name, + logger=logger, + capability_resolver=capability_resolver, + rate_limit_policy=rate_limit_policy, + response_format_fallback=response_format_fallback, + sleeper=sleeper, + random_fn=random_fn, + on_attempt=on_attempt, + ) + + +def invoke_openai_request_with_adaptive_retry( + *, + invoke: Callable[[dict[str, Any]], Any], + request: dict[str, Any], + provider_name: str, + logger: Any, + capability_resolver: OpenAIModelCapabilityResolver | None = None, + rate_limit_policy: OpenAIRateLimitRetryPolicy | None = None, + response_format_fallback: dict[str, Any] | None = None, + sleeper: Any = time.sleep, + random_fn: Any = random.random, + on_attempt: Any = None, +) -> Any: + """Invoke an OpenAI-compatible request while preserving agent-core retry semantics. + + The callback boundary lets SDK adapters such as LangChain share the same + capability learning, structured-output safeguards, and transient retry policy + as the native OpenAI clients. + """ fallback_request = dict(request) retried_without: set[str] = set() policy = rate_limit_policy or OpenAIRateLimitRetryPolicy.from_env() @@ -77,7 +111,7 @@ def create_chat_completion_with_adaptive_retry( on_attempt() attempt_started_at = time.monotonic() try: - return completions.create(**fallback_request) + return invoke(dict(fallback_request)) except RateLimitError as exc: attempt = _retry_after_transient_error( exc=exc, diff --git a/agent_core/llm/openai_provider.py b/agent_core/llm/openai_provider.py index 097891e..d631fcd 100644 --- a/agent_core/llm/openai_provider.py +++ b/agent_core/llm/openai_provider.py @@ -128,6 +128,7 @@ def complete_text( content=message.content or "", usage=token_usage_from_openai_response(response), provider="openai", + model_backend="native", model=model, provider_request_id=provider_request_id(response), duration_seconds=round(time.monotonic() - started_at, 3), @@ -178,6 +179,7 @@ def complete_with_tools( tool_calls=tool_calls, usage=token_usage_from_openai_response(response), provider="openai", + model_backend="native", model=model, provider_request_id=provider_request_id(response), duration_seconds=round(time.monotonic() - started_at, 3), diff --git a/agent_core/llm/provider_factory.py b/agent_core/llm/provider_factory.py index 02339d4..d1ae28f 100644 --- a/agent_core/llm/provider_factory.py +++ b/agent_core/llm/provider_factory.py @@ -5,6 +5,7 @@ from agent_core.llm.azure_anthropic_provider import AzureAnthropicProvider from agent_core.llm.azure_openai_provider import AzureOpenAIProvider from agent_core.llm.base import BaseLLMProvider +from agent_core.llm.langchain_azure_openai_provider import LangChainAzureOpenAIProvider from agent_core.llm.openai_provider import OpenAIProvider from agent_core.settings import CoreSettings @@ -21,10 +22,13 @@ class LLMProviderConfig: azure_anthropic_api_version: str | None = None azure_anthropic_version: str | None = None timeout_seconds: float = 120.0 + model_backend: str = "native" + langchain_tracing_enabled: bool = False _MEMORY_PROVIDER_FIELDS = ( "memory_llm_provider", + "memory_llm_model_backend", "memory_openai_api_key", "memory_azure_openai_endpoint", "memory_azure_openai_api_key", @@ -40,6 +44,10 @@ def normalize_provider_name(provider_name: str | None) -> str: return (provider_name or "openai").strip().lower().replace("-", "_") +def normalize_model_backend(model_backend: str | None) -> str: + return (model_backend or "native").strip().lower().replace("-", "_") + + def build_provider(settings: CoreSettings) -> BaseLLMProvider: return build_provider_from_config(_primary_provider_config(settings)) @@ -53,6 +61,27 @@ def build_memory_provider(settings: CoreSettings) -> BaseLLMProvider | None: def build_provider_from_config(config: LLMProviderConfig) -> BaseLLMProvider: provider_name = normalize_provider_name(config.provider) + model_backend = normalize_model_backend(config.model_backend) + + if model_backend not in {"native", "langchain"}: + raise ValueError( + f"Unsupported LLM model backend: {config.model_backend}. " + "Supported values are native and langchain." + ) + + if model_backend == "langchain": + if provider_name != "azure_openai": + raise ValueError( + "The LangChain model backend currently supports only provider=azure_openai. " + f"Received provider={config.provider}." + ) + return LangChainAzureOpenAIProvider( + azure_endpoint=config.azure_openai_endpoint, + api_key=config.azure_openai_api_key, + api_version=config.azure_openai_api_version, + timeout_seconds=config.timeout_seconds, + tracing_enabled=config.langchain_tracing_enabled, + ) if provider_name == "openai": return OpenAIProvider( @@ -86,6 +115,7 @@ def build_provider_from_config(config: LLMProviderConfig) -> BaseLLMProvider: def _primary_provider_config(settings: CoreSettings) -> LLMProviderConfig: return LLMProviderConfig( provider=settings.llm_provider, + model_backend=settings.llm_model_backend, openai_api_key=settings.openai_api_key, azure_openai_endpoint=settings.azure_openai_endpoint, azure_openai_api_key=settings.azure_openai_api_key, @@ -95,6 +125,7 @@ def _primary_provider_config(settings: CoreSettings) -> LLMProviderConfig: azure_anthropic_api_version=settings.azure_anthropic_api_version, azure_anthropic_version=settings.azure_anthropic_version, timeout_seconds=settings.llm_timeout_seconds, + langchain_tracing_enabled=settings.langchain_tracing_enabled, ) @@ -103,8 +134,10 @@ def _memory_provider_config(settings: CoreSettings) -> LLMProviderConfig | None: return None provider = _prefer_override(settings.memory_llm_provider, settings.llm_provider) or "openai" + model_backend = _prefer_override(settings.memory_llm_model_backend, settings.llm_model_backend) or "native" return LLMProviderConfig( provider=provider, + model_backend=model_backend, openai_api_key=_prefer_override(settings.memory_openai_api_key, settings.openai_api_key), azure_openai_endpoint=_prefer_override(settings.memory_azure_openai_endpoint, settings.azure_openai_endpoint), azure_openai_api_key=_prefer_override(settings.memory_azure_openai_api_key, settings.azure_openai_api_key), @@ -114,6 +147,7 @@ def _memory_provider_config(settings: CoreSettings) -> LLMProviderConfig | None: azure_anthropic_api_version=_prefer_override(settings.memory_azure_anthropic_api_version, settings.azure_anthropic_api_version), azure_anthropic_version=_prefer_override(settings.memory_azure_anthropic_version, settings.azure_anthropic_version), timeout_seconds=settings.llm_timeout_seconds, + langchain_tracing_enabled=settings.langchain_tracing_enabled, ) diff --git a/agent_core/orchestrator.py b/agent_core/orchestrator.py index 05476eb..84354c7 100644 --- a/agent_core/orchestrator.py +++ b/agent_core/orchestrator.py @@ -7,6 +7,7 @@ from typing import Any from uuid import uuid4 +from agent_core.agent_graph.direct import build_direct_turn_kernel from agent_core.context_planner import ( LLMContextPlanner, LLMContextPolicy, @@ -129,6 +130,10 @@ def __init__( max_handoff_chars=settings.memory_max_handoff_chars, max_turn_summary_chars=settings.memory_max_turn_summary_chars, ) + self._direct_turn_nodes, self._direct_turn_kernel = build_direct_turn_kernel( + backend=settings.agent_kernel_backend, + operations=self, + ) def _build_tool_history_item( self, @@ -192,6 +197,7 @@ def _start_run_trace( turn_index=turn_index, options={ "run_options": asdict(run_options), + "agent_kernel_backend": self._direct_turn_kernel.backend, "model": self.settings.model, "max_active_context_tokens": self.settings.max_active_context_tokens, "max_tool_calls_per_turn": self.settings.max_tool_calls_per_turn, @@ -729,6 +735,7 @@ def _run_turn_result_active( "thread_id": context.thread_id, "user_input_length": len(user_input), "mode": run_options.mode, + "agent_kernel_backend": self._direct_turn_kernel.backend, }, ) state = self.session_manager.get_state() @@ -829,6 +836,9 @@ def resume_turn( status="completed", content="Pending agent turn is incompatible with the current artifact contract.", ) + graph_checkpoint_error = self._validate_pending_graph_checkpoint(pending) + if graph_checkpoint_error is not None: + return AgentTurnResult(status="completed", content=graph_checkpoint_error) pending_run_id = pending.get("run_trace_id") bound_context = context.with_run_id( @@ -934,6 +944,18 @@ def _resume_turn_active( return AgentTurnResult(status="completed", content="Pending agent turn disappeared before resume.") trace = self._load_run_trace_from_pending(pending) + graph_checkpoint = pending.get("agent_graph_checkpoint") + if isinstance(graph_checkpoint, dict): + self._record_trace_event( + trace, + event_type="agent_graph_checkpoint_restored", + summary="Agent graph checkpoint restored", + payload={ + "graph": graph_checkpoint.get("graph"), + "backend": graph_checkpoint.get("backend"), + "resume_node": graph_checkpoint.get("resume_node"), + }, + ) self._record_trace_event( trace, event_type="pending_tool_result_received", @@ -995,32 +1017,42 @@ def _resume_turn_active( else result ) - if tool_step.budget_exhausted: - msg = "Maximum number of tool calls reached for this turn." - self._persist_conversation_turn( - turn_index=resumed.turn_index, - user_input=resumed.user_input, - assistant_content=msg, - ) - self._refresh_memory_after_turn(turn_index=resumed.turn_index) - result = AgentTurnResult(status="completed", content=msg) - else: - result = self._continue_turn( - user_input=resumed.user_input, - session_id=context.namespace_id, - context=execution_context, - messages=tool_step.messages, - turn_index=resumed.turn_index, - tool_calls_used=tool_step.tool_calls_used, - exchange_index=tool_step.exchange_index, - trace=trace, - ) + result = self._continue_turn( + user_input=resumed.user_input, + session_id=context.namespace_id, + context=execution_context, + messages=tool_step.messages, + turn_index=resumed.turn_index, + tool_calls_used=tool_step.tool_calls_used, + exchange_index=tool_step.exchange_index, + trace=trace, + resume_tool_step=tool_step, + ) return ( self._finalize_run_trace_result(trace=trace, result=result, expose_trace_id=bool(result.metadata)) if trace is not None else result ) + @staticmethod + def _validate_pending_graph_checkpoint(pending: dict[str, Any]) -> str | None: + raw_checkpoint = pending.get("agent_graph_checkpoint") + if raw_checkpoint is None: + return None + if not isinstance(raw_checkpoint, dict): + return "Pending agent graph checkpoint is corrupt." + expected_graph = ( + "investigation" if pending.get("mode") in {"investigate", "deep_investigate"} else "direct" + ) + if ( + raw_checkpoint.get("schema_version") != "1" + or raw_checkpoint.get("graph") != expected_graph + or raw_checkpoint.get("backend") not in {"native", "langgraph"} + or raw_checkpoint.get("resume_node") != "resume_tool_exchange" + ): + return "Pending agent graph checkpoint is incompatible with the current graph contract." + return None + def _build_investigation_prompt_set(self, *, options: RunOptions) -> InvestigationPromptSet: return self.domain_hooks.customize_investigation_prompts( prompt_set=DEFAULT_INVESTIGATION_PROMPTS, @@ -1603,145 +1635,17 @@ def _continue_turn( tool_calls_used: int, exchange_index: int, trace: RunTrace | None = None, + resume_tool_step: ToolExecutionStepResult | None = None, ) -> AgentTurnResult: - start_prompt_tokens = self._estimate_prompt_tokens(messages=messages) - tool_loop_reserve_tokens = max(1, self.settings.max_active_context_tokens) - prompt_reserve_warning_emitted = False - model_call_index = 0 - - while True: - model_call_index += 1 - prompt_tokens = self._estimate_prompt_tokens(messages=messages) - prompt_growth_tokens = max(0, prompt_tokens - start_prompt_tokens) - logger.debug( - "Calling LLM", - extra={ - "model": self.settings.model, - "message_count": len(messages), - "estimated_prompt_tokens": prompt_tokens, - "start_turn_prompt_tokens": start_prompt_tokens, - "tool_loop_reserve_tokens": tool_loop_reserve_tokens, - "prompt_growth_tokens": prompt_growth_tokens, - }, - ) - if ( - tool_calls_used > 0 - and prompt_growth_tokens >= tool_loop_reserve_tokens - and not prompt_reserve_warning_emitted - ): - logger.warning( - "Tool loop consumed the start-turn prompt reserve", - extra={ - "estimated_prompt_tokens": prompt_tokens, - "start_turn_prompt_tokens": start_prompt_tokens, - "prompt_growth_tokens": prompt_growth_tokens, - "tool_loop_reserve_tokens": tool_loop_reserve_tokens, - "tool_calls_used": tool_calls_used, - }, - ) - prompt_reserve_warning_emitted = True - self._record_trace_event( - trace, - event_type="llm_call_started", - summary="LLM call started", - iteration=model_call_index, - payload={ - "message_count": len(messages), - "estimated_prompt_tokens": prompt_tokens, - "tool_calls_used": tool_calls_used, - "exchange_index": exchange_index, - }, - ) - try: - llm_response = self._call_model_once(messages=messages) - except LLMProviderError as exc: - self._record_trace_event( - trace, - event_type="llm_provider_failure", - summary="LLM provider failure handled", - iteration=model_call_index, - payload={ - "kind": exc.kind, - "detail_preview": safe_preview(exc.detail or exc.user_message, limit=200), - }, - ) - return self._handle_provider_failure( - error=exc, - user_input=user_input, - turn_index=turn_index, - ) - - logger.debug( - "Received LLM response", - extra={ - "content_length": len(llm_response.content), - "tool_call_count": len(llm_response.tool_calls), - }, - ) - - assistant_message = LLMMessage( - role="assistant", - content=llm_response.content, - tool_calls=list(llm_response.tool_calls), - ) - messages.append(assistant_message) - self._record_trace_event( - trace, - event_type="assistant_response_received", - summary="Assistant response received", - iteration=model_call_index, - payload={ - "content_length": len(llm_response.content), - "tool_call_count": len(llm_response.tool_calls), - "tool_calls": [ - {"id": tool_call.id, "name": tool_call.name} for tool_call in llm_response.tool_calls - ], - }, - ) - - if not llm_response.tool_calls: - self._persist_conversation_turn( - turn_index=turn_index, - user_input=user_input, - assistant_content=llm_response.content, - ) - self._refresh_memory_after_turn(turn_index=turn_index) - logger.info("Completing run_turn without additional tool calls") - return AgentTurnResult(status="completed", content=llm_response.content) - - tool_step = self._execute_tool_calls_once( - user_input=user_input, - session_id=session_id, - context=context, - messages=messages, - turn_index=turn_index, - exchange_index=exchange_index, - tool_calls_used=tool_calls_used, - assistant_message=assistant_message, - max_tool_calls=self.settings.max_tool_calls_per_turn, - trace=trace, - ) - messages = tool_step.messages - exchange_index = tool_step.exchange_index - tool_calls_used = tool_step.tool_calls_used - - if tool_step.pending_result is not None: - return tool_step.pending_result - - if tool_step.budget_exhausted: - msg = "Maximum number of tool calls reached for this turn." - logger.error(msg) - self._record_trace_event( - trace, - event_type="tool_budget_exhausted", - summary=msg, - iteration=model_call_index, - payload={"tool_calls_used": tool_calls_used}, - ) - self._persist_conversation_turn( - turn_index=turn_index, - user_input=user_input, - assistant_content=msg, - ) - self._refresh_memory_after_turn(turn_index=turn_index) - return AgentTurnResult(status="completed", content=msg) + state = self._direct_turn_nodes.initial_state( + user_input=user_input, + session_id=session_id, + context=context, + messages=messages, + turn_index=turn_index, + tool_calls_used=tool_calls_used, + exchange_index=exchange_index, + trace=trace, + resume_tool_step=resume_tool_step, + ) + return self._direct_turn_kernel.run(state) diff --git a/agent_core/settings.py b/agent_core/settings.py index fa473aa..e526da4 100644 --- a/agent_core/settings.py +++ b/agent_core/settings.py @@ -70,6 +70,12 @@ class CoreSettings: base_system_prompt: str = "" turn_memory_synthesis_prompt: str = "" + # Appended to preserve the positional layout of the pre-existing dataclass fields. + llm_model_backend: str = "native" + memory_llm_model_backend: str | None = None + langchain_tracing_enabled: bool = False + agent_kernel_backend: str = "native" + def __post_init__(self) -> None: self.llm_budget = LLMBudget.from_any(self.llm_budget) self.llm_context_policy = LLMContextPolicy.from_any(self.llm_context_policy) diff --git a/agent_core/spi.py b/agent_core/spi.py index f3506e5..7f09756 100644 --- a/agent_core/spi.py +++ b/agent_core/spi.py @@ -19,6 +19,7 @@ build_memory_provider, build_provider, build_provider_from_config, + normalize_model_backend, normalize_provider_name, ) from agent_core.policy_engine import PolicyEngine @@ -54,5 +55,6 @@ "build_provider_from_config", "build_tool_definition", "load_prompt", + "normalize_model_backend", "normalize_provider_name", ] diff --git a/agent_core/structured_tasks.py b/agent_core/structured_tasks.py index 4c0b0fd..0249b77 100644 --- a/agent_core/structured_tasks.py +++ b/agent_core/structured_tasks.py @@ -2,11 +2,12 @@ import hashlib import json -from collections.abc import Callable +from collections.abc import Callable, Hashable from dataclasses import dataclass, field from inspect import Parameter, signature -from typing import Any, Literal +from typing import Any, Literal, Protocol, TypedDict, cast +from agent_core.agent_graph.state import normalize_agent_kernel_backend from agent_core.context_planner import ( LLMContextPlanner, LLMContextPolicy, @@ -496,6 +497,10 @@ def __init__( self.tool_registry = tool_registry self.policy_engine = policy_engine self.artifact_store = artifact_store or JsonFileArtifactStore(settings.artifacts_directory) + self._kernel_nodes, self._kernel = build_structured_task_kernel( + backend=settings.agent_kernel_backend, + operations=self, + ) def run( self, @@ -627,114 +632,122 @@ def _continue_from_checkpoint( task_id=spec.task_id, failure_reason=str(exc), ) + initial_state = self._kernel_nodes.initial_state( + spec=spec, + context=context, + registry=registry, + checkpoint=checkpoint, + on_checkpoint=on_checkpoint, + ) + return self._kernel.run(initial_state) - while True: - if checkpoint.phase == "result": - return self._continue_persisted_result(spec=spec, checkpoint=checkpoint) - - if checkpoint.phase == "finalization": - finalized = self._continue_finalization( - spec=spec, - checkpoint=checkpoint, - on_checkpoint=on_checkpoint, - ) - if finalized is not None: - return finalized - continue - - if checkpoint.phase == "tools": - blocked = next( - (item for item in checkpoint.pending_tool_calls if item.status == "running"), - None, - ) - if blocked is not None: - raise StructuredTaskRecoveryError( - kind="ambiguous_tool_execution", - message=( - "A tool call was running when execution stopped; automatic replay is blocked " - f"because its external effect is unknown: {blocked.tool_name} ({blocked.tool_call_id})." - ), - tool_call_id=blocked.tool_call_id, - ) - finalized = self._continue_tool_batch( - spec=spec, - context=context, - registry=registry, - checkpoint=checkpoint, - on_checkpoint=on_checkpoint, - ) - if finalized is not None: - return finalized - continue - - logger.debug( - "Calling structured task LLM", - extra={ - "task_id": spec.task_id, - "iteration": checkpoint.iterations, - "tool_count": len(registry.list_tool_names()), - }, + def _continue_model_request( + self, + *, + spec: StructuredTaskSpec, + registry: ToolRegistry, + checkpoint: StructuredTaskCheckpoint, + on_checkpoint: Callable[[StructuredTaskCheckpoint], None] | None, + ) -> StructuredTaskResult | None: + logger.debug( + "Calling structured task LLM", + extra={ + "task_id": spec.task_id, + "iteration": checkpoint.iterations, + "tool_count": len(registry.list_tool_names()), + }, + ) + try: + llm_response = self._call_model_once( + spec=spec, + messages=checkpoint.messages, + registry=registry, + final_output=not registry.list_tool_names(), + on_budget_reserved=lambda: self._emit_checkpoint(checkpoint, on_checkpoint), ) - try: - llm_response = self._call_model_once( - spec=spec, - messages=checkpoint.messages, - registry=registry, - final_output=not registry.list_tool_names(), - on_budget_reserved=lambda: self._emit_checkpoint(checkpoint, on_checkpoint), - ) - except LLMProviderError as exc: - logger.error( - "Structured task provider failure", - extra={"task_id": spec.task_id, "error_kind": exc.kind}, - ) - return StructuredTaskResult( - ok=False, - task_id=spec.task_id, - failure_reason=exc.user_message, - raw_content=exc.detail or exc.user_message, - tool_history=checkpoint.tool_history, - iterations=checkpoint.iterations, - tool_calls_used=checkpoint.tool_calls_used, - llm_calls=list(checkpoint.llm_calls), - ) - - self._record_llm_call( - checkpoint=checkpoint, - completion=llm_response, - purpose="structured_direct" if not registry.list_tool_names() else "structured_tool_loop", + except LLMProviderError as exc: + logger.error( + "Structured task provider failure", + extra={"task_id": spec.task_id, "error_kind": exc.kind}, ) - assistant_message = LLMMessage( + return StructuredTaskResult( + ok=False, + task_id=spec.task_id, + failure_reason=exc.user_message, + raw_content=exc.detail or exc.user_message, + tool_history=checkpoint.tool_history, + iterations=checkpoint.iterations, + tool_calls_used=checkpoint.tool_calls_used, + llm_calls=list(checkpoint.llm_calls), + ) + + self._record_llm_call( + checkpoint=checkpoint, + completion=llm_response, + purpose="structured_direct" if not registry.list_tool_names() else "structured_tool_loop", + ) + checkpoint.messages.append( + LLMMessage( role="assistant", content=llm_response.content, tool_calls=list(llm_response.tool_calls), ) - checkpoint.messages.append(assistant_message) - - if not llm_response.tool_calls: - if spec.output_contract is not None and registry.list_tool_names(): - checkpoint.phase = "finalization" - checkpoint.finalization_kind = "contract" - checkpoint.finalization_reason = "Investigation is complete." - checkpoint.raw_failure_content = llm_response.content - self._emit_checkpoint(checkpoint, on_checkpoint) - continue + ) + + if not llm_response.tool_calls: + if spec.output_contract is not None and registry.list_tool_names(): + checkpoint.phase = "finalization" + checkpoint.finalization_kind = "contract" + checkpoint.finalization_reason = "Investigation is complete." + checkpoint.raw_failure_content = llm_response.content + else: checkpoint.phase = "result" checkpoint.result_kind = "direct" - self._emit_checkpoint(checkpoint, on_checkpoint) - continue - - checkpoint.phase = "tools" - checkpoint.pending_tool_calls = [ - StructuredToolCallCheckpoint( - tool_call_id=tool_call.id, - tool_name=tool_call.name, - arguments_json=tool_call.arguments_json, - ) - for tool_call in llm_response.tool_calls - ] - checkpoint.next_tool_call_index = 0 self._emit_checkpoint(checkpoint, on_checkpoint) + return None + + checkpoint.phase = "tools" + checkpoint.pending_tool_calls = [ + StructuredToolCallCheckpoint( + tool_call_id=tool_call.id, + tool_name=tool_call.name, + arguments_json=tool_call.arguments_json, + ) + for tool_call in llm_response.tool_calls + ] + checkpoint.next_tool_call_index = 0 + self._emit_checkpoint(checkpoint, on_checkpoint) + return None + + def _continue_tools( + self, + *, + spec: StructuredTaskSpec, + context: ExecutionContext, + registry: ToolRegistry, + checkpoint: StructuredTaskCheckpoint, + on_checkpoint: Callable[[StructuredTaskCheckpoint], None] | None, + ) -> StructuredTaskResult | None: + blocked = next( + (item for item in checkpoint.pending_tool_calls if item.status == "running"), + None, + ) + if blocked is not None: + raise StructuredTaskRecoveryError( + kind="ambiguous_tool_execution", + message=( + "A tool call was running when execution stopped; automatic replay is blocked " + f"because its external effect is unknown: {blocked.tool_name} ({blocked.tool_call_id})." + ), + tool_call_id=blocked.tool_call_id, + ) + return self._continue_tool_batch( + spec=spec, + context=context, + registry=registry, + checkpoint=checkpoint, + on_checkpoint=on_checkpoint, + ) def _continue_tool_batch( self, @@ -1494,3 +1507,215 @@ def _provider_accepts_options(self, method_name: str) -> bool: except (TypeError, ValueError): return True return any(parameter.kind == Parameter.VAR_KEYWORD or parameter.name == "options" for parameter in parameters) + + +StructuredTaskKernelBackend = Literal["native", "langgraph"] +StructuredTaskKernelRoute = Literal["model_request", "tools", "finalization", "result", "end"] + + +class StructuredTaskGraphState(TypedDict): + """Ephemeral orchestration state; durable state remains in the checkpoint.""" + + spec: StructuredTaskSpec + context: ExecutionContext + registry: ToolRegistry + checkpoint: StructuredTaskCheckpoint + on_checkpoint: Callable[[StructuredTaskCheckpoint], None] | None + result: StructuredTaskResult | None + + +class StructuredTaskGraphUpdate(TypedDict, total=False): + checkpoint: StructuredTaskCheckpoint + result: StructuredTaskResult | None + + +class StructuredTaskOperations(Protocol): + settings: CoreSettings + + def _continue_model_request( + self, + *, + spec: StructuredTaskSpec, + registry: ToolRegistry, + checkpoint: StructuredTaskCheckpoint, + on_checkpoint: Callable[[StructuredTaskCheckpoint], None] | None, + ) -> StructuredTaskResult | None: ... + + def _continue_tools( + self, + *, + spec: StructuredTaskSpec, + context: ExecutionContext, + registry: ToolRegistry, + checkpoint: StructuredTaskCheckpoint, + on_checkpoint: Callable[[StructuredTaskCheckpoint], None] | None, + ) -> StructuredTaskResult | None: ... + + def _continue_finalization( + self, + *, + spec: StructuredTaskSpec, + checkpoint: StructuredTaskCheckpoint, + on_checkpoint: Callable[[StructuredTaskCheckpoint], None] | None, + ) -> StructuredTaskResult | None: ... + + def _continue_persisted_result( + self, + *, + spec: StructuredTaskSpec, + checkpoint: StructuredTaskCheckpoint, + ) -> StructuredTaskResult: ... + + +class StructuredTaskNodes: + """Structured-task behavior shared by the native and LangGraph kernels.""" + + def __init__(self, operations: StructuredTaskOperations) -> None: + self.operations = operations + + @staticmethod + def initial_state( + *, + spec: StructuredTaskSpec, + context: ExecutionContext, + registry: ToolRegistry, + checkpoint: StructuredTaskCheckpoint, + on_checkpoint: Callable[[StructuredTaskCheckpoint], None] | None, + ) -> StructuredTaskGraphState: + return StructuredTaskGraphState( + spec=spec, + context=context, + registry=registry, + checkpoint=checkpoint, + on_checkpoint=on_checkpoint, + result=None, + ) + + @staticmethod + def route(state: StructuredTaskGraphState) -> StructuredTaskKernelRoute: + if state["result"] is not None: + return "end" + return state["checkpoint"].phase + + def model_request(self, state: StructuredTaskGraphState) -> StructuredTaskGraphUpdate: + checkpoint = state["checkpoint"] + result = self.operations._continue_model_request( + spec=state["spec"], + registry=state["registry"], + checkpoint=checkpoint, + on_checkpoint=state["on_checkpoint"], + ) + return {"checkpoint": checkpoint, "result": result} + + def tools(self, state: StructuredTaskGraphState) -> StructuredTaskGraphUpdate: + checkpoint = state["checkpoint"] + result = self.operations._continue_tools( + spec=state["spec"], + context=state["context"], + registry=state["registry"], + checkpoint=checkpoint, + on_checkpoint=state["on_checkpoint"], + ) + return {"checkpoint": checkpoint, "result": result} + + def finalization(self, state: StructuredTaskGraphState) -> StructuredTaskGraphUpdate: + checkpoint = state["checkpoint"] + result = self.operations._continue_finalization( + spec=state["spec"], + checkpoint=checkpoint, + on_checkpoint=state["on_checkpoint"], + ) + return {"checkpoint": checkpoint, "result": result} + + def result(self, state: StructuredTaskGraphState) -> StructuredTaskGraphUpdate: + return { + "result": self.operations._continue_persisted_result( + spec=state["spec"], + checkpoint=state["checkpoint"], + ) + } + + +class StructuredTaskKernel(Protocol): + backend: StructuredTaskKernelBackend + + def run(self, initial_state: StructuredTaskGraphState) -> StructuredTaskResult: ... + + +class NativeStructuredTaskKernel: + backend: StructuredTaskKernelBackend = "native" + + def __init__(self, nodes: StructuredTaskNodes) -> None: + self.nodes = nodes + + def run(self, initial_state: StructuredTaskGraphState) -> StructuredTaskResult: + state = initial_state + while True: + route = self.nodes.route(state) + if route == "end": + break + node = getattr(self.nodes, route) + state.update(node(state)) + + result = state["result"] + if result is None: + raise RuntimeError("Native structured task kernel completed without a result") + return result + + +class LangGraphStructuredTaskKernel: + backend: StructuredTaskKernelBackend = "langgraph" + + def __init__(self, nodes: StructuredTaskNodes) -> None: + from langgraph.graph import END, START, StateGraph + + self.nodes = nodes + builder = StateGraph(StructuredTaskGraphState) + builder.add_node("model_request", nodes.model_request) + builder.add_node("tools", nodes.tools) + builder.add_node("finalization", nodes.finalization) + builder.add_node("result", nodes.result) + routes: dict[Hashable, str] = { + "model_request": "model_request", + "tools": "tools", + "finalization": "finalization", + "result": "result", + "end": END, + } + builder.add_conditional_edges(START, nodes.route, routes) + builder.add_conditional_edges("model_request", nodes.route, routes) + builder.add_conditional_edges("tools", nodes.route, routes) + builder.add_conditional_edges("finalization", nodes.route, routes) + builder.add_edge("result", END) + self.graph = builder.compile() + + def run(self, initial_state: StructuredTaskGraphState) -> StructuredTaskResult: + import langsmith as ls + + spec = initial_state["spec"] + recursion_limit = max(25, (spec.max_iterations * 2) + 8) + with ls.tracing_context(enabled=self.nodes.operations.settings.langchain_tracing_enabled): + final_state = cast( + StructuredTaskGraphState, + self.graph.invoke(initial_state, {"recursion_limit": recursion_limit}), + ) + result = final_state["result"] + if result is None: + raise RuntimeError("LangGraph structured task kernel completed without a result") + return result + + +def build_structured_task_kernel( + *, + backend: str, + operations: StructuredTaskOperations, +) -> tuple[StructuredTaskNodes, StructuredTaskKernel]: + normalized = normalize_agent_kernel_backend(backend) + nodes = StructuredTaskNodes(operations) + if normalized == "native": + return nodes, NativeStructuredTaskKernel(nodes) + if normalized == "langgraph": + return nodes, LangGraphStructuredTaskKernel(nodes) + raise ValueError( + f"Unsupported agent kernel backend: {backend!r}. Expected 'native' or 'langgraph'." + ) diff --git a/docs/langgraph_migration.md b/docs/langgraph_migration.md new file mode 100644 index 0000000..fc75db3 --- /dev/null +++ b/docs/langgraph_migration.md @@ -0,0 +1,197 @@ +# LangGraph kernel migration + +This document records the first seven steps of the incremental migration of the +conversation agent kernel. The public conversation API and every durable +storage contract remain unchanged. + +## Implemented scope + +1. The existing direct and investigation control flows are mapped below. +2. Direct execution has a typed internal `AgentGraphState`. +3. `CoreSettings.agent_kernel_backend` selects `native` or `langgraph`; native + remains the default. +4. The direct model/tool loop is a real multi-node LangGraph graph. The native + fallback executes the same node implementations with an ordinary loop. +5. Pending tools persist a versioned graph cursor and resume inside the selected + kernel after agent-core restores the tool exchange. +6. `investigate` and `deep_investigate` use a second multi-node graph when the + LangGraph backend is enabled. +7. Native and LangGraph kernels run against the same direct, investigation, + deep-investigation, and pending/resume contract tests. + +`AgentRunService` and `StructuredTaskRunner` remain outside this +conversation-kernel migration. + +## Existing orchestration map + +The public entry point binds a `RunContext`, creates budget, context-planning, +and artifact scopes, acquires the session scope, builds the prompt, and starts +the run trace. It then routes by `RunOptions.mode`: + +- `direct` enters the model/tool loop described below; +- `investigate` and `deep_investigate` enter `InvestigationController`, which + selects its native loop or LangGraph kernel; +- all paths finalize the existing `RunTrace` and return `AgentTurnResult`. + +Pending tools stop the current invocation after agent-core has persisted the +exact provider transcript, counters, artifact usage, budget usage, trace id, +and remaining tool cursor. `resume_turn()` restores those values, finishes any +remaining calls in the same tool exchange, validates the durable graph cursor, +clears the pending marker, then resumes the selected graph after its tool node. + +## Direct graph + +```mermaid +flowchart TD + S([START]) -->|new turn| M[call_model] + S -->|resumed exchange| R{route after tools} + R -->|exchange completed| M + R -->|tool budget exhausted| B[complete_budget] + R -->|pending result| E([END]) + M -->|provider failure| E([END]) + M -->|no tool calls| C[complete_response] + M -->|tool calls| T[execute_tools] + T -->|pending result| E + T -->|tool budget exhausted| B[complete_budget] + T -->|exchange completed| M + C --> E + B --> E +``` + +The graph nodes are intentionally coarse enough that a node completes one +agent-core atomic effect boundary: + +- `call_model` accounts the request, invokes the configured provider, appends + the assistant message, and records response telemetry; +- `execute_tools` delegates authorization, execution, artifact publication, + tool history, atomic exchange persistence, and pending persistence to the + existing implementation; +- `complete_response` persists the conversation turn and commits memory; +- `complete_budget` writes the deterministic budget terminal response and + commits memory. + +Provider failures are handled in `call_model` by the existing deterministic +failure path and route directly to `END`. + +## Investigation graph + +```mermaid +flowchart TD + S([START]) -->|new turn| P[initialize_plan] + S -->|resumed exchange| R[reflect_decide] + P --> A[assistant_step] + A -->|tool calls| T[execute_tools] + A -->|final draft| F[handle_final_draft] + T -->|completed exchange| R + T -->|pending result| E([END]) + T -->|tool budget| BT[complete_max_tools] + R -->|continue| A + R -->|terminal decision| E + F -->|critique rejected| A + F -->|accepted/final| E + R -->|iteration budget| BI[complete_max_iterations] + F -->|iteration budget| BI + BT --> E + BI --> E +``` + +Planning, reflection, decision, critique, and final synthesis remain implemented +by the existing controller operations. LangGraph now owns their ordering and +terminal routes. This keeps the structured-output contracts, recovery policy, +trace events, and conversation-memory behavior shared with the native kernel. + +## Typed ephemeral state + +`agent_core.agent_graph.state.AgentGraphState` and `InvestigationGraphState` +contain only the values needed to route one in-process invocation: + +- identity and input: `user_input`, `session_id`, `context`, `turn_index`; +- provider transcript: `messages`, `assistant_message`; +- loop counters: `model_call_index`, `tool_calls_used`, `exchange_index`; +- context-reserve accounting and its one-shot warning flag; +- current `ToolExecutionStepResult`, terminal `AgentTurnResult`, and `RunTrace`. + +The investigation state additionally carries `RunOptions`, the bounded +`InvestigationState`, progress counters, and the current final draft. + +This type is internal and is not a serialized schema. LangGraph message types +are not introduced: the graph keeps using agent-core's `LLMMessage` and +`AgentTurnResult` contracts. + +## State ownership + +| Concern | Owner in this phase | Reason | +| --- | --- | --- | +| In-process conversation transitions | LangGraph when enabled | Makes direct and investigation routes explicit | +| Provider and model translation | Existing `BaseLLMProvider` adapters | Preserves the Chantier 1 boundary | +| Tool authorization and execution | Agent-core | Preserves policy and SPI contracts | +| Tool artifacts and transcript projection | Agent-core | Preserves lossless artifact semantics | +| Session and conversation memory | `SessionManager` and memory journal | Avoids storage/schema migration | +| Pending tool resume | Agent-core pending payload plus versioned graph cursor | Preserves restart and idempotence behavior | +| LLM budget and context planning | Existing scoped controllers | One controller still spans model and memory calls | +| Run traces | Existing `RunTrace` repository | Preserves current audit format | +| LangGraph checkpointing | Disabled | Prevents dual writes and ambiguous recovery authority | +| LangSmith export | Explicit agent-core opt-in | Graph state can contain the complete transcript | + +The graphs are therefore compiled without a LangGraph `BaseCheckpointSaver`. +A pending payload embeds a small `agent_graph_checkpoint` containing its schema +version, graph name, backend, and resume node. On resume, agent-core validates +that cursor, restores the authoritative transcript and counters, records an +`agent_graph_checkpoint_restored` trace event, then creates a fresh in-process +graph invocation at the post-tool route. This is deliberate: there is still one +durable source of truth, not two competing checkpoints. +Graph execution is wrapped in a disabled LangSmith tracing context unless +`CoreSettings.langchain_tracing_enabled` is explicitly set, even when tracing +is enabled in the surrounding process environment. + +## Backend selection and rollback + +The default remains: + +```python +CoreSettings(agent_kernel_backend="native") +``` + +The migrated conversation paths are enabled with: + +```python +CoreSettings(agent_kernel_backend="langgraph") +``` + +The quickstart equivalent is +`AGENT_CORE_AGENT_KERNEL_BACKEND=langgraph`. Unknown values fail when the +orchestrator is constructed. Switching back to native does not require a data +migration because both backends use the same persistence contracts and node +behavior. + +## Compatibility evidence + +The deterministic parity suite runs the native and LangGraph direct, +investigation, and deep-investigation kernels against the same scripted +providers. It compares results, metadata, investigation state, provider calls, +context blocks, memory journals, tool history, and trace events. Dedicated +cases cover initial planning, reflection/decision, critique rejection, +pending/resume, corrupt cursors, provider failure, and budget exhaustion. The +existing direct, pending, trace, memory, budget, context-planning, and +investigation suites remain the broader regression net. +The opt-in paid Azure suite additionally runs the native and LangGraph kernels +against the same real LangChain-backed model, checking the complete tool loop, +persisted trace telemetry, and pending/resume cycles. Its paired kernel evals +also cover competing-tool selection, structured investigation output, and +fully real planning/reflection/decision/finalization and deep-critique +paths. Each pair asserts the same call targets, selected tools, persistence +projection, and trace-event sequence, while reporting token and latency deltas +without treating noisy one-shot performance measurements as correctness gates. + +## Remaining platform work + +The following work remains outside these seven steps: + +- decide whether LangGraph or agent-core becomes the single durable checkpoint + authority, then design and test a one-way storage migration; +- model external pending tools with LangGraph interrupts only after tool-node + side effects are made explicitly idempotent at the interrupt boundary; +- migrate headless structured runs only if a graph adds value without weakening + their stricter before/after-tool checkpoint protocol; +- expose graph streaming or inspection through a deliberate public contract, + rather than leaking internal LangGraph objects. diff --git a/docs/public_api.md b/docs/public_api.md index 560265b..dcf5d0a 100644 --- a/docs/public_api.md +++ b/docs/public_api.md @@ -38,6 +38,29 @@ usage through checkpoints and run results. Custom providers leave `usage` unset when the upstream service does not report it; local estimates must not be presented as provider usage. +`LLMProviderConfig.model_backend` and `CoreSettings.llm_model_backend` select +the implementation behind the stable provider contract. `native` is the +default. `langchain` currently selects the Azure OpenAI LangChain adapter; +provider classes and LangChain message types remain internal implementation +details. A dedicated memory provider may override this with +`CoreSettings.memory_llm_model_backend`. + +Every built-in completion sets `LLMCompletionResult.model_backend` to `native` +or `langchain`. `LLMCallRecord` persists the same additive field through run +results and structured-task checkpoints; older payloads without the field +continue to load with `model_backend=None`. + +LangSmith tracing is disabled for LangChain model calls unless +`CoreSettings.langchain_tracing_enabled` is explicitly enabled. This scoped +override takes precedence over an inherited `LANGSMITH_TRACING` environment +setting so host applications do not export prompts or tool results by accident. + +`CoreSettings.agent_kernel_backend` independently selects `native` or +`langgraph` control flow for direct, investigate, and deep-investigate +conversation turns. LangGraph state, nodes, cursors, and compiled graphs are +internal implementation details and are not exported from a supported facade. +Headless structured runs are unchanged by this setting. + ## Optional conversation API: `agent_core.conversation` Conversation support is an adapter over runs. It contains the orchestrator, diff --git a/examples/README.md b/examples/README.md index 3bd9b17..6c520ed 100644 --- a/examples/README.md +++ b/examples/README.md @@ -36,6 +36,37 @@ JSON Schema enforcement, and `StructuredTaskRunner` final output: .venv/bin/python examples/quickstart.py --compat-check ``` +To run the same checks through LangChain's Azure OpenAI model integration: + +```bash +LLM_PROVIDER=azure_openai +AGENT_CORE_MODEL_BACKEND=langchain +AZURE_OPENAI_ENDPOINT=https://.openai.azure.com +AZURE_OPENAI_API_KEY=... +AZURE_OPENAI_API_VERSION=2025-01-01-preview +AGENT_CORE_MODEL= +AGENT_CORE_MEMORY_MODEL= + +.venv/bin/python examples/quickstart.py --compat-check +``` + +Only model invocation changes. The agent loop, tool execution, structured +tasks, persistence, and memory lifecycle remain implemented by agent-core. + +## Optional paid Azure provider tests + +The `live_llm` suite is skipped during normal development. With Azure OpenAI +credentials supplied explicitly, it exercises both native and LangChain +backends without reading an application-specific `.env` file: + +```bash +AGENT_CORE_RUN_LIVE_LLM_TESTS=1 \ +AGENT_CORE_LIVE_LLM_MODEL=gpt-5.4-mini \ +AZURE_OPENAI_ENDPOINT=https://.openai.azure.com \ +AZURE_OPENAI_API_KEY=... \ +.venv/bin/python -m pytest -m live_llm -q +``` + For Azure Anthropic / Claude on Azure Foundry, use the `/anthropic` endpoint: ```bash diff --git a/examples/quickstart.py b/examples/quickstart.py index ae3aaf9..6f98446 100644 --- a/examples/quickstart.py +++ b/examples/quickstart.py @@ -20,6 +20,7 @@ from agent_core.spi import ( BaseLLMProvider, LLMCallOptions, + LLMCompletionResult, LLMMessage, LLMProviderError, LLMToolDefinition, @@ -29,6 +30,7 @@ build_memory_provider, build_provider, build_tool_definition, + normalize_model_backend, normalize_provider_name, ) @@ -109,6 +111,9 @@ def load_dotenv(path: Path) -> None: def build_settings(*, model: str, memory_model: str, session_file: Path) -> CoreSettings: return CoreSettings( llm_provider=os.getenv("LLM_PROVIDER", "openai"), + llm_model_backend=os.getenv("AGENT_CORE_MODEL_BACKEND", "native"), + agent_kernel_backend=os.getenv("AGENT_CORE_AGENT_KERNEL_BACKEND", "native"), + langchain_tracing_enabled=_env_flag("AGENT_CORE_LANGCHAIN_TRACING_ENABLED"), openai_api_key=os.getenv("OPENAI_API_KEY"), azure_openai_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"), azure_openai_api_key=os.getenv("AZURE_OPENAI_API_KEY"), @@ -118,6 +123,7 @@ def build_settings(*, model: str, memory_model: str, session_file: Path) -> Core azure_anthropic_api_version=os.getenv("AZURE_ANTHROPIC_API_VERSION"), azure_anthropic_version=os.getenv("AZURE_ANTHROPIC_VERSION"), memory_llm_provider=os.getenv("AGENT_CORE_MEMORY_LLM_PROVIDER"), + memory_llm_model_backend=os.getenv("AGENT_CORE_MEMORY_MODEL_BACKEND"), memory_openai_api_key=os.getenv("AGENT_CORE_MEMORY_OPENAI_API_KEY"), memory_azure_openai_endpoint=os.getenv("AGENT_CORE_MEMORY_AZURE_OPENAI_ENDPOINT"), memory_azure_openai_api_key=os.getenv("AGENT_CORE_MEMORY_AZURE_OPENAI_API_KEY"), @@ -185,6 +191,10 @@ def _print_check(name: str, ok: bool, detail: str) -> bool: return ok +def _completion_text(completion: LLMCompletionResult | str) -> str: + return completion.content if isinstance(completion, LLMCompletionResult) else completion + + def _json_object_matches(content: str, expected: dict[str, Any]) -> tuple[bool, str]: try: payload = json.loads(content) @@ -205,7 +215,7 @@ def _json_object_matches(content: str, expected: dict[str, Any]) -> tuple[bool, def _run_plain_chat_check(provider: BaseLLMProvider, *, model: str) -> bool: try: - content = provider.complete_text( + completion = provider.complete_text( messages=[ LLMMessage(role="system", content="You are a compatibility checker. Answer exactly as requested."), LLMMessage(role="user", content="Return exactly: OK"), @@ -215,6 +225,7 @@ def _run_plain_chat_check(provider: BaseLLMProvider, *, model: str) -> bool: ) except LLMProviderError as exc: return _print_check("plain chat", False, f"{exc.kind}: {exc.user_message}") + content = _completion_text(completion) normalized = content.strip().strip("\"'").rstrip(".") if normalized == "OK": return _print_check("plain chat", True, f"content={content!r}") @@ -278,7 +289,7 @@ def _run_json_schema_check(provider: BaseLLMProvider, *, model: str) -> bool: "additionalProperties": False, } try: - content = provider.complete_text( + completion = provider.complete_text( messages=[ LLMMessage( role="user", @@ -301,6 +312,7 @@ def _run_json_schema_check(provider: BaseLLMProvider, *, model: str) -> bool: except LLMProviderError as exc: return _print_check("response_format json_schema", False, f"{exc.kind}: {exc.user_message}") + content = _completion_text(completion) ok, detail = _json_object_matches(content, {"ok": True, "mode": "json_schema"}) return _print_check("response_format json_schema", ok, detail) @@ -354,9 +366,16 @@ def _optional_positive_int(value: str | None) -> int | None: return parsed if parsed > 0 else None +def _env_flag(name: str) -> bool: + return os.getenv(name, "").strip().lower() in {"1", "true", "yes", "on"} + + def run_compatibility_checks(settings: CoreSettings) -> int: provider = build_provider(settings) - print(f"Running provider compatibility checks with provider={settings.llm_provider!r}, model={settings.model!r}") + print( + "Running provider compatibility checks with " + f"provider={settings.llm_provider!r}, backend={settings.llm_model_backend!r}, model={settings.model!r}" + ) print("Checks cover plain chat, OpenAI-style tool calls, JSON Schema response_format, and StructuredTaskRunner.") checks = [ @@ -373,6 +392,14 @@ def run_compatibility_checks(settings: CoreSettings) -> int: def missing_provider_config(settings: CoreSettings) -> list[str]: provider_name = normalize_provider_name(settings.llm_provider) + model_backend = normalize_model_backend(settings.llm_model_backend) + agent_kernel_backend = settings.agent_kernel_backend.strip().lower().replace("-", "_") + if agent_kernel_backend not in {"native", "langgraph"}: + return [f"unsupported AGENT_CORE_AGENT_KERNEL_BACKEND={settings.agent_kernel_backend!r}"] + if model_backend not in {"native", "langchain"}: + return [f"unsupported AGENT_CORE_MODEL_BACKEND={settings.llm_model_backend!r}"] + if model_backend == "langchain" and provider_name != "azure_openai": + return ["AGENT_CORE_MODEL_BACKEND=langchain currently requires LLM_PROVIDER=azure_openai"] if provider_name == "openai": return [] if settings.openai_api_key else ["OPENAI_API_KEY"] if provider_name == "azure_openai": diff --git a/pyproject.toml b/pyproject.toml index 2dbe78a..66d95ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,11 @@ classifiers = [ dependencies = [ "anthropic>=0.111.0", "jsonschema[format]>=4.23.0,<5", - "openai>=1.30.0", + "langchain-core>=1.6.0,<2.0.0", + "langchain-openai>=1.6.0,<2.0.0", + "langgraph>=1.2.10,<2.0.0", + "langsmith>=0.11.0,<1.0.0", + "openai>=2.45.0,<4.0.0", "requests>=2.31.0", ] @@ -61,6 +65,7 @@ python_files = ["test_*.py"] addopts = "--strict-markers" markers = [ "chaos: deterministic fault-injection and resilience tests", + "live_llm: paid Azure OpenAI integration tests, disabled unless explicitly enabled", ] [tool.mypy] diff --git a/requirements.txt b/requirements.txt index 8d54809..38b8e2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,8 @@ anthropic>=0.111.0 jsonschema[format]>=4.23.0,<5 +langchain-core>=1.6.0,<2.0.0 +langchain-openai>=1.6.0,<2.0.0 +langgraph>=1.2.10,<2.0.0 openai>=1.30.0 pytest>=7.0.0 requests>=2.31.0 diff --git a/tests/test_azure_provider_contract.py b/tests/test_azure_provider_contract.py new file mode 100644 index 0000000..1f193ba --- /dev/null +++ b/tests/test_azure_provider_contract.py @@ -0,0 +1,659 @@ +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +from types import SimpleNamespace +from typing import Any + +import httpx +import pytest +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage +from langchain_openai import AzureChatOpenAI +from openai import APITimeoutError, AuthenticationError, BadRequestError, RateLimitError + +import agent_core.llm.langchain_azure_openai_provider as langchain_provider_module +from agent_core.llm.azure_openai_provider import AzureOpenAIProvider +from agent_core.llm.base import LLMCallOptions, LLMMessage, LLMToolCall, LLMToolDefinition +from agent_core.llm.errors import LLMProviderError +from agent_core.llm.langchain_azure_openai_provider import LangChainAzureOpenAIProvider + + +class ScriptedNativeCompletions: + def __init__(self, steps: list[object]) -> None: + self.steps = list(steps) + self.requests: list[dict[str, Any]] = [] + + def create(self, **kwargs: Any) -> object: + self.requests.append(kwargs) + step = self.steps.pop(0) + if isinstance(step, Exception): + raise step + return step + + +class ScriptedLangChainModel: + def __init__(self, steps: list[object]) -> None: + self.steps = list(steps) + self.requests: list[dict[str, Any]] = [] + + def bind_tools(self, tools: list[dict[str, Any]], **kwargs: Any) -> BoundLangChainModel: + return BoundLangChainModel(self, tools=tools, bind_options=kwargs) + + def invoke(self, messages: list[object], **kwargs: Any) -> object: + return self._invoke(messages, tools=None, bind_options={}, invoke_options=kwargs) + + def _invoke( + self, + messages: list[object], + *, + tools: list[dict[str, Any]] | None, + bind_options: dict[str, Any], + invoke_options: dict[str, Any], + ) -> object: + self.requests.append( + { + "messages": [_langchain_message_to_history(message) for message in messages], + "tools": tools, + **bind_options, + **invoke_options, + } + ) + step = self.steps.pop(0) + if isinstance(step, Exception): + raise step + return step + + +@dataclass +class RawLangChainResponse: + payload: dict[str, Any] + headers: dict[str, str] + + def parse(self) -> dict[str, Any]: + return self.payload + + +class FakeLangChainOpenAIClient: + def __init__(self, payload: dict[str, Any]) -> None: + self.payload = payload + self.requests: list[dict[str, Any]] = [] + self.with_raw_response = self + + def create(self, **kwargs: Any) -> RawLangChainResponse: + self.requests.append(kwargs) + return RawLangChainResponse(payload=self.payload, headers={}) + + def parse(self, **kwargs: Any) -> RawLangChainResponse: + self.requests.append(kwargs) + return RawLangChainResponse(payload=self.payload, headers={}) + + +@dataclass +class BoundLangChainModel: + model: ScriptedLangChainModel + tools: list[dict[str, Any]] + bind_options: dict[str, Any] + + def invoke(self, messages: list[object], **kwargs: Any) -> object: + return self.model._invoke( + messages, + tools=self.tools, + bind_options=self.bind_options, + invoke_options=kwargs, + ) + + +@dataclass +class AzureProviderHarness: + provider: Any + requests: list[dict[str, Any]] + + +def _provider_harness(backend: str, steps: list[object]) -> AzureProviderHarness: + if backend == "native": + completions = ScriptedNativeCompletions(steps) + provider = AzureOpenAIProvider( + azure_endpoint="https://example.openai.azure.com", + api_key="test-key", + api_version="2025-01-01-preview", + ) + provider.client = SimpleNamespace(chat=SimpleNamespace(completions=completions)) + return AzureProviderHarness(provider=provider, requests=completions.requests) + + model = ScriptedLangChainModel(steps) + provider = LangChainAzureOpenAIProvider( + azure_endpoint="https://example.openai.azure.com", + api_key="test-key", + api_version="2025-01-01-preview", + chat_model_factory=lambda deployment: model, + ) + return AzureProviderHarness(provider=provider, requests=model.requests) + + +def _text_response(backend: str, content: str) -> object: + if backend == "native": + return SimpleNamespace( + id="chatcmpl-contract", + usage=SimpleNamespace( + prompt_tokens=21, + completion_tokens=7, + total_tokens=28, + prompt_tokens_details=SimpleNamespace(cached_tokens=4), + completion_tokens_details=SimpleNamespace(reasoning_tokens=2), + ), + choices=[SimpleNamespace(message=SimpleNamespace(content=content, tool_calls=None))], + ) + return AIMessage( + content=content, + response_metadata={"id": "chatcmpl-contract"}, + usage_metadata={ + "input_tokens": 21, + "output_tokens": 7, + "total_tokens": 28, + "input_token_details": {"cache_read": 4}, + "output_token_details": {"reasoning": 2}, + }, + ) + + +def _tool_response(backend: str) -> object: + raw_tool_call = { + "id": "call-contract", + "type": "function", + "function": {"name": "shell", "arguments": '{"command":"pwd"}'}, + } + if backend == "native": + function = SimpleNamespace(name="shell", arguments='{"command":"pwd"}') + message = SimpleNamespace( + content="checking", + tool_calls=[SimpleNamespace(id="call-contract", function=function)], + ) + return SimpleNamespace(id="chatcmpl-tool", usage=None, choices=[SimpleNamespace(message=message)]) + return AIMessage( + content="checking", + additional_kwargs={"tool_calls": [raw_tool_call]}, + response_metadata={"id": "chatcmpl-tool"}, + ) + + +def _multiple_tool_response(backend: str) -> object: + calls = [ + ("call-valid", "shell", '{"command":"pwd"}'), + ("call-invalid", "shell", '{"command":'), + ] + if backend == "native": + return SimpleNamespace( + id="chatcmpl-multiple-tools", + usage=None, + choices=[ + SimpleNamespace( + message=SimpleNamespace( + content="", + tool_calls=[ + SimpleNamespace( + id=call_id, + function=SimpleNamespace(name=name, arguments=arguments), + ) + for call_id, name, arguments in calls + ], + ) + ) + ], + ) + return AIMessage( + content="", + additional_kwargs={ + "tool_calls": [ + { + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": arguments}, + } + for call_id, name, arguments in calls + ] + }, + response_metadata={"id": "chatcmpl-multiple-tools"}, + ) + + +def _unsupported_reasoning_effort_error() -> BadRequestError: + return BadRequestError( + "Unrecognized request argument supplied: reasoning_effort", + response=httpx.Response( + 400, + request=httpx.Request("POST", "https://example.openai.azure.com/openai/deployments/test/chat/completions"), + ), + body={"error": {"message": "Unrecognized request argument supplied: reasoning_effort"}}, + ) + + +def _authentication_error() -> AuthenticationError: + return AuthenticationError( + "Invalid API key", + response=httpx.Response( + 401, + request=httpx.Request("POST", "https://example.openai.azure.com/chat/completions"), + ), + body={"error": {"message": "Invalid API key"}}, + ) + + +def _timeout_error() -> APITimeoutError: + return APITimeoutError( + request=httpx.Request("POST", "https://example.openai.azure.com/chat/completions") + ) + + +def _rate_limit_error() -> RateLimitError: + return RateLimitError( + "Rate limit reached", + response=httpx.Response( + 429, + request=httpx.Request("POST", "https://example.openai.azure.com/chat/completions"), + ), + body={"error": {"message": "Rate limit reached"}}, + ) + + +@pytest.mark.parametrize("backend", ["native", "langchain"]) +def test_azure_provider_contract_preserves_text_usage_and_request_id(backend: str) -> None: + harness = _provider_harness(backend, [_text_response(backend, "same answer")]) + + result = harness.provider.complete_text( + messages=[LLMMessage(role="system", content="be concise"), LLMMessage(role="user", content="hello")], + model="deployment-name", + temperature=0.2, + ) + + assert result.content == "same answer" + assert result.tool_calls == [] + assert result.provider == "azure_openai" + assert result.model_backend == backend + assert result.model == "deployment-name" + assert result.provider_request_id == "chatcmpl-contract" + assert result.provider_attempts == 1 + assert result.usage is not None + assert result.usage.to_dict() == { + "input_tokens": 21, + "output_tokens": 7, + "total_tokens": 28, + "cached_input_tokens": 4, + "cache_creation_input_tokens": None, + "reasoning_output_tokens": 2, + "source": "provider", + } + assert harness.requests[0]["messages"] == [ + {"role": "system", "content": "be concise"}, + {"role": "user", "content": "hello"}, + ] + assert harness.requests[0]["temperature"] == 0.2 + + +@pytest.mark.parametrize("backend", ["native", "langchain"]) +def test_azure_provider_contract_preserves_tool_history_schema_and_result(backend: str) -> None: + harness = _provider_harness(backend, [_tool_response(backend)]) + messages = [ + LLMMessage(role="user", content="where am I?"), + LLMMessage( + role="assistant", + content="", + tool_calls=[LLMToolCall(id="call-before", name="shell", arguments_json='{"command":"whoami"}')], + ), + LLMMessage(role="tool", content="tester", tool_call_id="call-before"), + ] + tools = [ + LLMToolDefinition( + name="shell", + description="Run a shell command", + parameters={"type": "object", "properties": {"command": {"type": "string"}}}, + ) + ] + + result = harness.provider.complete_with_tools( + messages=messages, + tools=tools, + model="deployment-name", + temperature=0.0, + options=LLMCallOptions(max_output_tokens=250), + ) + + assert result.content == "checking" + assert result.tool_calls == [ + LLMToolCall(id="call-contract", name="shell", arguments_json='{"command":"pwd"}') + ] + assert harness.requests[0]["messages"] == [message.to_history_dict() for message in messages] + assert harness.requests[0]["tools"] == [ + { + "type": "function", + "function": { + "name": "shell", + "description": "Run a shell command", + "parameters": {"type": "object", "properties": {"command": {"type": "string"}}}, + }, + } + ] + assert harness.requests[0]["tool_choice"] == "auto" + assert harness.requests[0]["parallel_tool_calls"] is True + assert harness.requests[0]["max_tokens"] == 250 + + +@pytest.mark.parametrize("backend", ["native", "langchain"]) +def test_azure_provider_contract_preserves_multiple_and_invalid_tool_calls(backend: str) -> None: + harness = _provider_harness(backend, [_multiple_tool_response(backend)]) + + result = harness.provider.complete_with_tools( + messages=[LLMMessage(role="user", content="run both")], + tools=[ + LLMToolDefinition( + name="shell", + description="Run a shell command", + parameters={"type": "object", "properties": {"command": {"type": "string"}}}, + ) + ], + model="deployment-name", + temperature=0.0, + ) + + assert result.tool_calls == [ + LLMToolCall(id="call-valid", name="shell", arguments_json='{"command":"pwd"}'), + LLMToolCall(id="call-invalid", name="shell", arguments_json='{"command":'), + ] + + +@pytest.mark.parametrize("backend", ["native", "langchain"]) +def test_azure_provider_contract_combines_tools_and_structured_response_format(backend: str) -> None: + harness = _provider_harness(backend, [_tool_response(backend)]) + response_format = { + "type": "json_schema", + "json_schema": { + "name": "tool_result", + "strict": True, + "schema": { + "type": "object", + "properties": {"ok": {"type": "boolean"}}, + "required": ["ok"], + "additionalProperties": False, + }, + }, + } + + harness.provider.complete_with_tools( + messages=[LLMMessage(role="user", content="check")], + tools=[ + LLMToolDefinition( + name="shell", + description="Run a shell command", + parameters={"type": "object", "properties": {"command": {"type": "string"}}}, + ) + ], + model="deployment-name", + temperature=0.0, + options=LLMCallOptions(response_format=response_format), + ) + + assert harness.requests[0]["tools"] + assert harness.requests[0]["response_format"] == response_format + + +@pytest.mark.parametrize("backend", ["native", "langchain"]) +def test_azure_provider_contract_learns_rejected_reasoning_parameter(backend: str) -> None: + harness = _provider_harness( + backend, + [_unsupported_reasoning_effort_error(), _text_response(backend, "ok"), _text_response(backend, "again")], + ) + + first_result = harness.provider.complete_text( + messages=[LLMMessage(role="user", content="hello")], + model="custom-deployment", + temperature=0.0, + options=LLMCallOptions(reasoning_effort="high"), + ) + second_result = harness.provider.complete_text( + messages=[LLMMessage(role="user", content="again")], + model="custom-deployment", + temperature=0.0, + options=LLMCallOptions(reasoning_effort="high"), + ) + + assert first_result.provider_attempts == 2 + assert second_result.provider_attempts == 1 + assert harness.requests[0]["reasoning_effort"] == "high" + assert "reasoning_effort" not in harness.requests[1] + assert "reasoning_effort" not in harness.requests[2] + + +@pytest.mark.parametrize("backend", ["native", "langchain"]) +@pytest.mark.parametrize( + ("endpoint", "api_key", "expected_detail"), + [(None, "key", "ENDPOINT"), ("https://example.openai.azure.com", None, "API_KEY")], +) +def test_azure_provider_contract_rejects_missing_configuration( + backend: str, + endpoint: str | None, + api_key: str | None, + expected_detail: str, +) -> None: + if backend == "native": + provider: Any = AzureOpenAIProvider(azure_endpoint=endpoint, api_key=api_key) + else: + provider = LangChainAzureOpenAIProvider(azure_endpoint=endpoint, api_key=api_key) + + with pytest.raises(LLMProviderError) as exc_info: + provider.complete_text( + messages=[LLMMessage(role="user", content="hello")], + model="deployment-name", + temperature=0.0, + ) + + assert exc_info.value.kind == "configuration_error" + assert expected_detail in exc_info.value.detail + + +@pytest.mark.parametrize("backend", ["native", "langchain"]) +@pytest.mark.parametrize( + ("error", "expected_kind"), + [(_authentication_error(), "configuration_error"), (_timeout_error(), "request_error")], +) +def test_azure_provider_contract_maps_common_sdk_errors( + backend: str, + error: Exception, + expected_kind: str, + monkeypatch, +) -> None: + monkeypatch.setenv("AGENT_CORE_LLM_RETRY_MAX_ATTEMPTS", "1") + harness = _provider_harness(backend, [error]) + + with pytest.raises(LLMProviderError) as exc_info: + harness.provider.complete_text( + messages=[LLMMessage(role="user", content="hello")], + model="deployment-name", + temperature=0.0, + ) + + assert exc_info.value.kind == expected_kind + + +@pytest.mark.parametrize("backend", ["native", "langchain"]) +def test_azure_provider_contract_maps_exhausted_rate_limit(backend: str, monkeypatch) -> None: + monkeypatch.setenv("AGENT_CORE_LLM_RETRY_MAX_ATTEMPTS", "1") + harness = _provider_harness(backend, [_rate_limit_error()]) + + with pytest.raises(LLMProviderError) as exc_info: + harness.provider.complete_text( + messages=[LLMMessage(role="user", content="hello")], + model="deployment-name", + temperature=0.0, + ) + + assert exc_info.value.kind == "rate_limit_error" + assert len(harness.requests) == 1 + + +@pytest.mark.parametrize("backend", ["native", "langchain"]) +def test_azure_provider_contract_rejects_unusable_response(backend: str) -> None: + response: object = ( + SimpleNamespace(choices=[]) + if backend == "native" + else HumanMessage(content="not an assistant response") + ) + harness = _provider_harness(backend, [response]) + + with pytest.raises(LLMProviderError) as exc_info: + harness.provider.complete_text( + messages=[LLMMessage(role="user", content="hello")], + model="deployment-name", + temperature=0.0, + ) + + assert exc_info.value.kind == "response_error" + + +def test_langchain_adapter_flattens_text_content_blocks() -> None: + model = ScriptedLangChainModel( + [ + AIMessage( + content=[ + {"type": "text", "text": "hello "}, + {"type": "text", "text": "world"}, + {"type": "reasoning", "reasoning": "hidden"}, + ] + ) + ] + ) + provider = LangChainAzureOpenAIProvider( + azure_endpoint="https://example.openai.azure.com", + api_key="test-key", + chat_model_factory=lambda deployment: model, + ) + + result = provider.complete_text( + messages=[LLMMessage(role="user", content="hello")], + model="deployment-name", + temperature=0.0, + ) + + assert result.content == "hello world" + + +@pytest.mark.parametrize("tracing_enabled", [False, True]) +def test_langchain_adapter_scopes_langsmith_tracing(monkeypatch, tracing_enabled: bool) -> None: + observed: list[bool | None] = [] + + @contextmanager + def fake_tracing_context(*, enabled=None, **kwargs): + observed.append(enabled) + yield + + monkeypatch.setattr(langchain_provider_module.ls, "tracing_context", fake_tracing_context) + model = ScriptedLangChainModel([AIMessage(content="ok")]) + provider = LangChainAzureOpenAIProvider( + azure_endpoint="https://example.openai.azure.com", + api_key="test-key", + chat_model_factory=lambda deployment: model, + tracing_enabled=tracing_enabled, + ) + + provider.complete_text( + messages=[LLMMessage(role="user", content="hello")], + model="deployment-name", + temperature=0.0, + ) + + assert observed == [tracing_enabled] + + +def test_langchain_adapter_integrates_with_real_azure_chat_model_conversion() -> None: + client = FakeLangChainOpenAIClient( + { + "id": "chatcmpl-langchain-integration", + "model": "deployment-name", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": "checking", + "tool_calls": [ + { + "id": "call-integration", + "type": "function", + "function": {"name": "shell", "arguments": '{"command":"pwd"}'}, + } + ], + }, + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "prompt_tokens_details": {"cached_tokens": 3}, + "completion_tokens_details": {"reasoning_tokens": 1}, + }, + } + ) + chat_model = AzureChatOpenAI( + azure_endpoint="https://example.openai.azure.com", + api_key="test-key", # type: ignore[arg-type] + api_version="2025-01-01-preview", + azure_deployment="deployment-name", + max_retries=0, + ) + chat_model.client = client + chat_model.root_client = SimpleNamespace(chat=SimpleNamespace(completions=client)) + provider = LangChainAzureOpenAIProvider( + azure_endpoint="https://example.openai.azure.com", + api_key="test-key", + chat_model_factory=lambda deployment: chat_model, + ) + + response_format = { + "type": "json_schema", + "json_schema": { + "name": "result", + "strict": True, + "schema": {"type": "object", "properties": {}, "additionalProperties": False}, + }, + } + result = provider.complete_with_tools( + messages=[LLMMessage(role="user", content="where am I?")], + tools=[ + LLMToolDefinition( + name="shell", + description="Run a command", + parameters={"type": "object", "properties": {"command": {"type": "string"}}}, + ) + ], + model="deployment-name", + temperature=0.0, + options=LLMCallOptions(max_output_tokens=120, response_format=response_format), + ) + + assert result.content == "checking" + assert result.provider_request_id == "chatcmpl-langchain-integration" + assert result.tool_calls == [ + LLMToolCall(id="call-integration", name="shell", arguments_json='{"command":"pwd"}') + ] + assert result.usage is not None + assert result.usage.cached_input_tokens == 3 + assert result.usage.reasoning_output_tokens == 1 + assert client.requests[0]["messages"] == [{"content": "where am I?", "role": "user"}] + assert client.requests[0]["parallel_tool_calls"] is True + assert client.requests[0]["max_tokens"] == 120 + assert client.requests[0]["response_format"] == response_format + + +def _langchain_message_to_history(message: object) -> dict[str, Any]: + if isinstance(message, SystemMessage): + return {"role": "system", "content": message.content} + if isinstance(message, HumanMessage): + return {"role": "user", "content": message.content} + if isinstance(message, ToolMessage): + return {"role": "tool", "content": message.content, "tool_call_id": message.tool_call_id} + if isinstance(message, AIMessage): + payload: dict[str, Any] = {"role": "assistant", "content": message.content} + if raw_tool_calls := message.additional_kwargs.get("tool_calls"): + payload["tool_calls"] = raw_tool_calls + return payload + raise AssertionError(f"Unexpected LangChain message: {type(message).__name__}") diff --git a/tests/test_examples.py b/tests/test_examples.py index 351247e..d29d8c9 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -3,7 +3,7 @@ import importlib.util from pathlib import Path -from agent_core.llm.base import LLMCompletionResult +from agent_core.llm.base import LLMCompletionResult, LLMTokenUsage ROOT = Path(__file__).resolve().parents[1] @@ -84,6 +84,29 @@ def test_quickstart_build_settings_reads_azure_anthropic_env(tmp_path, monkeypat assert quickstart.missing_provider_config(settings) == [] +def test_quickstart_build_settings_reads_langchain_model_backends(tmp_path, monkeypatch) -> None: + quickstart = load_example("quickstart") + monkeypatch.setenv("LLM_PROVIDER", "azure_openai") + monkeypatch.setenv("AGENT_CORE_MODEL_BACKEND", "langchain") + monkeypatch.setenv("AGENT_CORE_MEMORY_MODEL_BACKEND", "native") + monkeypatch.setenv("AGENT_CORE_AGENT_KERNEL_BACKEND", "langgraph") + monkeypatch.setenv("AGENT_CORE_LANGCHAIN_TRACING_ENABLED", "true") + monkeypatch.setenv("AZURE_OPENAI_ENDPOINT", "https://example.openai.azure.com") + monkeypatch.setenv("AZURE_OPENAI_API_KEY", "test-key") + + settings = quickstart.build_settings( + model="deployment-name", + memory_model="memory-deployment", + session_file=tmp_path / "session.json", + ) + + assert settings.llm_model_backend == "langchain" + assert settings.memory_llm_model_backend == "native" + assert settings.agent_kernel_backend == "langgraph" + assert settings.langchain_tracing_enabled is True + assert quickstart.missing_provider_config(settings) == [] + + def test_quickstart_structured_task_compat_check_returns_bool(tmp_path) -> None: quickstart = load_example("quickstart") @@ -100,6 +123,29 @@ def complete_with_tools(self, **kwargs): assert quickstart._run_structured_task_check(settings, FakeProvider()) is True +def test_quickstart_plain_chat_check_accepts_typed_completion_result() -> None: + quickstart = load_example("quickstart") + + class FakeProvider: + def complete_text(self, **kwargs): + return LLMCompletionResult( + content="OK", + usage=LLMTokenUsage(input_tokens=3, output_tokens=1, total_tokens=4), + ) + + assert quickstart._run_plain_chat_check(FakeProvider(), model="fake-model") is True + + +def test_quickstart_json_schema_check_accepts_typed_completion_result() -> None: + quickstart = load_example("quickstart") + + class FakeProvider: + def complete_text(self, **kwargs): + return LLMCompletionResult(content='{"ok": true, "mode": "json_schema"}') + + assert quickstart._run_json_schema_check(FakeProvider(), model="fake-model") is True + + def test_pending_tool_resume_example_runs(tmp_path) -> None: pending_demo = load_example("pending_tool_resume") diff --git a/tests/test_investigation_modes.py b/tests/test_investigation_modes.py index bcbad37..dfc8f4b 100644 --- a/tests/test_investigation_modes.py +++ b/tests/test_investigation_modes.py @@ -205,6 +205,7 @@ def build_orchestrator( policy_engine: PolicyEngine | None = None, pending: bool = False, domain_hooks: DomainHooks | None = None, + agent_kernel_backend: str = "native", ): settings = CoreSettings( openai_api_key="test", @@ -214,6 +215,7 @@ def build_orchestrator( base_system_prompt="system", turn_memory_synthesis_prompt="memory", max_active_context_tokens=100000, + agent_kernel_backend=agent_kernel_backend, ) registry = ToolRegistry() registry.register(PendingTool() if pending else EchoTool()) diff --git a/tests/test_langgraph_agent_kernel.py b/tests/test_langgraph_agent_kernel.py new file mode 100644 index 0000000..8b08a44 --- /dev/null +++ b/tests/test_langgraph_agent_kernel.py @@ -0,0 +1,289 @@ +from __future__ import annotations + +import json +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import pytest + +from agent_core.llm.base import LLMCompletionResult, LLMToolCall +from agent_core.llm.errors import LLMProviderError +from agent_core.orchestrator import AgentOrchestrator +from agent_core.policy_engine import PolicyEngine +from agent_core.session_manager import SessionManager +from agent_core.session_repo import SessionRepository +from agent_core.settings import CoreSettings +from agent_core.tool_registry import ToolRegistry +from agent_core.tools import build_tool_definition +from agent_core.types import ToolResult +from tests.run_helpers import resume_turn, run_turn, turn_memory_payload + + +class ScriptedProvider: + def __init__(self, responses: list[LLMCompletionResult | Exception]) -> None: + self.responses = list(responses) + self.chat_calls = 0 + + def complete_with_tools(self, *, messages, tools, model, temperature, options=None): + self.chat_calls += 1 + response = self.responses.pop(0) + if isinstance(response, Exception): + raise response + return response + + def complete_text(self, *, messages, model, temperature, options=None): + return json.dumps(turn_memory_payload(objective="Kernel parity")) + + +class EchoTool: + name = "echo" + description = "Echo a value." + + def schema(self): + return build_tool_definition( + name=self.name, + description=self.description, + parameters={ + "type": "object", + "properties": {"value": {"type": "string"}}, + "required": ["value"], + "additionalProperties": False, + }, + ) + + def execute(self, arguments, context): + return ToolResult(ok=True, content=f"echo:{arguments['value']}") + + +class PendingTool(EchoTool): + name = "pending" + description = "Wait for an externally supplied result." + + def execute(self, arguments, context): + return ToolResult.pending_result("waiting", metadata={"job_id": arguments["value"]}) + + +def tool_call(name: str = "echo", *, value: str = "hello") -> LLMCompletionResult: + return LLMCompletionResult( + content="", + tool_calls=[ + LLMToolCall( + id="call-1", + name=name, + arguments_json=json.dumps({"value": value}), + ) + ], + provider="azure_openai", + model_backend="langchain", + model="deployment", + provider_request_id="request-tool", + ) + + +def build_orchestrator( + root: Path, + *, + backend: str, + provider: ScriptedProvider, + tool: EchoTool | None = None, + max_tool_calls: int = 100, +) -> AgentOrchestrator: + settings = CoreSettings( + openai_api_key="test", + model="fake", + memory_model="fake", + session_file=root / "session.json", + base_system_prompt="system", + turn_memory_synthesis_prompt="memory", + max_active_context_tokens=100000, + max_tool_calls_per_turn=max_tool_calls, + agent_kernel_backend=backend, + ) + registry = ToolRegistry() + registry.register(tool or EchoTool()) + return AgentOrchestrator( + settings=settings, + provider=provider, + registry=registry, + session_manager=SessionManager(SessionRepository(settings.session_file)), + policy_engine=PolicyEngine(), + ) + + +def _trace_event_types(orchestrator: AgentOrchestrator, run_trace_id: str) -> list[str]: + trace = orchestrator.session_manager.load_run_trace(run_trace_id) + assert trace is not None + events = trace["events"] + assert isinstance(events, list) + return [event["type"] for event in events if isinstance(event, dict)] + + +def _direct_tool_snapshot(root: Path, backend: str) -> dict[str, Any]: + provider = ScriptedProvider([tool_call(), LLMCompletionResult(content="final")]) + orchestrator = build_orchestrator(root, backend=backend, provider=provider) + + result = run_turn(orchestrator, "echo once") + trace_id = result.metadata["run_trace_id"] + assert isinstance(trace_id, str) + trace = orchestrator.session_manager.load_run_trace(trace_id) + assert trace is not None + assert trace["options"]["agent_kernel_backend"] == backend + blocks = orchestrator.session_manager.get_context_blocks() + return { + "kernel_backend": orchestrator._direct_turn_kernel.backend, + "status": result.status, + "content": result.content, + "provider_calls": provider.chat_calls, + "block_kinds": [block.kind for block in blocks], + "tool_statuses": [item["status"] for item in orchestrator.session_manager.get_state()["tool_history"]], + "trace_events": _trace_event_types(orchestrator, trace_id), + } + + +def test_langgraph_direct_tool_loop_matches_native_observable_behavior(tmp_path: Path) -> None: + native = _direct_tool_snapshot(tmp_path / "native", "native") + langgraph = _direct_tool_snapshot(tmp_path / "langgraph", "langgraph") + + assert native.pop("kernel_backend") == "native" + assert langgraph.pop("kernel_backend") == "langgraph" + assert langgraph == native + + +def test_langgraph_direct_kernel_is_a_multi_node_graph(tmp_path: Path) -> None: + orchestrator = build_orchestrator( + tmp_path, + backend="langgraph", + provider=ScriptedProvider([LLMCompletionResult(content="done")]), + ) + + node_names = set(orchestrator._direct_turn_kernel.graph.get_graph().nodes) + + assert {"call_model", "execute_tools", "complete_response", "complete_budget"}.issubset(node_names) + + +def test_langgraph_tracing_does_not_inherit_process_opt_in(tmp_path: Path, monkeypatch) -> None: + import langsmith as ls + + observed: list[bool | str | None] = [] + original_tracing_context = ls.tracing_context + + @contextmanager + def recording_tracing_context(*args, **kwargs): + observed.append(kwargs.get("enabled")) + with original_tracing_context(*args, **kwargs): + yield + + monkeypatch.setenv("LANGSMITH_TRACING", "true") + monkeypatch.setattr(ls, "tracing_context", recording_tracing_context) + orchestrator = build_orchestrator( + tmp_path, + backend="langgraph", + provider=ScriptedProvider([LLMCompletionResult(content="done")]), + ) + + result = run_turn(orchestrator, "answer") + + assert result.content == "done" + assert observed[0] is False + + +def test_langgraph_direct_pending_resume_uses_existing_persistence_contract(tmp_path: Path) -> None: + provider = ScriptedProvider([tool_call("pending", value="job-1"), LLMCompletionResult(content="resolved")]) + orchestrator = build_orchestrator( + tmp_path, + backend="langgraph", + provider=provider, + tool=PendingTool(), + ) + + pending = run_turn(orchestrator, "start external work") + assert pending.status == "pending_tool_result" + assert pending.pending_id + persisted = orchestrator.session_manager.get_state()["meta"][AgentOrchestrator.PENDING_TURN_META_KEY] + assert persisted["pending_id"] == pending.pending_id + assert persisted["agent_graph_checkpoint"] == { + "schema_version": "1", + "graph": "direct", + "backend": "langgraph", + "resume_node": "resume_tool_exchange", + } + trace_id = str(persisted["run_trace_id"]) + + completed = resume_turn(orchestrator, pending_id=pending.pending_id, tool_content="external result") + + assert completed.status == "completed" + assert completed.content == "resolved" + assert provider.chat_calls == 2 + assert [block.kind for block in orchestrator.session_manager.get_context_blocks()] == [ + "tool_exchange", + "conversation_turn", + ] + assert "agent_graph_checkpoint_restored" in _trace_event_types(orchestrator, trace_id) + + +def test_langgraph_direct_rejects_corrupt_pending_checkpoint(tmp_path: Path) -> None: + provider = ScriptedProvider([tool_call("pending", value="job-1"), LLMCompletionResult(content="unused")]) + orchestrator = build_orchestrator( + tmp_path, + backend="langgraph", + provider=provider, + tool=PendingTool(), + ) + + pending = run_turn(orchestrator, "start external work") + assert pending.pending_id + persisted = orchestrator.session_manager.get_state()["meta"][AgentOrchestrator.PENDING_TURN_META_KEY] + persisted["agent_graph_checkpoint"]["graph"] = "investigation" + orchestrator.session_manager.set_meta_value(AgentOrchestrator.PENDING_TURN_META_KEY, persisted) + + completed = resume_turn(orchestrator, pending_id=pending.pending_id, tool_content="must not be injected") + + assert completed.status == "completed" + assert completed.content == "Pending agent graph checkpoint is incompatible with the current graph contract." + assert provider.chat_calls == 1 + assert orchestrator.session_manager.get_context_blocks() == [] + + +def test_langgraph_direct_preserves_provider_failure_handling(tmp_path: Path) -> None: + provider = ScriptedProvider( + [ + LLMProviderError( + kind="request_error", + user_message="The model is temporarily unavailable.", + detail="synthetic provider failure", + ) + ] + ) + orchestrator = build_orchestrator(tmp_path, backend="langgraph", provider=provider) + + result = run_turn(orchestrator, "answer") + + assert result.status == "completed" + assert result.content == "The model is temporarily unavailable." + assert result.metadata["stop_reason"] == "provider_failure" + assert result.metadata["provider_error_kind"] == "request_error" + + +def test_langgraph_direct_preserves_tool_budget_completion(tmp_path: Path) -> None: + provider = ScriptedProvider([tool_call()]) + orchestrator = build_orchestrator( + tmp_path, + backend="langgraph", + provider=provider, + max_tool_calls=0, + ) + + result = run_turn(orchestrator, "echo") + + assert result.content == "Maximum number of tool calls reached for this turn." + assert orchestrator.session_manager.get_state()["tool_history"][0]["status"] == "budget_exhausted" + + +def test_unknown_agent_kernel_backend_fails_at_orchestrator_construction(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="Unsupported agent kernel backend"): + build_orchestrator( + tmp_path, + backend="custom", + provider=ScriptedProvider([LLMCompletionResult(content="unused")]), + ) diff --git a/tests/test_langgraph_investigation_kernel.py b/tests/test_langgraph_investigation_kernel.py new file mode 100644 index 0000000..cae4a12 --- /dev/null +++ b/tests/test_langgraph_investigation_kernel.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from agent_core.agent_graph.investigation import LangGraphInvestigationKernel +from agent_core.llm.base import LLMCompletionResult +from agent_core.orchestrator import AgentOrchestrator +from agent_core.run_options import RunOptions +from tests.run_helpers import resume_turn, run_turn +from tests.test_investigation_modes import ( + ScriptedProvider, + build_orchestrator, + critique_payload, + decision_payload, + reflection_payload, + tool_call, +) + + +def _trace_events(orchestrator: AgentOrchestrator, trace_id: str) -> list[str]: + trace = orchestrator.session_manager.load_run_trace(trace_id) + assert trace is not None + return [event["type"] for event in trace["events"]] + + +def _initial_plan() -> dict[str, Any]: + return { + "objective": "investigate", + "plan": ["collect evidence"], + "facts": [], + "hypotheses": [], + "evidence_gaps": ["evidence not collected"], + "completed_actions": [], + "next_actions": ["collect evidence"], + "risk_notes": [], + "confidence": 0.1, + "stop_reason": None, + "metadata": {}, + } + + +def _no_tool_snapshot(root: Path, backend: str) -> dict[str, Any]: + provider = ScriptedProvider( + chat=[LLMCompletionResult(content="draft")], + plans=[_initial_plan()], + finals=[LLMCompletionResult(content="final answer")], + ) + orchestrator = build_orchestrator(root, provider, agent_kernel_backend=backend) + + result = run_turn(orchestrator, "investigate", options=RunOptions.investigate(max_iterations=1)) + + trace_id = str(result.metadata["run_trace_id"]) + trace = orchestrator.session_manager.load_run_trace(trace_id) + assert trace is not None + return { + "content": result.content, + "mode": result.metadata["mode"], + "iterations_used": result.metadata["iterations_used"], + "tool_calls_used": result.metadata["tool_calls_used"], + "stop_reason": result.metadata["stop_reason"], + "final_response_origin": result.metadata["final_response_origin"], + "block_kinds": [block.kind for block in orchestrator.session_manager.get_context_blocks()], + "trace_events": _trace_events(orchestrator, trace_id), + "trace_backend": trace["options"]["agent_kernel_backend"], + } + + +def test_langgraph_investigation_initial_plan_and_finalization_match_native(tmp_path: Path) -> None: + native = _no_tool_snapshot(tmp_path / "native", "native") + langgraph = _no_tool_snapshot(tmp_path / "langgraph", "langgraph") + + assert native.pop("trace_backend") == "native" + assert langgraph.pop("trace_backend") == "langgraph" + assert langgraph == native + + +def test_langgraph_investigation_kernel_exposes_explicit_control_nodes(tmp_path: Path) -> None: + orchestrator = build_orchestrator( + tmp_path, + ScriptedProvider(chat=[]), + agent_kernel_backend="langgraph", + ) + kernel = LangGraphInvestigationKernel(orchestrator._build_investigation_controller()) + + assert { + "initialize_plan", + "assistant_step", + "execute_tools", + "reflect_decide", + "handle_final_draft", + "complete_max_tools", + "complete_max_iterations", + }.issubset(kernel.graph.get_graph().nodes) + + +def _tool_snapshot(root: Path, backend: str) -> dict[str, Any]: + provider = ScriptedProvider( + chat=[tool_call(value="fact")], + reflections=[ + reflection_payload( + new_facts=["echo returned fact"], + remaining_gaps=["need second source"], + recommended_next_actions=["verify independently"], + should_continue=False, + ) + ], + decisions=[decision_payload("final", reason_summary="enough evidence")], + ) + orchestrator = build_orchestrator(root, provider, agent_kernel_backend=backend) + + result = run_turn( + orchestrator, + "investigate", + options=RunOptions.investigate(max_iterations=2, require_initial_plan=False), + ) + + trace_id = str(result.metadata["run_trace_id"]) + return { + "content": result.content, + "stop_reason": result.metadata["stop_reason"], + "iterations_used": result.metadata["iterations_used"], + "tool_calls_used": result.metadata["tool_calls_used"], + "facts": result.metadata["investigation_state"]["facts"], + "journal_kinds": [item.kind for item in orchestrator.session_manager.get_memory_journal().exchanges], + "tool_statuses": [item["status"] for item in orchestrator.session_manager.get_state()["tool_history"]], + "trace_events": _trace_events(orchestrator, trace_id), + } + + +def test_langgraph_investigation_tool_reflection_decision_matches_native(tmp_path: Path) -> None: + assert _tool_snapshot(tmp_path / "langgraph", "langgraph") == _tool_snapshot(tmp_path / "native", "native") + + +def _deep_snapshot(root: Path, backend: str) -> dict[str, Any]: + provider = ScriptedProvider( + chat=[LLMCompletionResult(content="unsupported draft"), LLMCompletionResult(content="revised draft")], + critiques=[ + critique_payload(approved=False, unsupported_claims=["unsupported claim"]), + critique_payload(approved=True), + ], + ) + orchestrator = build_orchestrator(root, provider, agent_kernel_backend=backend) + + result = run_turn( + orchestrator, + "answer", + options=RunOptions.deep_investigate( + max_iterations=2, + require_initial_plan=False, + ), + ) + + return { + "content": result.content, + "mode": result.metadata["mode"], + "stop_reason": result.metadata["stop_reason"], + "iterations_used": result.metadata["iterations_used"], + "evidence_gaps": result.metadata["investigation_state"]["evidence_gaps"], + "next_actions": result.metadata["investigation_state"]["next_actions"], + } + + +def test_langgraph_deep_investigation_critique_loop_matches_native(tmp_path: Path) -> None: + assert _deep_snapshot(tmp_path / "langgraph", "langgraph") == _deep_snapshot(tmp_path / "native", "native") + + +@pytest.mark.parametrize("backend", ["native", "langgraph"]) +def test_investigation_pending_checkpoint_and_resume_contract(tmp_path: Path, backend: str) -> None: + provider = ScriptedProvider( + chat=[tool_call(name="pending", value="job-1")], + reflections=[reflection_payload(new_facts=["external result arrived"], should_continue=False)], + decisions=[decision_payload("final", reason_summary="pending result resolved")], + ) + orchestrator = build_orchestrator( + tmp_path, + provider, + pending=True, + agent_kernel_backend=backend, + ) + + pending = run_turn( + orchestrator, + "start pending", + options=RunOptions.investigate(max_iterations=2, require_initial_plan=False), + ) + + assert pending.status == "pending_tool_result" + assert pending.pending_id + payload = orchestrator.session_manager.get_state()["meta"][AgentOrchestrator.PENDING_TURN_META_KEY] + assert payload["agent_graph_checkpoint"] == { + "schema_version": "1", + "graph": "investigation", + "backend": backend, + "resume_node": "resume_tool_exchange", + } + + completed = resume_turn(orchestrator, pending_id=pending.pending_id, tool_content="done") + + assert completed.status == "completed" + assert "external result arrived" in completed.content + assert completed.metadata["mode"] == "investigate" + assert [block.kind for block in orchestrator.session_manager.get_context_blocks()] == [ + "tool_exchange", + "conversation_turn", + ] diff --git a/tests/test_langgraph_structured_task_kernel.py b/tests/test_langgraph_structured_task_kernel.py new file mode 100644 index 0000000..7c7cb4a --- /dev/null +++ b/tests/test_langgraph_structured_task_kernel.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from agent_core.llm.base import LLMCompletionResult, LLMToolCall +from agent_core.policy_engine import PolicyEngine +from agent_core.settings import CoreSettings +from agent_core.structured_tasks import ( + StructuredOutputContract, + StructuredTaskRunner, + StructuredTaskSpec, +) +from agent_core.tool_registry import ToolRegistry +from agent_core.tools import build_tool_definition +from agent_core.types import ToolResult +from tests.run_helpers import execution_context + + +class ScriptedProvider: + def __init__(self) -> None: + self.responses = [ + LLMCompletionResult( + content="", + tool_calls=[ + LLMToolCall( + id="echo-1", + name="echo", + arguments_json=json.dumps({"value": "inventory"}), + ) + ], + ), + LLMCompletionResult(content="Investigation complete."), + LLMCompletionResult(content=json.dumps({"summary": "echo:inventory"})), + ] + + def complete_with_tools(self, **kwargs) -> LLMCompletionResult: + _ = kwargs + return self.responses.pop(0) + + +class EchoTool: + name = "echo" + description = "Echo one value." + + def schema(self): + return build_tool_definition( + name=self.name, + description=self.description, + parameters={ + "type": "object", + "properties": {"value": {"type": "string"}}, + "required": ["value"], + "additionalProperties": False, + }, + ) + + def execute(self, arguments: dict, context) -> ToolResult: + _ = context + return ToolResult(ok=True, content=f"echo:{arguments['value']}") + + +def _runner(root: Path, backend: str) -> StructuredTaskRunner: + registry = ToolRegistry() + registry.register(EchoTool()) + return StructuredTaskRunner( + settings=CoreSettings( + allowed_read_roots=[root], + knowledge_base_dir=root / "knowledge", + agent_kernel_backend=backend, + ), + provider=ScriptedProvider(), + tool_registry=registry, + policy_engine=PolicyEngine(), + ) + + +def _spec() -> StructuredTaskSpec: + return StructuredTaskSpec( + task_id="pre_recon_inventory", + system_prompt="Inventory the target.", + objective="Return the inventory summary.", + allowed_tools=["echo"], + output_contract=StructuredOutputContract( + name="pre_recon_inventory", + schema={ + "type": "object", + "required": ["summary"], + "additionalProperties": False, + "properties": {"summary": {"type": "string"}}, + }, + ), + ) + + +def test_native_and_langgraph_structured_kernels_have_the_same_result_and_checkpoints( + tmp_path: Path, +) -> None: + outcomes: dict[str, dict] = {} + checkpoint_phases: dict[str, list[str]] = {} + + for backend in ("native", "langgraph"): + runner = _runner(tmp_path / backend, backend) + phases: list[str] = [] + result = runner.run( + spec=_spec(), + context=execution_context(runner.settings, namespace_id="assessment"), + on_checkpoint=lambda checkpoint, phases=phases: phases.append(checkpoint.phase), + ) + outcomes[backend] = result.to_dict() + for history_item in outcomes[backend]["tool_history"]: + history_item.pop("artifact_id", None) + checkpoint_phases[backend] = phases + assert runner._kernel.backend == backend + + assert outcomes["langgraph"] == outcomes["native"] + assert checkpoint_phases["langgraph"] == checkpoint_phases["native"] + assert outcomes["langgraph"]["output"] == {"summary": "echo:inventory"} + assert outcomes["langgraph"]["tool_calls_used"] == 1 + + +def test_unknown_backend_is_rejected_for_structured_tasks(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="Expected 'native' or 'langgraph'"): + _runner(tmp_path, "other") diff --git a/tests/test_live_azure_openai_provider.py b/tests/test_live_azure_openai_provider.py new file mode 100644 index 0000000..6ba002c --- /dev/null +++ b/tests/test_live_azure_openai_provider.py @@ -0,0 +1,961 @@ +from __future__ import annotations + +import json +import os +import time +from dataclasses import dataclass, field +from typing import Any + +import pytest + +from agent_core.execution_context import ExecutionContext +from agent_core.llm.base import LLMCallOptions, LLMCompletionResult, LLMMessage, LLMToolDefinition +from agent_core.llm.provider_factory import LLMProviderConfig, build_provider_from_config +from agent_core.orchestrator import AgentOrchestrator +from agent_core.output_contracts import StructuredOutputContract +from agent_core.policy_engine import PolicyEngine +from agent_core.run_context import RunContext +from agent_core.run_options import RunOptions +from agent_core.session_manager import SessionManager +from agent_core.session_repo import SessionRepository +from agent_core.settings import CoreSettings +from agent_core.structured_tasks import StructuredTaskRunner, StructuredTaskSpec +from agent_core.tool_registry import ToolRegistry +from agent_core.tools import build_tool_definition +from agent_core.types import ToolResult +from tests.run_helpers import resume_turn, run_turn, turn_memory_payload + +pytestmark = pytest.mark.live_llm + + +@dataclass(frozen=True, slots=True) +class LiveAzureConfig: + endpoint: str + api_key: str = field(repr=False) + api_version: str + model: str + enabled_backends: frozenset[str] + + +class DeterministicMemoryProvider: + """Keep live kernel checks focused on the agent model and graph flow.""" + + def complete_text(self, *, messages, model, temperature, options=None): + target = (options.metadata or {}).get("target") if options is not None else None + if target == "investigation_step_reflection": + return json.dumps( + { + "observation_summary": "The live tool returned the requested marker.", + "new_facts": ["live-investigation-result"], + "updated_hypotheses": [], + "rejected_hypotheses": [], + "remaining_gaps": [], + "resolved_gaps": [], + "recommended_next_actions": [], + "risk_notes": [], + "confidence": 1.0, + "should_continue": False, + "stop_reason": "live integration evidence collected", + } + ) + if target == "investigation_decision": + return json.dumps( + { + "kind": "final", + "reason_summary": "The live integration evidence is sufficient.", + "next_action": None, + "question": None, + "required_approval": False, + } + ) + return json.dumps(turn_memory_payload(objective="Live LangGraph kernel validation")) + + def complete_with_tools(self, *, messages, tools, model, temperature, options=None): + raise AssertionError("The deterministic memory provider must not run the agent tool loop") + + +class RecordingProvider: + """Record real provider latency and usage without changing its contract.""" + + def __init__(self, delegate: Any) -> None: + self.delegate = delegate + self.calls: list[dict[str, Any]] = [] + + def complete_text(self, *, messages, model, temperature, options=None): + return self._record( + method="complete_text", + target=(options.metadata or {}).get("target") if options is not None else None, + call=lambda: self.delegate.complete_text( + messages=messages, + model=model, + temperature=temperature, + options=options, + ), + ) + + def complete_with_tools(self, *, messages, tools, model, temperature, options=None): + return self._record( + method="complete_with_tools", + target=(options.metadata or {}).get("target") if options is not None else None, + call=lambda: self.delegate.complete_with_tools( + messages=messages, + tools=tools, + model=model, + temperature=temperature, + options=options, + ), + ) + + def _record(self, *, method: str, target: str | None, call: Any) -> LLMCompletionResult: + started_at = time.perf_counter() + result = call() + wall_seconds = time.perf_counter() - started_at + usage = result.usage + self.calls.append( + { + "method": method, + "target": target, + "wall_seconds": wall_seconds, + "provider_seconds": result.duration_seconds, + "input_tokens": usage.input_tokens if usage is not None else 0, + "output_tokens": usage.output_tokens if usage is not None else 0, + "reasoning_tokens": usage.reasoning_output_tokens if usage is not None else 0, + "total_tokens": usage.total_tokens if usage is not None else 0, + "tool_call_count": len(result.tool_calls), + "provider_attempts": result.provider_attempts, + } + ) + return result + + +@dataclass(frozen=True, slots=True) +class KernelEvalObservation: + scenario: str + backend: str + status: str + wall_seconds: float + provider_seconds: float + llm_calls: int + input_tokens: int + output_tokens: int + reasoning_tokens: int + total_tokens: int + call_targets: tuple[str, ...] + tool_names: tuple[str, ...] + tool_statuses: tuple[str, ...] + context_block_kinds: tuple[str, ...] + trace_event_types: tuple[str, ...] + stop_reason: str | None + iterations_used: int | None + + def to_dict(self) -> dict[str, Any]: + return { + "scenario": self.scenario, + "backend": self.backend, + "status": self.status, + "wall_seconds": round(self.wall_seconds, 4), + "provider_seconds": round(self.provider_seconds, 4), + "llm_calls": self.llm_calls, + "input_tokens": self.input_tokens, + "output_tokens": self.output_tokens, + "reasoning_tokens": self.reasoning_tokens, + "total_tokens": self.total_tokens, + "call_targets": list(self.call_targets), + "tool_names": list(self.tool_names), + "tool_statuses": list(self.tool_statuses), + "context_block_kinds": list(self.context_block_kinds), + "trace_event_types": list(self.trace_event_types), + "stop_reason": self.stop_reason, + "iterations_used": self.iterations_used, + } + + +class LiveEchoTool: + name = "live_echo" + description = "Return the supplied text unchanged. Always use this when the user asks for live_echo." + + def schema(self): + return build_tool_definition( + name=self.name, + description=self.description, + parameters={ + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], + "additionalProperties": False, + }, + ) + + def execute(self, arguments, context): + return ToolResult(ok=True, content=str(arguments["text"])) + + +class LivePendingTool(LiveEchoTool): + name = "live_pending" + description = "Start external work and wait for its result. Always use this when the user asks for live_pending." + + def execute(self, arguments, context): + return ToolResult.pending_result("live-pending-wait", metadata={"requested_text": arguments["text"]}) + + +class LiveReverseTool(LiveEchoTool): + name = "live_reverse" + description = "Return the supplied text reversed. Use only when the user explicitly asks for live_reverse." + + def execute(self, arguments, context): + return ToolResult(ok=True, content=str(arguments["text"])[::-1]) + + +@pytest.fixture(scope="session") +def live_azure_config() -> LiveAzureConfig: + if not _env_flag("AGENT_CORE_RUN_LIVE_LLM_TESTS"): + pytest.skip("Set AGENT_CORE_RUN_LIVE_LLM_TESTS=1 to run paid Azure OpenAI integration tests") + + endpoint = os.getenv("AZURE_OPENAI_ENDPOINT", "").strip() + api_key = os.getenv("AZURE_OPENAI_API_KEY", "").strip() + if not endpoint or not api_key: + pytest.fail("AZURE_OPENAI_ENDPOINT and AZURE_OPENAI_API_KEY are required for live_llm tests") + + raw_backends = os.getenv("AGENT_CORE_LIVE_LLM_BACKENDS", "native,langchain") + enabled_backends = frozenset(value.strip() for value in raw_backends.split(",") if value.strip()) + if not enabled_backends: + pytest.fail("AGENT_CORE_LIVE_LLM_BACKENDS must select native, langchain, or both") + unsupported = enabled_backends - {"native", "langchain"} + if unsupported: + pytest.fail(f"Unsupported AGENT_CORE_LIVE_LLM_BACKENDS values: {sorted(unsupported)}") + + return LiveAzureConfig( + endpoint=endpoint, + api_key=api_key, + api_version=os.getenv("AZURE_OPENAI_API_VERSION", "2025-01-01-preview").strip(), + model=os.getenv("AGENT_CORE_LIVE_LLM_MODEL", "gpt-5.4-mini").strip(), + enabled_backends=enabled_backends, + ) + + +@pytest.fixture(params=["native", "langchain"]) +def live_provider(request, live_azure_config: LiveAzureConfig): + backend = str(request.param) + if backend not in live_azure_config.enabled_backends: + pytest.skip(f"Backend {backend} not selected by AGENT_CORE_LIVE_LLM_BACKENDS") + provider = build_provider_from_config( + LLMProviderConfig( + provider="azure_openai", + model_backend=backend, + azure_openai_endpoint=live_azure_config.endpoint, + azure_openai_api_key=live_azure_config.api_key, + azure_openai_api_version=live_azure_config.api_version, + timeout_seconds=120.0, + langchain_tracing_enabled=False, + ) + ) + return backend, provider + + +def test_live_text_reasoning_and_usage(live_provider, live_azure_config: LiveAzureConfig) -> None: + backend, provider = live_provider + + result = provider.complete_text( + messages=[ + LLMMessage(role="system", content="Solve carefully and return only the integer."), + LLMMessage(role="user", content="What is 17 multiplied by 19?"), + ], + model=live_azure_config.model, + temperature=0.0, + options=LLMCallOptions(reasoning_effort="low", max_output_tokens=256), + ) + + assert result.content.strip() == "323" + assert result.provider == "azure_openai" + assert result.model_backend == backend + assert result.provider_request_id + assert result.provider_attempts >= 1 + assert result.usage is not None + assert result.usage.input_tokens > 0 + assert result.usage.output_tokens > 0 + + +def test_live_strict_json_schema(live_provider, live_azure_config: LiveAzureConfig) -> None: + backend, provider = live_provider + schema = { + "type": "object", + "properties": { + "ok": {"type": "boolean"}, + "backend_contract": {"type": "string", "enum": ["stable"]}, + }, + "required": ["ok", "backend_contract"], + "additionalProperties": False, + } + + result = provider.complete_text( + messages=[ + LLMMessage( + role="user", + content='Return the object with "ok" true and "backend_contract" set to "stable".', + ) + ], + model=live_azure_config.model, + temperature=0.0, + options=LLMCallOptions( + max_output_tokens=256, + response_format={ + "type": "json_schema", + "json_schema": {"name": "live_contract", "strict": True, "schema": schema}, + }, + ), + ) + + assert json.loads(result.content) == {"ok": True, "backend_contract": "stable"} + assert result.model_backend == backend + + +def test_live_tool_call_and_result_roundtrip(live_provider, live_azure_config: LiveAzureConfig) -> None: + backend, provider = live_provider + user_message = LLMMessage( + role="user", + content=( + "Call echo exactly once with text live-tool-result. " + "After receiving its result, answer exactly with the returned text." + ), + ) + tool = LLMToolDefinition( + name="echo", + description="Return the provided text unchanged.", + parameters={ + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], + "additionalProperties": False, + }, + ) + + first = provider.complete_with_tools( + messages=[user_message], + tools=[tool], + model=live_azure_config.model, + temperature=0.0, + options=LLMCallOptions(max_output_tokens=256), + ) + + assert len(first.tool_calls) == 1 + tool_call = first.tool_calls[0] + assert tool_call.name == "echo" + assert json.loads(tool_call.arguments_json) == {"text": "live-tool-result"} + + final = provider.complete_with_tools( + messages=[ + user_message, + LLMMessage(role="assistant", content=first.content, tool_calls=first.tool_calls), + LLMMessage(role="tool", content="live-tool-result", tool_call_id=tool_call.id), + ], + tools=[tool], + model=live_azure_config.model, + temperature=0.0, + options=LLMCallOptions(max_output_tokens=256), + ) + + assert final.tool_calls == [] + assert final.content.strip().strip("\"'").rstrip(".") == "live-tool-result" + assert first.model_backend == final.model_backend == backend + + +def test_live_structured_task_runner(live_provider, live_azure_config: LiveAzureConfig, tmp_path) -> None: + backend, provider = live_provider + settings = CoreSettings( + llm_provider="azure_openai", + llm_model_backend=backend, + agent_kernel_backend="langgraph", + azure_openai_endpoint=live_azure_config.endpoint, + azure_openai_api_key=live_azure_config.api_key, + azure_openai_api_version=live_azure_config.api_version, + model=live_azure_config.model, + memory_model=live_azure_config.model, + llm_max_output_tokens=256, + session_file=tmp_path / "session.json", + base_system_prompt="live test", + turn_memory_synthesis_prompt="live test", + ) + runner = StructuredTaskRunner( + settings=settings, + provider=provider, + tool_registry=ToolRegistry(), + policy_engine=PolicyEngine(), + ) + + result = runner.run( + spec=StructuredTaskSpec( + task_id=f"live-structured-{backend}", + system_prompt="Return the requested contract without prose.", + objective='Return {"ok": true, "component": "structured_task"}.', + output_contract=StructuredOutputContract( + name="live_structured_task", + strict=True, + schema={ + "type": "object", + "properties": { + "ok": {"type": "boolean"}, + "component": {"type": "string", "enum": ["structured_task"]}, + }, + "required": ["ok", "component"], + "additionalProperties": False, + }, + ), + allowed_tools=[], + max_iterations=1, + ), + context=ExecutionContext.from_run_context( + context=RunContext(namespace_id="live-tests", run_id=f"live-{backend}"), + settings=settings, + ), + ) + + assert result.ok + assert result.output == {"ok": True, "component": "structured_task"} + assert result.llm_calls + assert all(call.model_backend == backend for call in result.llm_calls) + assert all(call.model == live_azure_config.model for call in result.llm_calls) + assert runner._kernel.backend == "langgraph" + + +def test_live_langgraph_structured_task_tool_loop( + live_provider, + live_azure_config: LiveAzureConfig, + tmp_path, +) -> None: + backend, provider = live_provider + settings = CoreSettings( + llm_provider="azure_openai", + llm_model_backend=backend, + agent_kernel_backend="langgraph", + azure_openai_endpoint=live_azure_config.endpoint, + azure_openai_api_key=live_azure_config.api_key, + azure_openai_api_version=live_azure_config.api_version, + model=live_azure_config.model, + memory_model=live_azure_config.model, + llm_max_output_tokens=512, + session_file=tmp_path / "session.json", + base_system_prompt="live test", + turn_memory_synthesis_prompt="live test", + ) + registry = ToolRegistry() + registry.register(LiveEchoTool()) + runner = StructuredTaskRunner( + settings=settings, + provider=provider, + tool_registry=registry, + policy_engine=PolicyEngine(), + ) + checkpoint_phases: list[str] = [] + + result = runner.run( + spec=StructuredTaskSpec( + task_id=f"live-langgraph-tool-{backend}", + system_prompt="Follow the requested tool workflow and return the strict contract without prose.", + objective=( + "Call live_echo exactly once with text structured-langgraph-live. " + 'After receiving the tool result, return {"ok": true, "marker": "structured-langgraph-live"}.' + ), + constraints=["The live_echo tool call is mandatory."], + allowed_tools=["live_echo"], + output_contract=StructuredOutputContract( + name="live_langgraph_structured_tool", + strict=True, + schema={ + "type": "object", + "properties": { + "ok": {"type": "boolean"}, + "marker": { + "type": "string", + "enum": ["structured-langgraph-live"], + }, + }, + "required": ["ok", "marker"], + "additionalProperties": False, + }, + ), + max_tool_calls=1, + max_iterations=3, + ), + context=ExecutionContext.from_run_context( + context=RunContext(namespace_id="live-tests", run_id=f"live-langgraph-tool-{backend}"), + settings=settings, + ), + on_checkpoint=lambda checkpoint: checkpoint_phases.append(checkpoint.phase), + ) + + assert result.ok + assert result.output == {"ok": True, "marker": "structured-langgraph-live"} + assert result.tool_calls_used == 1 + assert [item["tool_name"] for item in result.tool_history] == ["live_echo"] + assert {"tools", "finalization", "result"}.issubset(checkpoint_phases) + assert all(call.model_backend == backend for call in result.llm_calls) + assert all(call.model == live_azure_config.model for call in result.llm_calls) + assert runner._kernel.backend == "langgraph" + + +def _build_live_conversation_orchestrator( + *, + live_azure_config: LiveAzureConfig, + tmp_path, + agent_kernel_backend: str, + tool: LiveEchoTool, + extra_tools: tuple[LiveEchoTool, ...] = (), + real_internal_synthesis: bool = False, +) -> AgentOrchestrator: + provider = RecordingProvider( + build_provider_from_config( + LLMProviderConfig( + provider="azure_openai", + model_backend="langchain", + azure_openai_endpoint=live_azure_config.endpoint, + azure_openai_api_key=live_azure_config.api_key, + azure_openai_api_version=live_azure_config.api_version, + timeout_seconds=120.0, + langchain_tracing_enabled=False, + ) + ) + ) + settings = CoreSettings( + llm_provider="azure_openai", + llm_model_backend="langchain", + agent_kernel_backend=agent_kernel_backend, + azure_openai_endpoint=live_azure_config.endpoint, + azure_openai_api_key=live_azure_config.api_key, + azure_openai_api_version=live_azure_config.api_version, + model=live_azure_config.model, + memory_model=live_azure_config.model, + llm_max_output_tokens=512, + session_file=tmp_path / "session.json", + base_system_prompt=( + "You are a deterministic integration-test assistant. Follow the user's explicit tool instruction exactly, " + "then return the exact requested marker without commentary." + ), + turn_memory_synthesis_prompt="unused by the deterministic memory provider", + ) + registry = ToolRegistry() + registry.register(tool) + for extra_tool in extra_tools: + registry.register(extra_tool) + return AgentOrchestrator( + settings=settings, + provider=provider, + memory_provider=provider if real_internal_synthesis else DeterministicMemoryProvider(), + registry=registry, + session_manager=SessionManager(SessionRepository(settings.session_file)), + policy_engine=PolicyEngine(), + ) + + +def _observe_live_kernel_run( + *, + scenario: str, + backend: str, + orchestrator: AgentOrchestrator, + result, + wall_seconds: float, + trace_id: str | None = None, +) -> KernelEvalObservation: + provider = orchestrator.provider + assert isinstance(provider, RecordingProvider) + resolved_trace_id = trace_id or str(result.metadata["run_trace_id"]) + trace = orchestrator.session_manager.load_run_trace(resolved_trace_id) + assert trace is not None + calls = provider.calls + return KernelEvalObservation( + scenario=scenario, + backend=backend, + status=result.status, + wall_seconds=wall_seconds, + provider_seconds=sum(float(call["provider_seconds"] or 0.0) for call in calls), + llm_calls=len(calls), + input_tokens=sum(int(call["input_tokens"]) for call in calls), + output_tokens=sum(int(call["output_tokens"]) for call in calls), + reasoning_tokens=sum(int(call["reasoning_tokens"] or 0) for call in calls), + total_tokens=sum(int(call["total_tokens"]) for call in calls), + call_targets=tuple(str(call["target"] or call["method"]) for call in calls), + tool_names=tuple( + str(item["tool_name"]) for item in orchestrator.session_manager.get_state()["tool_history"] + ), + tool_statuses=tuple( + str(item["status"]) for item in orchestrator.session_manager.get_state()["tool_history"] + ), + context_block_kinds=tuple( + block.kind for block in orchestrator.session_manager.get_context_blocks() + ), + trace_event_types=tuple(str(event["type"]) for event in trace["events"]), + stop_reason=( + str(result.metadata["stop_reason"]) + if result.metadata.get("stop_reason") is not None + else None + ), + iterations_used=( + int(result.metadata["iterations_used"]) + if isinstance(result.metadata.get("iterations_used"), int) + else None + ), + ) + + +def _percent_delta(current: float, baseline: float) -> float | None: + if baseline == 0: + return None + return round(((current - baseline) / baseline) * 100, 2) + + +def _report_kernel_pair(native: KernelEvalObservation, langgraph: KernelEvalObservation) -> None: + assert native.scenario == langgraph.scenario + assert native.status == langgraph.status + assert native.call_targets == langgraph.call_targets + assert native.tool_names == langgraph.tool_names + assert native.tool_statuses == langgraph.tool_statuses + assert native.context_block_kinds == langgraph.context_block_kinds + assert native.trace_event_types == langgraph.trace_event_types + report = { + "scenario": native.scenario, + "native": native.to_dict(), + "langgraph": langgraph.to_dict(), + "langgraph_delta_percent": { + "wall_seconds": _percent_delta(langgraph.wall_seconds, native.wall_seconds), + "provider_seconds": _percent_delta(langgraph.provider_seconds, native.provider_seconds), + "llm_calls": _percent_delta(float(langgraph.llm_calls), float(native.llm_calls)), + "input_tokens": _percent_delta(float(langgraph.input_tokens), float(native.input_tokens)), + "output_tokens": _percent_delta(float(langgraph.output_tokens), float(native.output_tokens)), + "total_tokens": _percent_delta(float(langgraph.total_tokens), float(native.total_tokens)), + }, + } + print(f"LIVE_KERNEL_EVAL {json.dumps(report, sort_keys=True)}") + + +def test_live_conversation_tool_loop_kernel_parity( + live_azure_config: LiveAzureConfig, + tmp_path, +) -> None: + if "langchain" not in live_azure_config.enabled_backends: + pytest.skip("The live agent-kernel matrix requires the LangChain model backend") + observations: dict[str, KernelEvalObservation] = {} + for backend in ("native", "langgraph"): + orchestrator = _build_live_conversation_orchestrator( + live_azure_config=live_azure_config, + tmp_path=tmp_path / backend, + agent_kernel_backend=backend, + tool=LiveEchoTool(), + extra_tools=(LiveReverseTool(),), + ) + + started_at = time.perf_counter() + result = run_turn( + orchestrator, + "Call live_echo exactly once with text live-kernel-result. Then answer exactly: live-kernel-result", + ) + elapsed = time.perf_counter() - started_at + + assert result.status == "completed" + assert result.content.strip().strip("\"'").rstrip(".") == "live-kernel-result" + trace_id = str(result.metadata["run_trace_id"]) + trace = orchestrator.session_manager.load_run_trace(trace_id) + assert trace is not None + assert trace["options"]["agent_kernel_backend"] == backend + assistant_events = [ + event for event in trace["events"] if event["type"] == "assistant_response_received" + ] + assert len(assistant_events) == 2 + assert all(event["payload"]["model_backend"] == "langchain" for event in assistant_events) + observations[backend] = _observe_live_kernel_run( + scenario="direct_tool_roundtrip", + backend=backend, + orchestrator=orchestrator, + result=result, + wall_seconds=elapsed, + ) + + native = observations["native"] + langgraph = observations["langgraph"] + assert native.llm_calls == langgraph.llm_calls == 2 + assert native.tool_names == langgraph.tool_names == ("live_echo",) + assert native.tool_statuses == langgraph.tool_statuses == ("ok",) + assert native.context_block_kinds == langgraph.context_block_kinds == ( + "tool_exchange", + "conversation_turn", + ) + _report_kernel_pair(native, langgraph) + + +def test_live_pending_resume_kernel_parity(live_azure_config: LiveAzureConfig, tmp_path) -> None: + if "langchain" not in live_azure_config.enabled_backends: + pytest.skip("The live agent-kernel matrix requires the LangChain model backend") + observations: dict[str, KernelEvalObservation] = {} + for backend in ("native", "langgraph"): + orchestrator = _build_live_conversation_orchestrator( + live_azure_config=live_azure_config, + tmp_path=tmp_path / backend, + agent_kernel_backend=backend, + tool=LivePendingTool(), + extra_tools=(LiveReverseTool(),), + ) + + started_at = time.perf_counter() + pending = run_turn( + orchestrator, + "Call live_pending exactly once with text live-resume-result. Wait for its result before answering.", + ) + + assert pending.status == "pending_tool_result" + assert pending.pending_id + assert pending.tool_name == "live_pending" + payload = orchestrator.session_manager.get_state()["meta"][AgentOrchestrator.PENDING_TURN_META_KEY] + assert payload["agent_graph_checkpoint"] == { + "schema_version": "1", + "graph": "direct", + "backend": backend, + "resume_node": "resume_tool_exchange", + } + trace_id = str(payload["run_trace_id"]) + + completed = resume_turn( + orchestrator, + pending_id=pending.pending_id, + tool_content="live-resume-result", + ) + elapsed = time.perf_counter() - started_at + + assert completed.status == "completed" + assert "live-resume-result" in completed.content + observations[backend] = _observe_live_kernel_run( + scenario="direct_pending_resume", + backend=backend, + orchestrator=orchestrator, + result=completed, + wall_seconds=elapsed, + trace_id=trace_id, + ) + + native = observations["native"] + langgraph = observations["langgraph"] + assert native.llm_calls == langgraph.llm_calls == 2 + assert native.tool_names == langgraph.tool_names == ("live_pending", "live_pending") + assert native.tool_statuses == langgraph.tool_statuses == ("pending", "ok") + assert native.context_block_kinds == langgraph.context_block_kinds == ( + "tool_exchange", + "conversation_turn", + ) + assert "agent_graph_checkpoint_restored" in native.trace_event_types + assert "agent_graph_checkpoint_restored" in langgraph.trace_event_types + _report_kernel_pair(native, langgraph) + + +def test_live_investigation_tool_flow_kernel_parity(live_azure_config: LiveAzureConfig, tmp_path) -> None: + if "langchain" not in live_azure_config.enabled_backends: + pytest.skip("The live agent-kernel matrix requires the LangChain model backend") + observations: dict[str, KernelEvalObservation] = {} + for backend in ("native", "langgraph"): + orchestrator = _build_live_conversation_orchestrator( + live_azure_config=live_azure_config, + tmp_path=tmp_path / backend, + agent_kernel_backend=backend, + tool=LiveEchoTool(), + extra_tools=(LiveReverseTool(),), + ) + + started_at = time.perf_counter() + result = run_turn( + orchestrator, + ( + "Investigate by calling live_echo exactly once with text live-investigation-result. " + "Use the returned evidence in the final answer." + ), + options=RunOptions.investigate(max_iterations=2, require_initial_plan=False), + ) + elapsed = time.perf_counter() - started_at + + assert result.status == "completed" + assert "live-investigation-result" in result.content + assert result.metadata["mode"] == "investigate" + assert result.metadata["investigation_state"]["facts"] == ["live-investigation-result"] + trace = orchestrator.session_manager.load_run_trace(str(result.metadata["run_trace_id"])) + assert trace is not None + assert trace["options"]["agent_kernel_backend"] == backend + assert "decision_completed" in [event["type"] for event in trace["events"]] + observations[backend] = _observe_live_kernel_run( + scenario="investigation_tool_reflect_decide", + backend=backend, + orchestrator=orchestrator, + result=result, + wall_seconds=elapsed, + ) + + native = observations["native"] + langgraph = observations["langgraph"] + assert native.llm_calls == langgraph.llm_calls == 2 + assert native.tool_names == langgraph.tool_names == ("live_echo",) + assert native.tool_statuses == langgraph.tool_statuses == ("ok",) + assert native.stop_reason == langgraph.stop_reason + assert native.iterations_used == langgraph.iterations_used == 1 + _report_kernel_pair(native, langgraph) + + +def test_live_full_real_investigation_kernel_parity(live_azure_config: LiveAzureConfig, tmp_path) -> None: + """Exercise plan, tool use, reflection, decision, and finalization with the real LLM.""" + + if "langchain" not in live_azure_config.enabled_backends: + pytest.skip("The live agent-kernel matrix requires the LangChain model backend") + observations: dict[str, KernelEvalObservation] = {} + for backend in ("native", "langgraph"): + orchestrator = _build_live_conversation_orchestrator( + live_azure_config=live_azure_config, + tmp_path=tmp_path / backend, + agent_kernel_backend=backend, + tool=LiveEchoTool(), + extra_tools=(LiveReverseTool(),), + real_internal_synthesis=True, + ) + + started_at = time.perf_counter() + result = run_turn( + orchestrator, + ( + "Investigate the marker by calling live_echo exactly once with text full-real-evidence. " + "The returned marker is sufficient evidence; finish after observing it and include it verbatim." + ), + options=RunOptions.investigate(max_iterations=2, require_initial_plan=True), + ) + elapsed = time.perf_counter() - started_at + + assert result.status == "completed" + assert "full-real-evidence" in result.content + assert result.metadata["mode"] == "investigate" + trace = orchestrator.session_manager.load_run_trace(str(result.metadata["run_trace_id"])) + assert trace is not None + event_types = [event["type"] for event in trace["events"]] + assert { + "initial_plan_created", + "tool_step_completed", + "reflection_completed", + "decision_completed", + }.issubset(event_types) + observations[backend] = _observe_live_kernel_run( + scenario="full_real_investigation", + backend=backend, + orchestrator=orchestrator, + result=result, + wall_seconds=elapsed, + ) + + native = observations["native"] + langgraph = observations["langgraph"] + assert native.status == langgraph.status == "completed" + assert native.tool_names == langgraph.tool_names == ("live_echo",) + assert native.tool_statuses == langgraph.tool_statuses == ("ok",) + assert native.context_block_kinds == langgraph.context_block_kinds + _report_kernel_pair(native, langgraph) + + +def test_live_structured_investigation_output_kernel_parity( + live_azure_config: LiveAzureConfig, + tmp_path, +) -> None: + if "langchain" not in live_azure_config.enabled_backends: + pytest.skip("The live agent-kernel matrix requires the LangChain model backend") + contract = StructuredOutputContract( + name="live_kernel_structured_output", + strict=True, + schema={ + "type": "object", + "properties": { + "ok": {"type": "boolean"}, + "marker": {"type": "string", "enum": ["structured-kernel-result"]}, + }, + "required": ["ok", "marker"], + "additionalProperties": False, + }, + ) + observations: dict[str, KernelEvalObservation] = {} + for backend in ("native", "langgraph"): + orchestrator = _build_live_conversation_orchestrator( + live_azure_config=live_azure_config, + tmp_path=tmp_path / backend, + agent_kernel_backend=backend, + tool=LiveEchoTool(), + extra_tools=(LiveReverseTool(),), + ) + + started_at = time.perf_counter() + result = run_turn( + orchestrator, + 'Return an answer that establishes ok=true and the marker "structured-kernel-result".', + options=RunOptions.investigate( + max_iterations=1, + require_initial_plan=False, + final_output_mode="json_schema", + final_output_contract=contract, + ), + ) + elapsed = time.perf_counter() - started_at + + assert result.status == "completed" + assert json.loads(result.content) == {"ok": True, "marker": "structured-kernel-result"} + assert result.metadata["final_output_mode"] == "json_schema" + observations[backend] = _observe_live_kernel_run( + scenario="investigation_structured_output", + backend=backend, + orchestrator=orchestrator, + result=result, + wall_seconds=elapsed, + ) + + native = observations["native"] + langgraph = observations["langgraph"] + assert native.llm_calls == langgraph.llm_calls == 2 + assert native.tool_names == langgraph.tool_names == () + assert native.tool_statuses == langgraph.tool_statuses == () + assert native.stop_reason == langgraph.stop_reason + _report_kernel_pair(native, langgraph) + + +def test_live_full_real_deep_critique_kernel_parity(live_azure_config: LiveAzureConfig, tmp_path) -> None: + if "langchain" not in live_azure_config.enabled_backends: + pytest.skip("The live agent-kernel matrix requires the LangChain model backend") + observations: dict[str, KernelEvalObservation] = {} + for backend in ("native", "langgraph"): + orchestrator = _build_live_conversation_orchestrator( + live_azure_config=live_azure_config, + tmp_path=tmp_path / backend, + agent_kernel_backend=backend, + tool=LiveEchoTool(), + extra_tools=(LiveReverseTool(),), + real_internal_synthesis=True, + ) + + started_at = time.perf_counter() + result = run_turn( + orchestrator, + "Answer concisely with the exact factual marker deep-critique-result and no unsupported claims.", + options=RunOptions.deep_investigate(max_iterations=1, require_initial_plan=False), + ) + elapsed = time.perf_counter() - started_at + + assert result.status == "completed" + assert "deep-critique-result" in result.content + assert result.metadata["mode"] == "deep_investigate" + trace = orchestrator.session_manager.load_run_trace(str(result.metadata["run_trace_id"])) + assert trace is not None + event_types = [event["type"] for event in trace["events"]] + assert "final_critique_completed" in event_types + observations[backend] = _observe_live_kernel_run( + scenario="full_real_deep_critique", + backend=backend, + orchestrator=orchestrator, + result=result, + wall_seconds=elapsed, + ) + + native = observations["native"] + langgraph = observations["langgraph"] + assert native.status == langgraph.status == "completed" + assert native.tool_names == langgraph.tool_names == () + assert native.tool_statuses == langgraph.tool_statuses == () + assert native.context_block_kinds == langgraph.context_block_kinds + _report_kernel_pair(native, langgraph) + + +def _env_flag(name: str) -> bool: + return os.getenv(name, "").strip().lower() in {"1", "true", "yes", "on"} diff --git a/tests/test_llm_usage.py b/tests/test_llm_usage.py index b12ea4b..d7c4384 100644 --- a/tests/test_llm_usage.py +++ b/tests/test_llm_usage.py @@ -94,6 +94,7 @@ def test_completion_capture_records_conversation_provider_calls() -> None: content="done", usage=LLMTokenUsage(input_tokens=7, output_tokens=3, total_tokens=10), provider="azure_openai", + model_backend="langchain", model="deployment", provider_request_id="request-1", ) @@ -106,3 +107,20 @@ def test_completion_capture_records_conversation_provider_calls() -> None: assert calls[0].usage is not None assert calls[0].usage.total_tokens == 10 assert calls[0].provider_request_id == "request-1" + assert calls[0].model_backend == "langchain" + assert LLMCallRecord.from_dict(calls[0].to_dict()) == calls[0] + + +def test_call_record_loads_legacy_payload_without_model_backend() -> None: + record = LLMCallRecord.from_dict( + { + "call_id": "llm-0001", + "call_index": 1, + "purpose": "tool_loop", + "provider": "azure_openai", + "model": "deployment", + } + ) + + assert record is not None + assert record.model_backend is None diff --git a/tests/test_provider_factory.py b/tests/test_provider_factory.py index f5d0bee..c9a0390 100644 --- a/tests/test_provider_factory.py +++ b/tests/test_provider_factory.py @@ -1,8 +1,14 @@ from __future__ import annotations from agent_core.llm.azure_openai_provider import AzureOpenAIProvider +from agent_core.llm.langchain_azure_openai_provider import LangChainAzureOpenAIProvider from agent_core.llm.openai_provider import OpenAIProvider -from agent_core.llm.provider_factory import build_memory_provider, build_provider +from agent_core.llm.provider_factory import ( + LLMProviderConfig, + build_memory_provider, + build_provider, + build_provider_from_config, +) from agent_core.settings import CoreSettings @@ -44,3 +50,85 @@ def test_build_memory_provider_can_override_azure_openai_endpoint_and_inherit_ke assert memory_provider.azure_endpoint == "https://memory.openai.azure.com" assert memory_provider.api_key == "shared-key" assert memory_provider.api_version == "2025-01-01-preview" + + +def test_build_provider_selects_langchain_azure_openai_backend() -> None: + settings = CoreSettings( + llm_provider="azure_openai", + llm_model_backend="langchain", + azure_openai_endpoint="https://primary.openai.azure.com", + azure_openai_api_key="shared-key", + ) + + provider = build_provider(settings) + + assert isinstance(provider, LangChainAzureOpenAIProvider) + assert provider.azure_endpoint == "https://primary.openai.azure.com" + assert provider.api_key == "shared-key" + assert provider.tracing_enabled is False + + +def test_build_provider_explicitly_enables_langchain_tracing() -> None: + settings = CoreSettings( + llm_provider="azure_openai", + llm_model_backend="langchain", + azure_openai_endpoint="https://primary.openai.azure.com", + azure_openai_api_key="shared-key", + langchain_tracing_enabled=True, + ) + + provider = build_provider(settings) + + assert isinstance(provider, LangChainAzureOpenAIProvider) + assert provider.tracing_enabled is True + + +def test_build_memory_provider_inherits_langchain_backend_from_primary() -> None: + settings = CoreSettings( + llm_provider="azure_openai", + llm_model_backend="langchain", + azure_openai_endpoint="https://primary.openai.azure.com", + azure_openai_api_key="shared-key", + memory_azure_openai_endpoint="https://memory.openai.azure.com", + ) + + provider = build_memory_provider(settings) + + assert isinstance(provider, LangChainAzureOpenAIProvider) + assert provider.azure_endpoint == "https://memory.openai.azure.com" + + +def test_build_memory_provider_can_override_langchain_backend_with_native() -> None: + settings = CoreSettings( + llm_provider="azure_openai", + llm_model_backend="langchain", + azure_openai_endpoint="https://primary.openai.azure.com", + azure_openai_api_key="shared-key", + memory_llm_model_backend="native", + ) + + provider = build_memory_provider(settings) + + assert isinstance(provider, AzureOpenAIProvider) + + +def test_build_provider_rejects_langchain_backend_for_unsupported_provider() -> None: + config = LLMProviderConfig(provider="openai", model_backend="langchain", openai_api_key="test-key") + + try: + build_provider_from_config(config) + except ValueError as exc: + assert "currently supports only provider=azure_openai" in str(exc) + else: + raise AssertionError("Expected unsupported backend/provider combination to fail") + + +def test_build_provider_rejects_unknown_model_backend() -> None: + config = LLMProviderConfig(provider="azure_openai", model_backend="unknown") + + try: + build_provider_from_config(config) + except ValueError as exc: + assert "Unsupported LLM model backend" in str(exc) + else: + raise AssertionError("Expected unknown model backend to fail") diff --git a/tests/test_public_api.py b/tests/test_public_api.py index 1c1f6f2..ff5b816 100644 --- a/tests/test_public_api.py +++ b/tests/test_public_api.py @@ -107,6 +107,7 @@ def test_extension_conversation_and_observability_facades_are_explicit() -> None "build_provider_from_config", "build_tool_definition", "load_prompt", + "normalize_model_backend", "normalize_provider_name", } assert set(conversation.__all__) == { diff --git a/tests/test_run_recovery.py b/tests/test_run_recovery.py index cd45141..17758f5 100644 --- a/tests/test_run_recovery.py +++ b/tests/test_run_recovery.py @@ -125,12 +125,21 @@ def complete_with_tools(self, **kwargs): return LLMCompletionResult(content=json.dumps({"summary": "validated final"})) -def _service(tmp_path, *, provider, tool: CountingTool | None = None): +def _service( + tmp_path, + *, + provider, + tool: CountingTool | None = None, + agent_kernel_backend: str = "native", +): registry = ToolRegistry() if tool is not None: registry.register(tool) return AgentRunService( - settings=CoreSettings(session_file=tmp_path / "session.json"), + settings=CoreSettings( + session_file=tmp_path / "session.json", + agent_kernel_backend=agent_kernel_backend, + ), provider=provider, tool_registry=registry, policy_engine=PolicyEngine(), @@ -184,7 +193,12 @@ def test_resume_continues_after_completed_tool_without_replaying_it(tmp_path) -> assert tool.calls == 1 final_provider = FinalProvider() - resumed = _service(tmp_path, provider=final_provider, tool=tool).resume( + resumed = _service( + tmp_path, + provider=final_provider, + tool=tool, + agent_kernel_backend="langgraph", + ).resume( spec=_spec(), context=context, run_id="run-1", diff --git a/tests/test_run_trace.py b/tests/test_run_trace.py index 3400075..961bbbe 100644 --- a/tests/test_run_trace.py +++ b/tests/test_run_trace.py @@ -92,6 +92,10 @@ def tool_call(*, value: str = "hello") -> LLMCompletionResult: arguments_json=json.dumps({"value": value}), ) ], + provider="azure_openai", + model_backend="langchain", + model="deployment", + provider_request_id="request-tool", ) @@ -215,6 +219,12 @@ def test_direct_run_persists_tool_audit_trace_and_exposes_run_id(tmp_path) -> No "tool_exchange_completed", "run_completed", }.issubset(event_types(trace_payload)) + events = trace_payload["events"] + assistant_event = next(event for event in events if event["type"] == "assistant_response_received") + assert assistant_event["payload"]["provider"] == "azure_openai" + assert assistant_event["payload"]["model_backend"] == "langchain" + assert assistant_event["payload"]["model"] == "deployment" + assert assistant_event["payload"]["provider_request_id"] == "request-tool" def test_investigation_run_persists_process_events_and_trace_id(tmp_path) -> None: