From c4ecfc1441e8ec65bac63759c8a82c161e1ffad8 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 1 Apr 2026 12:42:11 +0100 Subject: [PATCH 1/5] feat: add Claude Agent SDK test script Standalone script that tests posthog.ai.claude_agent_sdk integration. Supports single-shot and interactive modes. Requires local posthog-python with the claude_agent_sdk integration (PostHog/posthog-python#477). Usage: uv pip install -e ../posthog-python uv run --no-sync scripts/test_claude_agent_sdk.py uv run --no-sync scripts/test_claude_agent_sdk.py --interactive --- pyproject.toml | 1 + scripts/test_claude_agent_sdk.py | 174 +++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 scripts/test_claude_agent_sdk.py diff --git a/pyproject.toml b/pyproject.toml index 54df274..0ef6df3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "langchain-openai", "openai", "openai-agents", + "claude-agent-sdk", "litellm==1.81.13", "pydantic-ai", "opentelemetry-api", diff --git a/scripts/test_claude_agent_sdk.py b/scripts/test_claude_agent_sdk.py new file mode 100644 index 0000000..e168a8e --- /dev/null +++ b/scripts/test_claude_agent_sdk.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +""" +Claude Agent SDK test script for PostHog LLM Analytics. + +Tests the posthog.ai.claude_agent_sdk integration by running a multi-turn +conversation with tool calls. Sends $ai_generation, $ai_span, and $ai_trace +events to the configured PostHog instance. + +Usage: + uv run scripts/test_claude_agent_sdk.py + uv run scripts/test_claude_agent_sdk.py --prompt "What files are in the current directory?" + uv run scripts/test_claude_agent_sdk.py --interactive +""" + +import argparse +import asyncio +import os +import sys + +from dotenv import load_dotenv + +load_dotenv() + + +def _check_deps(): + missing = [] + try: + import claude_agent_sdk # noqa: F401 + except ImportError: + missing.append("claude-agent-sdk") + try: + import posthog # noqa: F401 + except ImportError: + missing.append("posthog") + try: + from posthog.ai.claude_agent_sdk import query # noqa: F401 + except ImportError: + missing.append("posthog (with claude_agent_sdk integration — needs posthog>=7.10.0)") + if missing: + print(f"Missing dependencies: {', '.join(missing)}") + print("Install with: uv add claude-agent-sdk posthog") + sys.exit(1) + + +def _setup_posthog(): + from posthog import Posthog + + api_key = os.environ.get("POSTHOG_API_KEY") + host = os.environ.get("POSTHOG_HOST", "https://us.i.posthog.com") + if not api_key: + print("POSTHOG_API_KEY not set in environment. Events won't be sent.") + return None + return Posthog(api_key, host=host) + + +async def run_query(prompt: str, posthog_client, distinct_id: str, extra_props: dict): + from claude_agent_sdk import ClaudeAgentOptions, AssistantMessage, ResultMessage + from claude_agent_sdk.types import TextBlock, ToolUseBlock, StreamEvent + from posthog.ai.claude_agent_sdk import query + + options = ClaudeAgentOptions( + max_turns=10, + allowed_tools=["Read", "Glob", "Grep", "Bash"], + permission_mode="bypassPermissions", + ) + + print(f"\n> {prompt}\n") + + async for message in query( + prompt=prompt, + options=options, + posthog_client=posthog_client, + posthog_distinct_id=distinct_id, + posthog_properties=extra_props, + ): + if isinstance(message, AssistantMessage): + for block in message.content: + if isinstance(block, TextBlock): + print(f" {block.text[:300]}") + elif isinstance(block, ToolUseBlock): + print(f" [tool] {block.name}({list(block.input.keys())})") + elif isinstance(message, StreamEvent): + event_type = message.event.get("type") + if event_type == "message_start": + print(" [streaming...]") + elif isinstance(message, ResultMessage): + print(f"\n --- Result ---") + print(f" Cost: ${message.total_cost_usd}") + print(f" Turns: {message.num_turns}") + print(f" Duration: {message.duration_ms}ms") + print(f" Error: {message.is_error}") + + +async def run_interactive(posthog_client, distinct_id: str): + from claude_agent_sdk import ClaudeAgentOptions, AssistantMessage, ResultMessage + from claude_agent_sdk.types import TextBlock, ToolUseBlock + from posthog.ai.claude_agent_sdk import instrument + + ph = instrument( + client=posthog_client, + distinct_id=distinct_id, + properties={"app": "llm-analytics-apps", "mode": "interactive"}, + ) + + options = ClaudeAgentOptions( + max_turns=10, + allowed_tools=["Read", "Glob", "Grep", "Bash"], + permission_mode="bypassPermissions", + ) + + print("\nClaude Agent SDK — Interactive Mode") + print("Type 'quit' to exit\n") + + while True: + try: + prompt = input("> ").strip() + if not prompt: + continue + if prompt.lower() in ("quit", "exit", "q"): + break + + async for message in ph.query(prompt=prompt, options=options): + if isinstance(message, AssistantMessage): + for block in message.content: + if isinstance(block, TextBlock): + print(f" {block.text[:500]}") + elif isinstance(block, ToolUseBlock): + print(f" [tool] {block.name}") + elif isinstance(message, ResultMessage): + print(f" [{message.num_turns} turns, ${message.total_cost_usd:.4f}]") + print() + + except KeyboardInterrupt: + break + except Exception as e: + print(f" [error] {e}") + + +def main(): + parser = argparse.ArgumentParser(description="Test Claude Agent SDK + PostHog LLM Analytics") + parser.add_argument( + "--prompt", + default="List the files in the current directory and tell me what this project is about. Be brief.", + help="Prompt to send", + ) + parser.add_argument("--interactive", "-i", action="store_true", help="Interactive chat mode") + parser.add_argument( + "--distinct-id", + default=os.environ.get("POSTHOG_DISTINCT_ID", "claude-agent-sdk-test"), + help="PostHog distinct ID", + ) + args = parser.parse_args() + + _check_deps() + posthog_client = _setup_posthog() + distinct_id = args.distinct_id + + extra_props = { + "app": "llm-analytics-apps", + "script": "test_claude_agent_sdk", + } + + if args.interactive: + asyncio.run(run_interactive(posthog_client, distinct_id)) + else: + asyncio.run(run_query(args.prompt, posthog_client, distinct_id, extra_props)) + + if posthog_client: + posthog_client.shutdown() + print("\nPostHog events flushed.") + + +if __name__ == "__main__": + main() From becb126fc5369d3eff27970aab519f5966a87547 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 1 Apr 2026 12:44:38 +0100 Subject: [PATCH 2/5] chore: add Makefile targets and docs for Claude Agent SDK test --- CLAUDE.md | 22 ++++++++++++++++++++++ Makefile | 15 ++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7527bce..14ffe0f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,12 +44,34 @@ uv run scripts/test_litellm.py ├── run-examples.sh # SDK example runner ├── scripts/ # demo data and test scripts │ ├── generate_demo_data.py # multi-provider demo data +│ ├── test_claude_agent_sdk.py # Claude Agent SDK integration test │ ├── test_*.py # Python integration tests │ └── test_*.ts # Node integration tests (Vercel AI, OTel) ├── trace-generator/ # mock trace builder └── screenshot-demo/ # UI screenshot tool ``` +## Claude Agent SDK integration + +Tests the `posthog.ai.claude_agent_sdk` integration (PostHog/posthog-python#477). +Requires installing local posthog-python since the integration isn't released yet. + +```bash +# One-time: install local posthog-python with the integration +make install-local-sdk + +# Run single query +make test-claude-agent-sdk + +# Interactive chat mode +make test-claude-agent-sdk-interactive + +# Custom prompt +uv run --no-sync scripts/test_claude_agent_sdk.py --prompt "your prompt here" +``` + +**Important:** Use `uv run --no-sync` (or the Makefile targets) to avoid `uv` overwriting the local SDK install with the PyPI version. + ## Configuration Environment variables in `.env`: diff --git a/Makefile b/Makefile index 37234fe..8e8d120 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: setup examples examples-list examples-all examples-parallel examples-install run-trace-generator run-trace-generator-debug demo-data demo-data-quick demo-data-tools demo-data-negative +.PHONY: setup examples examples-list examples-all examples-parallel examples-install run-trace-generator run-trace-generator-debug demo-data demo-data-quick demo-data-tools demo-data-negative install-local-sdk test-claude-agent-sdk test-claude-agent-sdk-interactive ## Install all dependencies setup: @@ -47,3 +47,16 @@ demo-data-tools: ## Generate negative/angry demo conversations for sentiment testing demo-data-negative: @uv run scripts/generate_demo_data.py --conversations 3 --max-turns 4 --parallel 3 --providers openai_chat --persona "an extremely frustrated customer who has been passed around to 5 different support agents" --topic "complaining about a product that keeps breaking" + +## Install local posthog-python for development (required for claude-agent-sdk integration) +install-local-sdk: + @uv pip install -e ../posthog-python + @echo "Local posthog-python installed. Use 'uv run --no-sync' to avoid overwriting." + +## Claude Agent SDK test (requires local posthog-python with integration) +test-claude-agent-sdk: + @uv run --no-sync scripts/test_claude_agent_sdk.py + +## Claude Agent SDK interactive mode +test-claude-agent-sdk-interactive: + @uv run --no-sync scripts/test_claude_agent_sdk.py --interactive From ff7c20f94ae0ca22475f36f7de53ab13ca20422c Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 1 Apr 2026 12:46:25 +0100 Subject: [PATCH 3/5] chore: add uv exclude-newer for supply chain safety --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 0ef6df3..a5e35ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,3 +24,6 @@ dependencies = [ "opentelemetry-instrumentation-openai", "opentelemetry-instrumentation-langchain", ] + +[tool.uv] +exclude-newer = "7 days" From 2389f5b9a8b9fb72965cf53d73a87639cdf5843d Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 1 Apr 2026 12:58:48 +0100 Subject: [PATCH 4/5] fix: run uv sync before installing local SDK --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8e8d120..e97eb7c 100644 --- a/Makefile +++ b/Makefile @@ -50,8 +50,9 @@ demo-data-negative: ## Install local posthog-python for development (required for claude-agent-sdk integration) install-local-sdk: + @uv sync @uv pip install -e ../posthog-python - @echo "Local posthog-python installed. Use 'uv run --no-sync' to avoid overwriting." + @echo "Local posthog-python installed. Use 'make test-claude-agent-sdk' to run." ## Claude Agent SDK test (requires local posthog-python with integration) test-claude-agent-sdk: From 921746bfaa3c60b35219494d9b70b729bad58c5e Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 1 Apr 2026 13:06:52 +0100 Subject: [PATCH 5/5] feat: use PostHogClaudeSDKClient for stateful interactive mode --- scripts/test_claude_agent_sdk.py | 61 ++++++++++++++++---------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/scripts/test_claude_agent_sdk.py b/scripts/test_claude_agent_sdk.py index e168a8e..838eab4 100644 --- a/scripts/test_claude_agent_sdk.py +++ b/scripts/test_claude_agent_sdk.py @@ -94,13 +94,7 @@ async def run_query(prompt: str, posthog_client, distinct_id: str, extra_props: async def run_interactive(posthog_client, distinct_id: str): from claude_agent_sdk import ClaudeAgentOptions, AssistantMessage, ResultMessage from claude_agent_sdk.types import TextBlock, ToolUseBlock - from posthog.ai.claude_agent_sdk import instrument - - ph = instrument( - client=posthog_client, - distinct_id=distinct_id, - properties={"app": "llm-analytics-apps", "mode": "interactive"}, - ) + from posthog.ai.claude_agent_sdk import PostHogClaudeSDKClient options = ClaudeAgentOptions( max_turns=10, @@ -108,32 +102,39 @@ async def run_interactive(posthog_client, distinct_id: str): permission_mode="bypassPermissions", ) - print("\nClaude Agent SDK — Interactive Mode") + print("\nClaude Agent SDK — Interactive Mode (stateful, multi-turn)") print("Type 'quit' to exit\n") - while True: - try: - prompt = input("> ").strip() - if not prompt: - continue - if prompt.lower() in ("quit", "exit", "q"): + async with PostHogClaudeSDKClient( + options, + posthog_client=posthog_client, + posthog_distinct_id=distinct_id, + posthog_properties={"app": "llm-analytics-apps", "mode": "interactive"}, + ) as client: + while True: + try: + prompt = input("> ").strip() + if not prompt: + continue + if prompt.lower() in ("quit", "exit", "q"): + break + + await client.query(prompt) + async for message in client.receive_response(): + if isinstance(message, AssistantMessage): + for block in message.content: + if isinstance(block, TextBlock): + print(f" {block.text[:500]}") + elif isinstance(block, ToolUseBlock): + print(f" [tool] {block.name}") + elif isinstance(message, ResultMessage): + print(f" [{message.num_turns} turns, ${message.total_cost_usd:.4f}]") + print() + + except KeyboardInterrupt: break - - async for message in ph.query(prompt=prompt, options=options): - if isinstance(message, AssistantMessage): - for block in message.content: - if isinstance(block, TextBlock): - print(f" {block.text[:500]}") - elif isinstance(block, ToolUseBlock): - print(f" [tool] {block.name}") - elif isinstance(message, ResultMessage): - print(f" [{message.num_turns} turns, ${message.total_cost_usd:.4f}]") - print() - - except KeyboardInterrupt: - break - except Exception as e: - print(f" [error] {e}") + except Exception as e: + print(f" [error] {e}") def main():