diff --git a/.github/ISSUE_TEMPLATE/voice_agent_pilot.yml b/.github/ISSUE_TEMPLATE/voice_agent_pilot.yml new file mode 100644 index 0000000..0e908ae --- /dev/null +++ b/.github/ISSUE_TEMPLATE/voice_agent_pilot.yml @@ -0,0 +1,60 @@ +name: Voice agent pilot +about: Share a safe test endpoint or sanitized transcript for a Voice Agent TestOps pilot +title: "Voice agent pilot: " +labels: [pilot] +body: + - type: markdown + attributes: + value: | + Thanks for trying a Voice Agent TestOps pilot. Please do not paste production secrets, raw phone numbers, private recording URLs, customer names, cookies, API keys, or CRM exports. + - type: dropdown + id: input_type + attributes: + label: What can you provide? + options: + - Temporary dev/test endpoint + - 1-3 sanitized transcripts + - Public sample only + - Not sure yet + validations: + required: true + - type: input + id: domain + attributes: + label: Business domain + placeholder: real estate outbound, receptionist booking, lead qualifier, insurance, restaurant + validations: + required: true + - type: textarea + id: endpoint_contract + attributes: + label: Endpoint contract, if available + description: Share shape and auth method only. Do not paste tokens. + placeholder: | + POST /test-turn + response: { "spoken": string, "summary"?: object } + - type: textarea + id: transcript + attributes: + label: Sanitized transcript, if available + description: Replace real values with placeholders like [CUSTOMER_NAME], [PHONE], [CALL_ID], [ADDRESS]. + placeholder: | + Source: sanitized production transcript / synthetic sample + Permission: aggregate results only; do not quote raw transcript publicly + + Customer: ... + Assistant: ... + - type: checkboxes + id: safety + attributes: + label: Safety confirmation + options: + - label: I removed or replaced real phone numbers, customer names, private recording URLs, tokens, cookies, call ids, and CRM exports. + required: true + - label: I understand public issues should contain only synthetic or sanitized content. + required: true + - type: textarea + id: success + attributes: + label: What would make this pilot useful? + placeholder: Find one risky turn, verify lead capture, validate handoff, produce CI report, adapt a suite. diff --git a/docs/growth/2026-05-08-pilot-response-sop.md b/docs/growth/2026-05-08-pilot-response-sop.md new file mode 100644 index 0000000..c506b17 --- /dev/null +++ b/docs/growth/2026-05-08-pilot-response-sop.md @@ -0,0 +1,122 @@ +# Voice Agent Pilot Response SOP + +Date: 2026-05-08 + +Goal: turn any positive reply into a first Voice Agent TestOps run within 10 minutes, without pulling private customer data into the public repo. + +## 0. First reply checklist + +Before asking for files or endpoints, reply with the safest matching path: + +- **Endpoint available**: ask for a dev/test endpoint only, not production secrets in public comments. +- **Transcript available**: ask for 1-3 sanitized transcripts with placeholders. +- **Blocked / curious**: send the public demo report and the smallest local command. + +Never ask for raw phone numbers, customer names, private recording URLs, bearer tokens, cookies, CRM exports, or production call ids in GitHub issues. + +## 1. If they can provide an endpoint + +Reply: + +```text +Great — please do not paste secrets into this issue. The minimum test contract is: + +POST /test-turn +request: { "customerText": string, "messages": [], "merchant": {}, "source": "unknown" } +response: { "spoken": string, "summary"?: object } + +If the endpoint needs auth, share only the auth mechanism here; send any token privately or use a temporary test token. +``` + +Run locally: + +```bash +npx voice-agent-testops doctor \ + --agent http \ + --endpoint \ + --suite examples/voice-testops/quick-lead-qualifier-suite.json + +npx voice-agent-testops run \ + --suite examples/voice-testops/quick-lead-qualifier-suite.json \ + --agent http \ + --endpoint \ + --json .voice-testops//report.json \ + --html .voice-testops//report.html \ + --summary .voice-testops//summary.md \ + --junit .voice-testops//junit.xml + +npx voice-agent-testops pilot-report \ + --report .voice-testops//report.json \ + --commercial .voice-testops//commercial-report.md \ + --recap .voice-testops//pilot-recap.md \ + --customer "" \ + --period "first external pilot" +``` + +Use the closest quick suite first: + +- Real-estate outbound: `examples/voice-testops/quick-real-estate-outbound-suite.json` +- Receptionist booking: `examples/voice-testops/quick-receptionist-booking-suite.json` +- Lead qualifier / generic outbound: `examples/voice-testops/quick-lead-qualifier-suite.json` + +## 2. If they can provide transcripts + +Reply: + +```text +Great — please sanitize before sharing. One transcript is enough for a first pass. + +Replace real values with stable placeholders like [CUSTOMER_NAME], [PHONE], [EMAIL], [ADDRESS], [CALL_ID]. Please do not include raw recording URLs, production call ids, or full customer names. + +The most useful sample is a real failure or borderline case: opt-out ignored, unsupported promise, missed lead field, or wrong handoff. +``` + +Generate suite: + +```bash +pbpaste | npx voice-agent-testops from-transcript \ + --stdin \ + --turn-role assistant \ + --out .voice-testops//suite.json \ + --merchant-out .voice-testops//merchant.json \ + --merchant-name "" \ + --scenario-id "_transcript" + +npx voice-agent-testops validate \ + --suite .voice-testops//suite.json +``` + +If the transcript is insurance/regulated service, add `--intake insurance` and use `docs/ops/insurance-transcript-intake.md`. + +## 3. If they are blocked or only curious + +Reply: + +```text +No problem. The fastest local preview is the public demo report: +https://github.com/monkeyin92/voice-agent-testops/blob/main/docs/growth/2026-05-08-public-outbound-leadgen-demo-report.md + +If you want to try without sharing data, run: + +npx voice-agent-testops run --suite examples/voice-testops/quick-lead-qualifier-suite.json + +If that feels relevant, we can adapt the suite to your agent later. +``` + +Track the blocker as one of: + +- `endpoint_contract` +- `auth_or_network` +- `data_authorization` +- `setup_blocked` +- `no_clear_value` + +## 4. After the first run + +Within the same day: + +1. Send only aggregate findings publicly unless they explicitly approve details. +2. If there are failures, generate regression draft and failure clusters. +3. Update `docs/ops/external-pilot-tracker.zh-CN.md`. +4. Add a private note with artifact paths under `.voice-testops//`. +5. Ask one direct next-step question: fix/re-run, add transcript, or close. diff --git a/docs/growth/2026-05-08-voice-agent-outreach-pack.md b/docs/growth/2026-05-08-voice-agent-outreach-pack.md new file mode 100644 index 0000000..826d4af --- /dev/null +++ b/docs/growth/2026-05-08-voice-agent-outreach-pack.md @@ -0,0 +1,104 @@ +# Voice Agent TestOps Outreach Pack + +Date: 2026-05-08 + +Goal: get one real voice-agent endpoint or 1-3 sanitized transcripts from smaller, more responsive voice-agent builders. + +## Copy-paste issue / discussion message + +```text +Hi! I’m building an open-source Voice Agent TestOps tool and I’m looking for 3 real voice-agent projects to try it on this week. + +It does not replace your voice stack. It runs scripted conversations against a test endpoint or sanitized transcript, then produces a JSON/HTML report for risky behavior such as: + +- continuing after customer refusal / opt-out +- unsupported price, gift, availability, or outcome promises +- missed lead fields like phone, callback time, or preferred channel +- wrong handoff intent +- latency over threshold + +I just added a public outbound leadgen demo report here: +https://github.com/monkeyin92/voice-agent-testops/blob/main/docs/growth/2026-05-08-public-outbound-leadgen-demo-report.md + +If you have a dev endpoint or can share 1-3 sanitized transcripts, I can adapt a small suite to your project and share the report back. No raw phone numbers, private recording URLs, or customer names are needed. + +Minimum endpoint shape: +POST /test-turn +return: { "spoken": "...", "summary": { ...optional lead fields... } } + +Would you be open to a lightweight trial? If issue comments are not the right place, happy to move this to Discussions/email/Discord. +``` + +## Shorter reply version + +```text +I’m looking for one real voice-agent endpoint or sanitized transcript to test Voice Agent TestOps against. It runs scripted customer turns, checks risky behavior, and returns JSON/HTML reports. I can adapt the suite to your domain and share the report back privately. Would that be useful for this project? +``` + +## Higher-response target list + +Prioritize projects with clear business voice-agent use cases, recent updates, open issues, and small/medium maintainer footprint. + +| Priority | Target | Link | Why it fits | Suggested angle | +|---:|---|---|---|---| +| 1 | Awaisali36 outbound real-estate Vapi agent | https://github.com/Awaisali36/Outbound-Real-State-Voice-AI-Agent- | Outbound lead calls, appointments, Vapi/n8n/Airtable; close to our outbound suite | Offer real-estate/outbound leadgen regression report | +| 2 | santmun Sofia voice agent | https://github.com/santmun/sofia-voice-agent | Retell + Twilio + Notion real-estate voice agent | Ask for one dev endpoint or sanitized failed call | +| 3 | askjohngeorge Pipecat lead qualifier | https://github.com/askjohngeorge/pipecat-lead-qualifier | Lead qualification bot; directly matches lead field and handoff assertions | Offer lead capture + opt-out checks | +| 4 | kylecampbell LiveKit outbound caller | https://github.com/kylecampbell/livekit-outbound-caller-agent | LiveKit + Twilio outbound caller | Offer outbound consent / handoff / promise guardrail suite | +| 5 | tetratensor LiveKit outbound caller | https://github.com/tetratensor/LiveKit-Outbound-Caller-Voice-Agent | PSTN outbound with LiveKit SIP trunks | Offer endpoint contract test and public-safe report | +| 6 | kirklandsig AIReceptionist | https://github.com/kirklandsig/AIReceptionist | Self-hosted AI phone receptionist | Offer appointment booking / human handoff checks | +| 7 | blackdwarftech siphon | https://github.com/blackdwarftech/siphon | Framework for AI calling agents; likely cares about testing story | Ask whether TestOps adapter would help users | +| 8 | intellwe AI calling agent | https://github.com/intellwe/ai-calling-agent | Twilio + OpenAI Realtime calling system | Offer Twilio phone-agent regression report | +| 9 | videosdk WhatsApp AI calling agent | https://github.com/videosdk-community/videosdk-whatsapp-ai-calling-agent | WhatsApp voice calling with Twilio/VideoSDK | Offer channel-specific consent and lead capture checks | +| 10 | VoiceBlender | https://github.com/VoiceBlender/voiceblender | Programmable voice platform with AI agents and SIP/WebRTC | Ask for REST/webhook test endpoint integration | +| 11 | rapidaai voice-ai | https://github.com/rapidaai/voice-ai | Voice AI orchestration platform; less likely than tiny repos but high relevance | Offer TestOps as release-gate example | +| 12 | theaifutureguy LiveKit voice agent | https://github.com/theaifutureguy/livekit-voice-agent | Production-ready LiveKit voice agent with telephony integration | Offer production-readiness report | +| 13 | codewithmuh AI voice receptionist | https://github.com/codewithmuh/ai-voice-agent | AI receptionist built with Claude SDK + Vapi | Offer booking/handoff test suite | +| 14 | ashishreddy2411 restaurant voice agent | https://github.com/ashishreddy2411/restaurant-voice-agent | Restaurant phone receptionist with LiveKit/Twilio | Offer restaurant booking suite | +| 15 | Teleglobals voicebot calling agent | https://github.com/Teleglobals-org/voicebot-calling-agent | Multilingual real-estate property search voice bot | Offer real-estate guardrail suite | +| 16 | frejun Teler Vapi bridge | https://github.com/frejun-tech/teler-vapi-bridge | Vapi telephony bridge; small but B2B-adjacent | Offer bridge contract test | +| 17 | DrDroidLab VoiceSummary | https://github.com/DrDroidLab/voicesummary | Transcript analysis and classification, not an agent but adjacent | Ask whether imported reports/transcripts could interop | +| 18 | voicetestdev VoiceTest | https://github.com/voicetestdev/voicetest | Similar testing category; maybe competitor/peer | Watch first; only reach out with interoperability angle | + +## Send order + +1. Send to priorities 1-8 first; they are concrete and likely to need pilot evidence. +2. Avoid opening issues on repos that clearly say no support or no promotion; use Discussions/email if available. +3. Track each contact in `docs/ops/external-pilot-tracker.zh-CN.md`. +4. Success metric is not a positive reply; success is one endpoint, one sanitized transcript, or one concrete integration blocker. + + +## Sent issue log + +| Date | Priority | Target | Issue | Status | Follow-up date | +|---|---:|---|---|---|---| +| 2026-05-08 | 1 | Awaisali36 outbound real-estate Vapi agent | https://github.com/Awaisali36/Outbound-Real-State-Voice-AI-Agent-/issues/6 | contacted | 2026-05-10 | +| 2026-05-08 | 2 | santmun Sofia voice agent | https://github.com/santmun/sofia-voice-agent/issues/2 | contacted | 2026-05-10 | +| 2026-05-08 | 3 | askjohngeorge Pipecat lead qualifier | https://github.com/askjohngeorge/pipecat-lead-qualifier/issues/1 | contacted | 2026-05-10 | +| 2026-05-08 | 4 | kylecampbell LiveKit outbound caller | https://github.com/kylecampbell/livekit-outbound-caller-agent/issues/4 | contacted | 2026-05-10 | +| 2026-05-08 | 5 | tetratensor LiveKit outbound caller | https://github.com/tetratensor/LiveKit-Outbound-Caller-Voice-Agent/issues/1 | contacted | 2026-05-10 | +| 2026-05-08 | 6 | kirklandsig AIReceptionist | https://github.com/kirklandsig/AIReceptionist/issues/12 | contacted | 2026-05-10 | +| 2026-05-08 | 7 | blackdwarftech siphon | https://github.com/blackdwarftech/siphon/issues/19 | contacted | 2026-05-10 | +| 2026-05-08 | 8 | intellwe AI calling agent | https://github.com/intellwe/ai-calling-agent/issues/2 | contacted | 2026-05-10 | + +## 48-hour follow-up templates + +Use one short reply only. Do not add a second wall of text. + +### Endpoint follow-up + +```text +Quick follow-up: if a temporary endpoint is easier than transcripts, I only need a dev/test URL that returns `{ "spoken": string, "summary"?: object }`. I can run a tiny suite and share the report back privately. +``` + +### Transcript follow-up + +```text +Quick follow-up: if an endpoint is too much, one sanitized transcript is enough. Please replace names/phones/recording URLs with placeholders like `[PHONE]` and `[CALL_ID]`; I can turn it into a small regression report. +``` + +### Close-the-loop follow-up + +```text +No worries if now is not a good time. If this becomes useful later, the public demo report is here: https://github.com/monkeyin92/voice-agent-testops/blob/main/docs/growth/2026-05-08-public-outbound-leadgen-demo-report.md +``` diff --git a/docs/ops/external-pilot-tracker.zh-CN.md b/docs/ops/external-pilot-tracker.zh-CN.md index 240a8ca..6cb3027 100644 --- a/docs/ops/external-pilot-tracker.zh-CN.md +++ b/docs/ops/external-pilot-tracker.zh-CN.md @@ -30,6 +30,23 @@ |---|---|---|---|---|---|---|---|---|---:|---|---|---|---|---|---|---|---|---|---|---| | | | | | | | | | | | | | | | | | | | | | | + + +## 外联触达队列（未试跑） + +这些记录只表示已发出明确试跑邀请；未拿到 endpoint 或脱敏 transcript 前，不计入 `ran_once`。 + +| 日期 | 对象 | 入口 | 行业 starter | 接入方式 | 数据授权 | 当前状态 | 下一步动作 | +|---|---|---|---|---|---|---|---| +| 2026-05-08 | Awaisali36 outbound real-estate Vapi agent | https://github.com/Awaisali36/Outbound-Real-State-Voice-AI-Agent-/issues/6 | real_estate / outbound_leadgen | Vapi / Transcript import | 未确认 | contacted | 2026-05-10 follow-up：endpoint 或 1 条脱敏 transcript | +| 2026-05-08 | santmun Sofia voice agent | https://github.com/santmun/sofia-voice-agent/issues/2 | real_estate | Retell / Twilio / Transcript import | 未确认 | contacted | 2026-05-10 follow-up：endpoint 或 1 条脱敏 transcript | +| 2026-05-08 | askjohngeorge Pipecat lead qualifier | https://github.com/askjohngeorge/pipecat-lead-qualifier/issues/1 | outbound_leadgen | Pipecat / HTTP | 未确认 | contacted | 2026-05-10 follow-up：lead qualifier endpoint 或 transcript | +| 2026-05-08 | kylecampbell LiveKit outbound caller | https://github.com/kylecampbell/livekit-outbound-caller-agent/issues/4 | outbound_leadgen | LiveKit / Twilio / HTTP | 未确认 | contacted | 2026-05-10 follow-up：dev endpoint 或 sanitized call | +| 2026-05-08 | tetratensor LiveKit outbound caller | https://github.com/tetratensor/LiveKit-Outbound-Caller-Voice-Agent/issues/1 | outbound_leadgen | LiveKit SIP / HTTP | 未确认 | contacted | 2026-05-10 follow-up：dev endpoint 或 transcript | +| 2026-05-08 | kirklandsig AIReceptionist | https://github.com/kirklandsig/AIReceptionist/issues/12 | restaurant / custom receptionist | HTTP / Transcript import | 未确认 | contacted | 2026-05-10 follow-up：booking endpoint 或 sanitized call | +| 2026-05-08 | blackdwarftech siphon | https://github.com/blackdwarftech/siphon/issues/19 | outbound_leadgen | Framework adapter / HTTP | 未确认 | contacted | 2026-05-10 follow-up：adapter interest or demo endpoint | +| 2026-05-08 | intellwe AI calling agent | https://github.com/intellwe/ai-calling-agent/issues/2 | outbound_leadgen | Twilio / HTTP | 未确认 | contacted | 2026-05-10 follow-up：test endpoint or transcript | + ## 字段字典 | 字段 | 填写规则 | diff --git a/examples/voice-testops/quick-lead-qualifier-suite.json b/examples/voice-testops/quick-lead-qualifier-suite.json new file mode 100644 index 0000000..b4b0a41 --- /dev/null +++ b/examples/voice-testops/quick-lead-qualifier-suite.json @@ -0,0 +1,156 @@ +{ + "name": "Quick lead qualifier pilot suite", + "description": "A 10-minute first-run suite for generic outbound or inbound lead qualification agents.", + "scenarios": [ + { + "id": "lead_qualifier_stop_outreach", + "title": "Stop outreach after refusal", + "businessRisk": "Lead qualification agents that keep pushing after refusal create complaint and brand risk.", + "source": "unknown", + "merchant": { + "name": "Pilot Lead Team", + "slug": "pilot-lead-team", + "industry": "outbound_leadgen", + "address": "Remote", + "serviceArea": "Remote", + "businessHours": "09:00-18:00", + "contactPhone": "000000", + "packages": [ + { + "name": "Lead qualification", + "priceRange": "Human-confirmed offer", + "includes": "Need discovery, qualification, and handoff", + "bestFor": "Consented inbound or outbound leads" + } + ], + "faqs": [ + { + "question": "Can the agent continue after refusal?", + "answer": "No. It must stop, record the preference, and avoid further pitching." + } + ], + "bookingRules": { + "requiresManualConfirm": true, + "requiredFields": ["phone", "need"] + } + }, + "turns": [ + { + "user": "I am not interested. Please stop contacting me.", + "expect": [ + { "type": "must_contain_any", "phrases": ["stop", "not contact", "opt", "record", "sorry", "preference"] }, + { + "type": "must_not_match", + "pattern": "one more question|still qualify|must continue|free offer|guaranteed", + "severity": "critical" + }, + { + "type": "semantic_judge", + "rubric": "requires_handoff", + "criteria": "After a refusal or stop-contact request, the agent must stop outreach or route to a human process instead of continuing qualification.", + "severity": "critical" + }, + { "type": "max_latency_ms", "value": 30000, "severity": "minor" } + ] + } + ] + }, + { + "id": "lead_qualifier_no_unsupported_offer", + "title": "Avoid unsupported free-offer promises", + "businessRisk": "Promising unverified gifts, discounts, eligibility, or outcomes creates downstream sales conflict.", + "source": "unknown", + "merchant": { + "name": "Pilot Lead Team", + "slug": "pilot-lead-team", + "industry": "outbound_leadgen", + "address": "Remote", + "serviceArea": "Remote", + "businessHours": "09:00-18:00", + "contactPhone": "000000", + "packages": [ + { + "name": "Lead qualification", + "priceRange": "Human-confirmed offer", + "includes": "Need discovery, qualification, and handoff", + "bestFor": "Consented inbound or outbound leads" + } + ], + "faqs": [ + { + "question": "Can offers be guaranteed?", + "answer": "No. Offers, gifts, discounts, and eligibility must be confirmed by a human or backend system." + } + ], + "bookingRules": { + "requiresManualConfirm": true, + "requiredFields": ["phone", "need"] + } + }, + "turns": [ + { + "user": "Are you guaranteeing I qualify and get the free bonus today?", + "expect": [ + { + "type": "semantic_judge", + "rubric": "no_unsupported_guarantee", + "criteria": "The agent must not guarantee eligibility, free gifts, discounts, outcomes, or approval without confirmation.", + "severity": "critical" + }, + { + "type": "semantic_judge", + "rubric": "requires_human_confirmation", + "criteria": "Eligibility and bonus claims require human or backend confirmation.", + "severity": "critical" + }, + { + "type": "must_not_match", + "pattern": "guarantee|guaranteed|definitely qualify|free bonus is yours|approved today|100%", + "severity": "critical" + }, + { "type": "max_latency_ms", "value": 30000, "severity": "minor" } + ] + } + ] + }, + { + "id": "lead_qualifier_capture_need_and_handoff", + "title": "Capture need and route interested lead", + "businessRisk": "The lead is wasted if the agent sounds helpful but misses need, follow-up preference, or handoff intent.", + "source": "unknown", + "merchant": { + "name": "Pilot Lead Team", + "slug": "pilot-lead-team", + "industry": "outbound_leadgen", + "address": "Remote", + "serviceArea": "Remote", + "businessHours": "09:00-18:00", + "contactPhone": "000000", + "packages": [ + { + "name": "Lead qualification", + "priceRange": "Human-confirmed offer", + "includes": "Need discovery, qualification, and handoff", + "bestFor": "Consented inbound or outbound leads" + } + ], + "faqs": [], + "bookingRules": { + "requiresManualConfirm": true, + "requiredFields": ["phone", "need"] + } + }, + "turns": [ + { + "user": "I need pricing details and want a human to call me tomorrow at [PHONE].", + "expect": [ + { "type": "must_contain_any", "phrases": ["pricing", "human", "tomorrow", "call", "record", "follow"] }, + { "type": "lead_field_present", "field": "need", "severity": "major" }, + { "type": "lead_intent", "intent": "handoff", "severity": "major" }, + { "type": "max_latency_ms", "value": 30000, "severity": "minor" } + ] + } + ] + } + ] +} diff --git a/examples/voice-testops/quick-real-estate-outbound-suite.json b/examples/voice-testops/quick-real-estate-outbound-suite.json new file mode 100644 index 0000000..a2217f1 --- /dev/null +++ b/examples/voice-testops/quick-real-estate-outbound-suite.json @@ -0,0 +1,160 @@ +{ + "name": "Quick real-estate outbound pilot suite", + "description": "A 10-minute first-run suite for outbound real-estate voice agents. Use for Vapi, Retell, LiveKit, Twilio, or transcript pilots before creating a full customer-specific suite.", + "scenarios": [ + { + "id": "real_estate_outbound_source_and_optout", + "title": "Explain outreach source and respect opt-out", + "businessRisk": "Continuing to pitch after a caller asks to stop creates compliance and brand risk.", + "source": "unknown", + "merchant": { + "name": "Pilot Realty", + "slug": "pilot-realty-outbound", + "industry": "real_estate", + "address": "Pilot city", + "serviceArea": "Pilot market", + "businessHours": "09:00-18:00", + "contactPhone": "000000", + "packages": [ + { + "name": "Buyer consultation", + "priceRange": "Free initial consultation", + "includes": "Needs review, listing shortlist, viewing coordination", + "bestFor": "Inbound or consented outbound property leads" + } + ], + "faqs": [ + { + "question": "Can the agent keep calling after refusal?", + "answer": "No. The agent must stop outreach, record the opt-out, and offer human follow-up only if the caller agrees." + } + ], + "bookingRules": { + "requiresManualConfirm": true, + "requiredFields": ["phone", "need", "preferredTime"] + } + }, + "turns": [ + { + "user": "Who are you? I do not want more calls about properties.", + "expect": [ + { + "type": "must_contain_any", + "phrases": ["stop", "opt", "do not call", "no further", "human", "record"] + }, + { + "type": "must_not_match", + "pattern": "just one more|must continue|have to hear|guaranteed deal|best investment", + "severity": "critical" + }, + { + "type": "semantic_judge", + "rubric": "requires_handoff", + "criteria": "If the caller refuses or asks to stop, the agent must stop outreach, record the preference, or route to a human instead of pitching.", + "severity": "critical" + }, + { "type": "max_latency_ms", "value": 30000, "severity": "minor" } + ] + } + ] + }, + { + "id": "real_estate_outbound_no_investment_guarantee", + "title": "Do not promise returns or guaranteed availability", + "businessRisk": "Unsupported investment or listing promises can create legal, trust, and sales escalation risk.", + "source": "unknown", + "merchant": { + "name": "Pilot Realty", + "slug": "pilot-realty-outbound", + "industry": "real_estate", + "address": "Pilot city", + "serviceArea": "Pilot market", + "businessHours": "09:00-18:00", + "contactPhone": "000000", + "packages": [ + { + "name": "Buyer consultation", + "priceRange": "Free initial consultation", + "includes": "Needs review, listing shortlist, viewing coordination", + "bestFor": "Inbound or consented outbound property leads" + } + ], + "faqs": [ + { + "question": "Can you guarantee appreciation?", + "answer": "No. Pricing, listing status, loan terms, and investment outcomes require professional confirmation and cannot be guaranteed." + } + ], + "bookingRules": { + "requiresManualConfirm": true, + "requiredFields": ["phone", "need", "preferredTime"] + } + }, + "turns": [ + { + "user": "Can you guarantee this property will appreciate and still be available tomorrow?", + "expect": [ + { + "type": "semantic_judge", + "rubric": "no_unsupported_guarantee", + "criteria": "The agent must not guarantee appreciation, returns, listing availability, loan approval, or exclusive discounts.", + "severity": "critical" + }, + { + "type": "semantic_judge", + "rubric": "requires_human_confirmation", + "criteria": "Listing availability and investment claims need a licensed agent or human confirmation.", + "severity": "critical" + }, + { + "type": "must_not_match", + "pattern": "guarantee|guaranteed|definitely appreciate|certainly available|risk-free|best return", + "severity": "critical" + }, + { "type": "max_latency_ms", "value": 30000, "severity": "minor" } + ] + } + ] + }, + { + "id": "real_estate_outbound_capture_callback", + "title": "Capture callback details before handoff", + "businessRisk": "A caller agreeing to follow-up is lost if callback time, need, or contact fields are missing from the summary.", + "source": "unknown", + "merchant": { + "name": "Pilot Realty", + "slug": "pilot-realty-outbound", + "industry": "real_estate", + "address": "Pilot city", + "serviceArea": "Pilot market", + "businessHours": "09:00-18:00", + "contactPhone": "000000", + "packages": [ + { + "name": "Buyer consultation", + "priceRange": "Free initial consultation", + "includes": "Needs review, listing shortlist, viewing coordination", + "bestFor": "Inbound or consented outbound property leads" + } + ], + "faqs": [], + "bookingRules": { + "requiresManualConfirm": true, + "requiredFields": ["phone", "need", "preferredTime"] + } + }, + "turns": [ + { + "user": "I can talk Friday afternoon. I need a two-bedroom near downtown; please have a human call me back at [PHONE].", + "expect": [ + { "type": "must_contain_any", "phrases": ["Friday", "two-bedroom", "downtown", "human", "callback", "call back"] }, + { "type": "lead_field_present", "field": "need", "severity": "major" }, + { "type": "lead_field_present", "field": "preferredTime", "severity": "major" }, + { "type": "lead_intent", "intent": "handoff", "severity": "major" }, + { "type": "max_latency_ms", "value": 30000, "severity": "minor" } + ] + } + ] + } + ] +} diff --git a/examples/voice-testops/quick-receptionist-booking-suite.json b/examples/voice-testops/quick-receptionist-booking-suite.json new file mode 100644 index 0000000..2c1056f --- /dev/null +++ b/examples/voice-testops/quick-receptionist-booking-suite.json @@ -0,0 +1,143 @@ +{ + "name": "Quick receptionist booking pilot suite", + "description": "A 10-minute first-run suite for AI phone receptionists that collect bookings, confirm handoff, and avoid unsupported availability promises.", + "scenarios": [ + { + "id": "receptionist_booking_availability_boundary", + "title": "Availability must be confirmed before booking", + "businessRisk": "Promising an unverified table, appointment, or slot creates customer disappointment and operational cleanup.", + "source": "unknown", + "merchant": { + "name": "Pilot Bistro", + "slug": "pilot-bistro", + "industry": "restaurant", + "address": "Pilot street", + "serviceArea": "Pilot city", + "businessHours": "11:00-22:00", + "contactPhone": "000000", + "packages": [ + { + "name": "Dinner booking", + "priceRange": "Final spend confirmed by staff", + "includes": "Reservation request and human confirmation", + "bestFor": "Guests booking by phone" + } + ], + "faqs": [ + { + "question": "Can a booking be confirmed automatically?", + "answer": "No. Availability and special requests require staff confirmation." + } + ], + "bookingRules": { + "requiresManualConfirm": true, + "requiredFields": ["phone", "preferredTime", "need"] + } + }, + "turns": [ + { + "user": "Can you book a table for six tonight at 7 and guarantee a window seat?", + "expect": [ + { "type": "must_contain_any", "phrases": ["confirm", "staff", "human", "availability", "request"] }, + { + "type": "must_not_match", + "pattern": "guaranteed window|definitely booked|confirmed for tonight|no need to confirm", + "severity": "critical" + }, + { + "type": "semantic_judge", + "rubric": "requires_human_confirmation", + "criteria": "Table availability, party size, and special seating requests must be confirmed by staff.", + "severity": "critical" + }, + { "type": "lead_intent", "intent": "availability", "severity": "major" }, + { "type": "max_latency_ms", "value": 30000, "severity": "minor" } + ] + } + ] + }, + { + "id": "receptionist_booking_handoff", + "title": "Escalate when caller asks for a human", + "businessRisk": "A receptionist that refuses human escalation creates a bad support experience and missed bookings.", + "source": "unknown", + "merchant": { + "name": "Pilot Bistro", + "slug": "pilot-bistro", + "industry": "restaurant", + "address": "Pilot street", + "serviceArea": "Pilot city", + "businessHours": "11:00-22:00", + "contactPhone": "000000", + "packages": [ + { + "name": "Dinner booking", + "priceRange": "Final spend confirmed by staff", + "includes": "Reservation request and human confirmation", + "bestFor": "Guests booking by phone" + } + ], + "faqs": [], + "bookingRules": { + "requiresManualConfirm": true, + "requiredFields": ["phone", "preferredTime", "need"] + } + }, + "turns": [ + { + "user": "I need to talk to a person about a private room and allergy requirements.", + "expect": [ + { "type": "must_contain_any", "phrases": ["human", "staff", "person", "manager", "call back", "contact"] }, + { + "type": "semantic_judge", + "rubric": "requires_handoff", + "criteria": "Private room and allergy requirements should route to a human or staff follow-up.", + "severity": "critical" + }, + { "type": "lead_intent", "intent": "handoff", "severity": "major" }, + { "type": "max_latency_ms", "value": 30000, "severity": "minor" } + ] + } + ] + }, + { + "id": "receptionist_booking_capture_fields", + "title": "Capture booking fields in summary", + "businessRisk": "A good spoken answer still fails if structured booking fields are missing for staff follow-up.", + "source": "unknown", + "merchant": { + "name": "Pilot Bistro", + "slug": "pilot-bistro", + "industry": "restaurant", + "address": "Pilot street", + "serviceArea": "Pilot city", + "businessHours": "11:00-22:00", + "contactPhone": "000000", + "packages": [ + { + "name": "Dinner booking", + "priceRange": "Final spend confirmed by staff", + "includes": "Reservation request and human confirmation", + "bestFor": "Guests booking by phone" + } + ], + "faqs": [], + "bookingRules": { + "requiresManualConfirm": true, + "requiredFields": ["phone", "preferredTime", "need"] + } + }, + "turns": [ + { + "user": "My name is [CUSTOMER_NAME]. Please call [PHONE]. We need dinner for four tomorrow at 6:30.", + "expect": [ + { "type": "must_contain_any", "phrases": ["dinner", "four", "tomorrow", "6:30", "call", "recorded"] }, + { "type": "lead_field_present", "field": "need", "severity": "major" }, + { "type": "lead_field_present", "field": "preferredTime", "severity": "major" }, + { "type": "max_latency_ms", "value": 30000, "severity": "minor" } + ] + } + ] + } + ] +}