From 3efd82a906ee03eb79851bd11a79e09a010a3356 Mon Sep 17 00:00:00 2001 From: Alex Ultra <111590326+alex-mextner@users.noreply.github.com> Date: Mon, 2 Nov 2026 06:08:23 +0100 Subject: [PATCH 1/4] fix(ai): serialize agent runs per chat and make create_event idempotent and same-day safe (Refs #929, #520) GramIO starts every update's handler without waiting for the previous one, so two quick messages in one chat ran two agents on overlapping history (2026-11-01: every event was created twice). - ChatRunGate: CalendarBotAgent.run is FIFO per chat id, other chats parallel. A waiting message shows the typing indicator; a run ahead that outlives its lease (budget + 30 s) no longer blocks the chat and the user is told once (never for a scheduled run; every run that starts beside a still-unfinished one is told). Shutdown ends the wait and starts no lease timer. - A message's place in its chat's line is reserved in the chat-logging middleware, in the same step that saves its row, so arrival order is row order even when a handler waits on a scene or retry lookup before the agent; the update's handling runs in that reservation (AsyncLocalStorage), so its agent run takes the place over, the update's end (or 60 s) releases an unused one. Turns queued behind a run are left out of its history and of get_history. - A run that owns its message (took over a reservation; never a scheduled or supplement run) reads it after the earlier run's rows without changing its id (chat_history.turn_order, migration 069): the per-user chatHistoryIds map, action log and built contexts stay valid. - create_event (EventRepository.createUnlessDuplicate, one immediate transaction): same calendar only (own events with no Google calendar / primary alias / the registered primary calendar id, owner_id, or the group; secondary-calendar imports never match), all-day values compared as floating dates with allDayDates in each event's own zone, and the event row and its reminders are written in the same transaction (also for allow_duplicate). Exact repeat (same title, start/end, stated details, 10 min) -> success with alreadyExists; same title on the same local day otherwise -> SAME_DAY_EVENT (not_applied, use update_event); allow_duplicate: true makes a deliberate copy. - Intent answers: ToolResult.alreadyExists (success, mutationState not_applied) is carried by IntentExecutor into ToolAnswer, rendered as "already in the calendar", and replaces the closing respond of basis.event.create / create_invite, so they never say "created". - Routing to AI agent, Agent run started and Agent run complete log updateId and messageId. Note for the session rewriting write-outcomes.ts / write-receipt.ts (not edited here): a create_event that found its event returns success: true with ToolResult.alreadyExists = { entity: 'event', id } and no created-event evidence. The receipt fold must read result.alreadyExists (not the output prose) and report "already in the calendar" with that id, not count it as a created write. A same-day conflict is success: false, mutationState: not_applied, error starting SAME_DAY_EVENT. Review: single-seat review (Opus only; Sol quota exhausted). Remaining medium/low findings are deferred in #929. --- docs/reference/deploy-runbook.md | 17 + src/bot/handlers/message.handler.ts | 11 +- src/bot/handlers/message/agent-context.ts | 10 +- src/bot/index.ts | 1 + src/bot/middleware/chat-logging.ts | 26 +- src/bot/pipeline/ai-agent-layer.ts | 27 +- src/bot/pipeline/pipeline.ts | 2 + src/bot/pipeline/types.ts | 2 + src/bot/types.ts | 5 + .../constants/messages/en/agent-status.ts | 2 + src/config/constants/messages/en/ai-tools.ts | 3 + .../constants/messages/ru/agent-status.ts | 2 + src/config/constants/messages/ru/ai-tools.ts | 4 + src/database/migrations.ts | 8 + .../repositories/chat-history.repository.ts | 100 ++- src/database/repositories/event.repository.ts | 175 +++++- src/services/ai/agent.ts | 59 +- src/services/ai/agent/conversation.ts | 11 +- src/services/ai/agent/queued-turn.ts | 98 +++ src/services/ai/agent/run-limits.ts | 5 + src/services/ai/agent/turn-finish.ts | 2 + src/services/ai/chat-run-gate.ts | 330 ++++++++++ src/services/ai/chat-turn-context.ts | 19 + .../ai/tool-handlers/events/create.ts | 103 ++- src/services/ai/tool-handlers/events/types.ts | 1 + src/services/ai/tool-handlers/history.ts | 2 + src/services/ai/tool-handlers/shared.ts | 2 +- src/services/ai/tool-schemas.ts | 1 + src/services/ai/tools.ts | 6 + src/services/ai/types.ts | 14 + src/services/event/event-service.ts | 93 ++- src/services/intent/answer-renderers.ts | 3 + src/services/intent/answer-wording.ts | 5 + src/services/intent/intent-executor.ts | 30 +- test/bot/handlers/message.handler.test.ts | 9 + test/bot/middleware/chat-logging.test.ts | 86 +++ .../pipeline/ai-agent-layer-overlap.test.ts | 265 ++++++++ .../scenes/connect-telegram.secrets.test.ts | 1 + .../chat-history.repository.test.ts | 107 ++++ test/helpers/manual-time.ts | 27 + test/helpers/overlap-fixtures.ts | 152 +++++ .../canonical-basis-behavior.test.ts | 18 + test/services/ai/agent-chat-queue.test.ts | 328 ++++++++++ test/services/ai/agent-shutdown-drain.test.ts | 23 + test/services/ai/chat-run-gate.test.ts | 510 +++++++++++++++ .../events-duplicate-create.test.ts | 595 ++++++++++++++++++ .../tool-handlers/events-local-times.test.ts | 8 +- .../services/ai/tool-handlers/history.test.ts | 17 + test/services/intent/intent-executor.test.ts | 73 +++ .../intent/response-formatter.test.ts | 14 + 50 files changed, 3294 insertions(+), 118 deletions(-) create mode 100644 src/services/ai/agent/queued-turn.ts create mode 100644 src/services/ai/chat-run-gate.ts create mode 100644 src/services/ai/chat-turn-context.ts create mode 100644 test/bot/pipeline/ai-agent-layer-overlap.test.ts create mode 100644 test/helpers/manual-time.ts create mode 100644 test/helpers/overlap-fixtures.ts create mode 100644 test/services/ai/agent-chat-queue.test.ts create mode 100644 test/services/ai/chat-run-gate.test.ts create mode 100644 test/services/ai/tool-handlers/events-duplicate-create.test.ts diff --git a/docs/reference/deploy-runbook.md b/docs/reference/deploy-runbook.md index 0cfb754b5..d0962ba13 100644 --- a/docs/reference/deploy-runbook.md +++ b/docs/reference/deploy-runbook.md @@ -110,6 +110,23 @@ Known limits: the probe runs on the PC, so a PC that is off or offline raises no Also known: (1) an outage seen by the PC probe and, separately, by a queued alert (the DO watchdog's POST, a crash alert) can start two sessions 30 minutes apart: dedupe is per alert id and per probe incident, not across sources; the second session's brief tells it to check `omp-steer list` first. (2) The unit starts at login or boot (linger); if the keyring is still locked, `keychain exec` fails with its own exit code, which `RestartPreventExitStatus=78` does not cover: the unit restarts every 30 s without being able to tell you. After a reboot check `systemctl --user is-active hcb-alert-watcher`; `keychain unlock` fixes it. (3) Bun loads the script once: after a merge that changes `scripts/hcb-alert-watcher.ts` run `systemctl --user restart hcb-alert-watcher` (the installer restarts only when the unit text changes). (4) A pop whose response times out before any header arrives may have consumed an alert on the server; only the journal line "queue step failed" shows it. (5) The queue poll refuses redirects so the token never travels to another host. +## Agent runs per chat and repeated creates (#929, #520) + +GramIO hands every Telegram update to its handler without waiting for the previous one (`EventQueue.process` in `@gramio/composer` 0.3.4, webhook and long polling alike), so two messages of one chat used to run two agents at once; on 2026-11-01 two quick «Да» created every event twice. Now `CalendarBotAgent.run` goes through `ChatRunGate` (`src/services/ai/chat-run-gate.ts`): one run at a time per chat id, first come first served, other chats unaffected. This covers every caller (message pipeline, `/cal`, button continuations, voice, scheduled AI). + +- The place in a chat's line is reserved when the message is saved, in the same synchronous step (`chat-logging` middleware → `CalendarBotAgent.reserveChatTurn` → `ChatRunGate.reserve`), not when its handler reaches the agent: a handler that waits on a scene or retry lookup first cannot be overtaken by a message saved after it. The update's handling runs inside that reservation (`withChatTurn`, an async context, `src/services/ai/chat-turn-context.ts`), so the agent run it starts (text pipeline, `/cal`, a button) takes the place over with no id passed along; a run started elsewhere (scheduled, retry) finds none and joins the end without a message of its own, whatever row id it borrows for the action log; the reservation is released when the update's handling ends (an intent answered it, a command, an error) or after 60 s unused (`Not waiting any longer` / `A reserved place in the chat was never used` in the log). +- A queued run starts after the earlier one has written its turn. It re-reads the user and the ±2 week schedule and reads its own message after the earlier run's rows (`ChatHistoryRepository.placeUserTurnAfterLatest` sets `chat_history.turn_order`; the row keeps its id, so the action log, the per-user `chatHistoryIds` map and built contexts stay valid). Only a run that took over a reserved place has an own message to place (and only after a bot row of the same conversation, private or group); scheduled/synthetic runs borrow the user's latest row id for the action log and never move it. It is not shown turns queued behind it, in its history or through `get_history` (`ctx.deferredTurnRowIds`), so its model reads «request, what was done, this message». The request budget (90 s) counts from the start of the run, not from arrival. +- A run that never finishes holds the chat for at most its budget plus 30 s (`CHAT_RUN_GRACE_MS`); then the next run goes on and the log says `Not waiting any longer for the earlier agent run in this chat`. A queued run is in `CalendarBotAgent.inFlight`, so the shutdown drain waits for it; at shutdown the wait ends at once and the run takes its normal fail-fast path (notice and stored retry), so no queued message is dropped. +- Log lines to look for: `Agent run started after waiting for the previous run in this chat` (`waitedMs`, `queuedBehind`) and `Routing to AI agent` with `updateId`/`messageId`: the same ids twice mean a Telegram redelivery, new ids a second message. +- While a message waits the chat shows the typing indicator (refreshed every 4 s). If the run ahead outlives its lease (`CHAT_RUN_GRACE_MS` after the 90 s budget, so about 120 s of waiting for the run directly ahead; the lease bounds the wait for that one run only, so the Nth message in a long queue of slow runs waits for every run ahead of it in turn, up to about (N-1) leases), the waiting message goes on beside it and the user is told once (`agent_queue_lease_expired`, ru and en; not for a scheduled run, which has no message of theirs, nor for a supplement); every run that starts beside a still-unfinished earlier run gets the notice, so it is never a silent parallel run. `Agent run started` and `Agent run complete` carry `updateId`/`messageId` too. +- Every kind of run uses the gate, keyed by its chat id: supplement runs (silent, optional text: no typing, no notice, so a supplement can sit behind a main run for up to the same ~120 s and then speaks to nobody) and scheduled/system runs (their chat is the user's private chat, so a scheduled reminder run waits behind an active conversation and vice versa). The worst case for any run is that wait plus its own 90 s budget. +- `create_event` (`EventRepository.createUnlessDuplicate`, one immediate SQLite transaction, so it holds across connections and restarts) compares with the live, non-recurring events of the same calendar (the owner's own events with no Google calendar, the `primary` alias or the id registered as the user's primary calendar, a secretary's `owner_id`, or the same group; events imported from a secondary Google calendar never count) that have the same title (trimmed, whitespace collapsed, case folded) and all-day flag: + - `duplicate`: same start and end (all-day: the same floating dates, as `allDayDates` reads them), the same description, location and reminders the caller stated, created within 10 minutes (`RECENT_DUPLICATE_WINDOW_MS`). It comes back as `Event already exists …`: success, `alreadyExists` on the tool result, no second row, reminders, notifications or calendar push. + - `same_day`: any other candidate on the same local day, whatever its age (the «move by creating» case of #520): `SAME_DAY_EVENT` error, `mutationState: not_applied`, pointing at `update_event` with the existing id. + - The model makes a deliberate second copy with `allow_duplicate: true`. + - The event row and its reminders are written in the same transaction (also for `allow_duplicate` and every other `EventService.createEvent` caller); a failed reminder write leaves no event, so a retry starts clean. All-day dates are read in the zone each event was saved with. + - The result of a repeat is `success: true`, `mutationState: not_applied`, `alreadyExists`. Intent answers: it is carried through `intent-executor` into `ToolAnswer`, and both the renderer and a closing `respond:` step (`basis.event.create`, `basis.event.create_invite`) say the event is already in the calendar instead of «created». The run-failure receipt wording («Completed: Create event») lives in `write-outcomes.ts` and is left to the receipt rewrite tracked in #929. + ## Public ingress (Cloudflare tunnel) `hypercal.mextner.com` is a proxied CNAME in the `mextner.com` Cloudflare zone (account `invntrm@gmail.com`) to the tunnel `odroid-home` (`872e1f2e-3488-4e80-955e-8aba2fada806`). cloudflared on the odroid (`cloudflared.service`, `--token-file /etc/cloudflared/token`) runs it with a remotely managed configuration, so the hostname rules are edited in Cloudflare, not on the odroid. Bot traffic is the rule `hypercal.mextner.com` -> `http://localhost:3001`; every path reaches the bot, which serves only its own routes (`/health`, `/ready`, `/webhook/telegram`, `/webhooks/google-calendar`, `/oauth/google/callback`, `/admin/alerts*` behind the bearer token). While the container is being replaced the tunnel answers 502; Telegram and Google retry, but there is no proxy holding requests as the retired DO Caddy did (`lb_try_duration`). diff --git a/src/bot/handlers/message.handler.ts b/src/bot/handlers/message.handler.ts index be53376de..5825e606d 100644 --- a/src/bot/handlers/message.handler.ts +++ b/src/bot/handlers/message.handler.ts @@ -420,7 +420,16 @@ export function createMessageHandler(deps: MessageHandlerDeps) { ); const run = () => - runPipeline(ctx, messageText, layers, groupContext, incomingMsgId, admission.wasExplicitInvocation); + runPipeline( + ctx, + messageText, + layers, + groupContext, + incomingMsgId, + admission.wasExplicitInvocation, + undefined, + ctx.chatHistoryId, + ); if (admission.isGroupSessionMessage && deps.botToken) { await runPipelineWithTyping(run, chatId, deps.botToken); } else { diff --git a/src/bot/handlers/message/agent-context.ts b/src/bot/handlers/message/agent-context.ts index fe15cb5f0..b30cc38d1 100644 --- a/src/bot/handlers/message/agent-context.ts +++ b/src/bot/handlers/message/agent-context.ts @@ -2,12 +2,11 @@ // Builds the per-request AgentContext (sender, repositories, chat-history ids) for the message handler. import type { EventOccurrence, User } from '../../../database/types.ts'; +import { scheduleWindowAround } from '../../../services/ai/agent/queued-turn.ts'; import type { AgentContext } from '../../../services/ai/types.ts'; import { cmdLogger } from '../../../utils/logger.ts'; import type { MessageHandlerDeps } from './types.ts'; -const SCHEDULE_CONTEXT_WINDOW_MS = 14 * 24 * 60 * 60 * 1000; - interface AgentGroupInfo { isGroup: boolean; groupChatId?: number; @@ -29,13 +28,10 @@ function describeSecretaryAccess(deps: MessageHandlerDeps, user: User): string | .join(', '); } -/** Events two weeks either side of now, so the model knows the user's nearby schedule; empty on a read failure. */ +/** The nearby schedule the model is shown; empty on a read failure. */ function recentEventsWindow(deps: MessageHandlerDeps, user: User): EventOccurrence[] { try { - const now = new Date(); - const start = new Date(now.getTime() - SCHEDULE_CONTEXT_WINDOW_MS).toISOString(); - const end = new Date(now.getTime() + SCHEDULE_CONTEXT_WINDOW_MS).toISOString(); - return deps.eventService.getEventsInRange(user.telegram_id, start, end); + return scheduleWindowAround(deps.eventService, user.telegram_id); } catch (err) { cmdLogger.error({ err, userId: user.telegram_id }, 'Failed to build schedule context'); return []; diff --git a/src/bot/index.ts b/src/bot/index.ts index b3662731c..5038d4c0a 100644 --- a/src/bot/index.ts +++ b/src/bot/index.ts @@ -498,6 +498,7 @@ export function createBot(token: string, db: DatabaseService, aiConfig: AgentCon conversationLogger, actionLog: db.actionLog, chatHistoryIds, + reserveChatTurn: (chatId, rowId) => agent.reserveChatTurn(chatId, rowId), isConnectWizardInput: connectWizardGuard.isConnectWizardInput, }), ) diff --git a/src/bot/middleware/chat-logging.ts b/src/bot/middleware/chat-logging.ts index f7ea250de..8595af700 100644 --- a/src/bot/middleware/chat-logging.ts +++ b/src/bot/middleware/chat-logging.ts @@ -7,6 +7,8 @@ import type { Next, TelegramUpdate } from 'gramio'; import type { ActionLogRepository } from '../../database/repositories/action-log.repository.ts'; import type { User } from '../../database/types.ts'; +import type { ChatTurnReservation } from '../../services/ai/chat-run-gate.ts'; +import { withChatTurn } from '../../services/ai/chat-turn-context.ts'; import type { ConversationLogger } from '../../services/conversation-logger.ts'; import { resolveCallbackButtonLabel } from '../callback-label.ts'; import { parseAiBtnPayload } from '../handlers/callback.handler.ts'; @@ -24,6 +26,8 @@ interface ChatLoggingContext { update?: TelegramUpdate; send?: ReplyFn; editText?: ReplyFn; + /** Set to the row saved for this update's user turn, so the handler reads its own row, not the user's latest. */ + chatHistoryId?: number; } export interface ChatLoggingDeps { @@ -31,6 +35,8 @@ export interface ChatLoggingDeps { actionLog: Pick; /** Latest user chat_history row per user, read by the AI pipeline for the current turn. */ chatHistoryIds: Map; + /** Holds a message's place in its chat's agent line the moment the message is saved; see ChatRunGate.reserve. */ + reserveChatTurn: (chatId: number, rowId: number) => ChatTurnReservation; /** The connect-wizard guard's verdict on this update's text or edit, reached before this middleware. */ isConnectWizardInput: ConnectWizardGuard['isConnectWizardInput']; } @@ -41,13 +47,16 @@ interface LoggingTurn { logChatId: number | undefined; /** Chat id for the action log, falling back to the user's private chat. */ actionChatId: number; + /** The chat_history row this update's user turn was saved as; absent when the update saved none. */ + savedRowId?: number; } export function createChatLogging(deps: ChatLoggingDeps) { - const { conversationLogger, actionLog, chatHistoryIds, isConnectWizardInput } = deps; + const { conversationLogger, actionLog, chatHistoryIds, reserveChatTurn, isConnectWizardInput } = deps; function saveUserTurn(turn: LoggingTurn, text: string): void { - chatHistoryIds.set(turn.userId, conversationLogger.logUserMessage(turn.userId, text, turn.logChatId)); + turn.savedRowId = conversationLogger.logUserMessage(turn.userId, text, turn.logChatId); + chatHistoryIds.set(turn.userId, turn.savedRowId); } function logIncomingText(turn: LoggingTurn, text: string, messageId: number | undefined, inConnectWizard: boolean) { @@ -165,7 +174,18 @@ export function createChatLogging(deps: ChatLoggingDeps) { }; logIncomingUpdate(update, turn, isConnectWizardInput(context)); + if (turn.savedRowId === undefined) { + wrapBotReplies(context, turn); + return next(); + } + context.chatHistoryId = turn.savedRowId; + // Saved and reserved in one synchronous step, so the order of places in the chat's line is the order of rows. + const reservation = reserveChatTurn(turn.actionChatId, turn.savedRowId); wrapBotReplies(context, turn); - return next(); + try { + return await withChatTurn(reservation, () => next()); + } finally { + reservation.release(); + } }; } diff --git a/src/bot/pipeline/ai-agent-layer.ts b/src/bot/pipeline/ai-agent-layer.ts index f8f18fb60..5794358c3 100644 --- a/src/bot/pipeline/ai-agent-layer.ts +++ b/src/bot/pipeline/ai-agent-layer.ts @@ -32,12 +32,30 @@ interface AgentLayerExtra { feedbackContext?: FeedbackThreadContext; groupContext?: GroupContext; incomingMessageId?: number; + chatHistoryId?: number; supplementMode?: boolean; supplementAutoResponse?: string; wasExplicitInvocation?: boolean; retryAttempt?: number; } +/** + * The routing line. `updateId` (Telegram's, per update) and `messageId` tell a redelivery of one update + * (same ids twice) from a second message (new ids). The message text is what the line always carried. + */ +function logRouting(ctx: BotCommandContext, user: User, messageText: string, extra: AgentLayerExtra | undefined): void { + cmdLogger.info( + { + userId: user.telegram_id, + messageText, + ...(ctx.updateId !== undefined && { updateId: ctx.updateId }), + ...(extra?.incomingMessageId !== undefined && { messageId: extra.incomingMessageId }), + ...(extra?.supplementMode && { supplementMode: true }), + }, + 'Routing to AI agent', + ); +} + export function createAiAgentLayer(deps: AgentLayerDeps) { /** Cancel any pending retry when a fresh user message arrives. */ async function cancelPendingRetry(userId: number): Promise { @@ -157,10 +175,14 @@ export function createAiAgentLayer(deps: AgentLayerDeps) { extra?.incomingMessageId, ); + // The row saved for this very message; the per-user map may already name a newer message's row. + if (extra?.chatHistoryId !== undefined) agentContext.chatHistoryId = extra.chatHistoryId; + if (extra?.feedbackContext && agentContext.feedback) { agentContext.feedback.feedbackContext = extra.feedbackContext; } + if (ctx.updateId !== undefined) agentContext.updateId = ctx.updateId; agentContext.retryAttempt = currentAttempt; if (deps.retryQueue) { @@ -176,10 +198,7 @@ export function createAiAgentLayer(deps: AgentLayerDeps) { await attachScenePause(agentContext, user.telegram_id); - cmdLogger.info( - { userId: user.telegram_id, messageText, ...(extra?.supplementMode && { supplementMode: true }) }, - 'Routing to AI agent', - ); + logRouting(ctx, user, messageText, extra); await runAgent(agentContext, ctx, user, messageText, extra); return { handled: true }; diff --git a/src/bot/pipeline/pipeline.ts b/src/bot/pipeline/pipeline.ts index 22aa6b199..ce7703090 100644 --- a/src/bot/pipeline/pipeline.ts +++ b/src/bot/pipeline/pipeline.ts @@ -12,6 +12,7 @@ export async function runPipeline( incomingMessageId?: number, wasExplicitInvocation?: boolean, retryAttempt?: number, + chatHistoryId?: number, ): Promise { let feedbackContext: FeedbackThreadContext | undefined; let supplementMode = false; @@ -27,6 +28,7 @@ export async function runPipeline( supplementAutoResponse, wasExplicitInvocation, retryAttempt, + chatHistoryId, }); if (result.handled) { if ('needsSupplement' in result) { diff --git a/src/bot/pipeline/types.ts b/src/bot/pipeline/types.ts index f233e41dc..0868fc8f0 100644 --- a/src/bot/pipeline/types.ts +++ b/src/bot/pipeline/types.ts @@ -57,6 +57,8 @@ export type PipelineLayer = ( feedbackContext?: FeedbackThreadContext; groupContext?: GroupContext; incomingMessageId?: number; + /** The chat_history row saved for this very message (see DerivedProps.chatHistoryId). */ + chatHistoryId?: number; supplementMode?: boolean; supplementAutoResponse?: string; /** True when the user explicitly addressed the bot (DM, @mention, "Бот,", reply). diff --git a/src/bot/types.ts b/src/bot/types.ts index 02f8fd2b0..30f1a6d87 100644 --- a/src/bot/types.ts +++ b/src/bot/types.ts @@ -11,6 +11,11 @@ export interface DerivedProps { dbUser: User | undefined; userTimezone: string | undefined; lang: 'en' | 'ru'; + /** + * The chat_history row the logging middleware saved for this update's user turn. Unlike the per-user + * `chatHistoryIds` map it cannot be overwritten by a message that arrives while this one is handled. + */ + chatHistoryId?: number; } /** diff --git a/src/config/constants/messages/en/agent-status.ts b/src/config/constants/messages/en/agent-status.ts index af548c9e2..c87460b3e 100644 --- a/src/config/constants/messages/en/agent-status.ts +++ b/src/config/constants/messages/en/agent-status.ts @@ -24,6 +24,8 @@ export const enAgentStatus = { agent_restarting_unconfirmed: '🔄 Restarting for an update — if I have not answered this in a few minutes, send it again.', agent_timeout: '⚠️ That took too long — timing out.', + agent_queue_lease_expired: + '⏳ My previous answer in this chat is taking too long, so I am handling your message now. If something shows up twice, check your calendar.', ai_response_blocked: 'The provider stopped this response. I will not automatically repeat the request.', ai_unanswered: '⚠️ I could not answer that — please ask again.', ai_unanswered_writes: (writes: string) => `⚠️ I could not put an answer together — here is what I did:\n${writes}`, diff --git a/src/config/constants/messages/en/ai-tools.ts b/src/config/constants/messages/en/ai-tools.ts index 50b03d057..11c0e1627 100644 --- a/src/config/constants/messages/en/ai-tools.ts +++ b/src/config/constants/messages/en/ai-tools.ts @@ -157,6 +157,9 @@ export const enAiTools = { noUpcomingEvents: 'No upcoming events.', upcomingEvents: (count: number, lines: string) => `Next ${count} events:\n${lines}`, eventCreated: (parts: string) => `Event created: ${parts}`, + eventAlreadyExists: (parts: string) => `Event already exists (saved moments ago, not created again): ${parts}`, + eventSameDay: (title: string, id: number, when: string) => + `“${title}” (id: ${id}) is already in the calendar on that day (${when}), so nothing was created.`, eventUpdated: (parts: string) => `Event updated: ${parts}`, eventDeleted: (title: string, id: number) => `Event "${title}" (id: ${id}) deleted.`, eventDeclined: (id: number) => diff --git a/src/config/constants/messages/ru/agent-status.ts b/src/config/constants/messages/ru/agent-status.ts index 8826a14f2..e295e5593 100644 --- a/src/config/constants/messages/ru/agent-status.ts +++ b/src/config/constants/messages/ru/agent-status.ts @@ -24,6 +24,8 @@ export const ruAgentStatus = { agent_restarting_unconfirmed: '🔄 Перезапускаюсь на обновление — если не отвечу на это через пару минут, повтори запрос.', agent_timeout: '⚠️ Что-то долго думаю — прерываю.', + agent_queue_lease_expired: + '⏳ Предыдущий ответ в этом чате слишком затянулся, поэтому берусь за твоё сообщение сейчас. Если что-то продублируется, проверь календарь.', ai_response_blocked: 'Провайдер остановил ответ. Я не буду автоматически повторять этот запрос.', ai_unanswered: '⚠️ Не получилось ответить — повтори вопрос.', ai_unanswered_writes: (writes: string) => `⚠️ Ответ не получился — вот что сделано:\n${writes}`, diff --git a/src/config/constants/messages/ru/ai-tools.ts b/src/config/constants/messages/ru/ai-tools.ts index ec84082ac..beb78fc14 100644 --- a/src/config/constants/messages/ru/ai-tools.ts +++ b/src/config/constants/messages/ru/ai-tools.ts @@ -162,6 +162,10 @@ export const ruAiTools = { noUpcomingEvents: 'Предстоящих событий нет.', upcomingEvents: (count: number, lines: string) => `Следующие ${count}:\n${lines}`, eventCreated: (parts: string) => `Событие создано: ${parts}`, + eventAlreadyExists: (parts: string) => + `Событие уже есть в календаре (сохранено только что, повторно не создавалось): ${parts}`, + eventSameDay: (title: string, id: number, when: string) => + `«${title}» (id: ${id}) уже есть в календаре в этот день (${when}), поэтому ничего не создано.`, eventUpdated: (parts: string) => `Событие обновлено: ${parts}`, eventDeleted: (title: string, id: number) => `Событие «${title}» (id: ${id}) удалено.`, eventDeclined: (id: number) => `Ты отклонил(а) общее событие (id: ${id}). Оно удалено из твоего календаря.`, diff --git a/src/database/migrations.ts b/src/database/migrations.ts index 7536439b0..b857a2922 100644 --- a/src/database/migrations.ts +++ b/src/database/migrations.ts @@ -1160,4 +1160,12 @@ export const migrations: Migration[] = [ ); }, }, + { + name: '069_chat_history_turn_order', + up(db) { + // A user turn that waited behind an earlier run of its chat is read after what that run wrote, without + // changing its id (the action log and the per-user row map hold ids). NULL: the turn reads at its own id. + db.exec('ALTER TABLE chat_history ADD COLUMN turn_order REAL'); + }, + }, ]; diff --git a/src/database/repositories/chat-history.repository.ts b/src/database/repositories/chat-history.repository.ts index 0b7499f1e..108b973bd 100644 --- a/src/database/repositories/chat-history.repository.ts +++ b/src/database/repositories/chat-history.repository.ts @@ -2,6 +2,16 @@ import type { Database } from 'bun:sqlite'; import type { ChatHistoryMessage } from '../types.ts'; +/** `AND id NOT IN (?, …)` for the given rows; nothing when there are none. */ +function excludedRowsSql(excludeIds: ReadonlySet): string { + return excludeIds.size > 0 ? `AND id NOT IN (${[...excludeIds].map(() => '?').join(', ')})` : ''; +} + +const COLUMNS = 'id, user_id, role, content, chat_id, created_at'; +/** Reading order of a conversation: by time, then by where a turn was placed (its own id unless it was moved after later rows). */ +const ORDER_ASC = 'created_at ASC, COALESCE(turn_order, id) ASC, id ASC'; +const ORDER_DESC = 'created_at DESC, COALESCE(turn_order, id) DESC, id DESC'; + export class ChatHistoryRepository { constructor(private db: Database) {} @@ -12,23 +22,57 @@ export class ChatHistoryRepository { return Number(result.lastInsertRowid); } - getRecentByChat(chatId: number, limit = 10): ChatHistoryMessage[] { + /** + * Puts a user turn after everything saved so far, keeping its id. A message that waited behind an earlier + * run was saved before what that run wrote, so by row order it sits in the middle of that run's output; the + * model must read «earlier request, what was done, this message». The row keeps its id (the action log, + * the per-user map and every context built from it stay valid, and ids stay the order messages arrived + * in); only where it reads moves. Returns false when the row is not a user turn of `userId`. + */ + placeUserTurnAfterLatest(rowId: number, userId: number): boolean { + // "After" means after a bot row of the same conversation: the group's rows (whoever triggered them), or + // the user's private rows. + const result = this.db + .prepare( + `UPDATE chat_history + SET created_at = datetime('now'), turn_order = (SELECT MAX(id) FROM chat_history) + 0.5 + WHERE id = ? AND user_id = ? AND role = 'user' + AND EXISTS ( + SELECT 1 FROM chat_history later + WHERE later.id > chat_history.id AND later.role != 'user' + AND (CASE WHEN chat_history.chat_id IS NULL + THEN later.chat_id IS NULL AND later.user_id = chat_history.user_id + ELSE later.chat_id = chat_history.chat_id END))`, + ) + .run(rowId, userId); + return result.changes > 0; + } + + /** `excludeIds` rows (turns queued behind the reader) are left out BEFORE the limit, so they cost no context. */ + getRecentByChat(chatId: number, limit = 10, excludeIds: ReadonlySet = new Set()): ChatHistoryMessage[] { return this.db .prepare(` - SELECT * FROM ( - SELECT * FROM chat_history - WHERE chat_id = ? - ORDER BY created_at DESC, id DESC + SELECT ${COLUMNS} FROM ( + SELECT ${COLUMNS}, turn_order FROM chat_history + WHERE chat_id = ? ${excludedRowsSql(excludeIds)} + ORDER BY ${ORDER_DESC} LIMIT ? - ) sub ORDER BY created_at ASC, id ASC + ) sub ORDER BY ${ORDER_ASC} `) - .all(chatId, limit) as ChatHistoryMessage[]; + .all(chatId, ...excludeIds, limit) as ChatHistoryMessage[]; } - searchByChat(chatId: number, params: { limit?: number; search?: string }): ChatHistoryMessage[] { - const { limit = 50, search } = params; + searchByChat( + chatId: number, + params: { limit?: number; search?: string; excludeIds?: ReadonlySet }, + ): ChatHistoryMessage[] { + const { limit = 50, search, excludeIds = new Set() } = params; const conditions: string[] = ['chat_id = ?']; const args: (number | string)[] = [chatId]; + if (excludeIds.size > 0) { + conditions.push(`id NOT IN (${[...excludeIds].map(() => '?').join(', ')})`); + args.push(...excludeIds); + } if (search) { conditions.push('content LIKE ?'); @@ -38,34 +82,38 @@ export class ChatHistoryRepository { const where = conditions.join(' AND '); return this.db .prepare( - `SELECT * FROM ( - SELECT * FROM chat_history WHERE ${where} - ORDER BY created_at DESC, id DESC LIMIT ? - ) sub ORDER BY created_at ASC, id ASC`, + `SELECT ${COLUMNS} FROM ( + SELECT ${COLUMNS}, turn_order FROM chat_history WHERE ${where} + ORDER BY ${ORDER_DESC} LIMIT ? + ) sub ORDER BY ${ORDER_ASC}`, ) .all(...args, limit) as ChatHistoryMessage[]; } - getRecent(userId: number, limit = 10): ChatHistoryMessage[] { + getRecent(userId: number, limit = 10, excludeIds: ReadonlySet = new Set()): ChatHistoryMessage[] { return this.db .prepare(` - SELECT * FROM ( - SELECT * FROM chat_history - WHERE user_id = ? AND chat_id IS NULL - ORDER BY created_at DESC, id DESC + SELECT ${COLUMNS} FROM ( + SELECT ${COLUMNS}, turn_order FROM chat_history + WHERE user_id = ? AND chat_id IS NULL ${excludedRowsSql(excludeIds)} + ORDER BY ${ORDER_DESC} LIMIT ? - ) sub ORDER BY created_at ASC, id ASC + ) sub ORDER BY ${ORDER_ASC} `) - .all(userId, limit) as ChatHistoryMessage[]; + .all(userId, ...excludeIds, limit) as ChatHistoryMessage[]; } search( userId: number, - params: { limit?: number; search?: string; before?: string; after?: string }, + params: { limit?: number; search?: string; before?: string; after?: string; excludeIds?: ReadonlySet }, ): ChatHistoryMessage[] { - const { limit = 50, search, before, after } = params; + const { limit = 50, search, before, after, excludeIds = new Set() } = params; const conditions: string[] = ['user_id = ?']; const args: (number | string)[] = [userId]; + if (excludeIds.size > 0) { + conditions.push(`id NOT IN (${[...excludeIds].map(() => '?').join(', ')})`); + args.push(...excludeIds); + } if (search) { conditions.push('content LIKE ?'); @@ -83,10 +131,10 @@ export class ChatHistoryRepository { const where = conditions.join(' AND '); return this.db .prepare( - `SELECT * FROM ( - SELECT * FROM chat_history WHERE ${where} - ORDER BY created_at DESC, id DESC LIMIT ? - ) sub ORDER BY created_at ASC, id ASC`, + `SELECT ${COLUMNS} FROM ( + SELECT ${COLUMNS}, turn_order FROM chat_history WHERE ${where} + ORDER BY ${ORDER_DESC} LIMIT ? + ) sub ORDER BY ${ORDER_ASC}`, ) .all(...args, limit) as ChatHistoryMessage[]; } diff --git a/src/database/repositories/event.repository.ts b/src/database/repositories/event.repository.ts index 3fbdb377a..7d7256206 100644 --- a/src/database/repositories/event.repository.ts +++ b/src/database/repositories/event.repository.ts @@ -1,5 +1,6 @@ // src/database/repositories/event.repository.ts import type { Database, SQLQueryBindings, Statement } from 'bun:sqlite'; +import { allDayDates } from '../../utils/date.ts'; import { assertValidEventTimestamps } from '../../utils/event-timestamps.ts'; import type { CalendarEvent, CreateEventData, UpdateEventData } from '../types.ts'; @@ -78,6 +79,59 @@ function takeTitleMatches( } } +export type CreateOutcome = 'created' | 'duplicate' | 'same_day'; + +/** The calendar day (YYYY-MM-DD) an instant falls on in `timezone`. */ +function localDay(instant: string, timezone: string): string { + try { + return new Intl.DateTimeFormat('en-CA', { + timeZone: timezone, + year: 'numeric', + month: '2-digit', + day: '2-digit', + }).format(new Date(instant)); + } catch { + return instant.slice(0, 10); + } +} + +/** Timed events are the same span when start and end are the same instants, however the offset is spelled. */ +function sameInstants(data: CreateEventData) { + const start = Date.parse(data.start_at); + const end = data.end_at ? Date.parse(data.end_at) : null; + return (c: { start_at: string; end_at: string | null }) => + Date.parse(c.start_at) === start && (c.end_at ? Date.parse(c.end_at) : null) === end; +} + +/** What the caller stated beyond title and time (description, location, reminders) must not differ from the saved row. */ +function sameDetails( + data: CreateEventData, + saved: { description: string | null; location: string | null; reminder_overrides: string | null }, +): boolean { + const same = (stated: string | undefined, have: string | null) => + stated === undefined || stated.trim() === (have ?? '').trim(); + return ( + same(data.description, saved.description) && + same(data.location, saved.location) && + (data.reminder_minutes === undefined || + JSON.stringify(data.reminder_minutes) === JSON.stringify(parseMinutes(saved.reminder_overrides))) + ); +} + +function parseMinutes(raw: string | null): unknown { + if (raw === null) return null; + try { + return JSON.parse(raw); + } catch { + return raw; + } +} + +/** Titles are the same event's when they differ only in surrounding or repeated whitespace, case or Unicode form. */ +function normalizedTitle(title: string): string { + return title.normalize('NFKC').trim().replace(/\s+/g, ' ').toLowerCase(); +} + export const EVENT_UPDATE_FIELDS = [ 'title', 'description', @@ -204,13 +258,126 @@ export class EventRepository { data.event_type ?? null, data.reminder_minutes === undefined ? null : JSON.stringify(data.reminder_minutes), ); - const id = Number(result.lastInsertRowid); - if (data.owner_type === 'group' && data.group_id != null) { - return this.findByIdInGroup(id, data.group_id)!; - } + return this.findCreated(Number(result.lastInsertRowid), data); + } + + /** The row a create just wrote or matched, read back the way the creator's scope sees it. */ + private findCreated(id: number, data: CreateEventData): CalendarEvent { + if (data.owner_type === 'group' && data.group_id != null) return this.findByIdInGroup(id, data.group_id)!; return this.findById(id, data.user_id)!; } + /** Creates the event and runs `onCreated` (its reminders) in one transaction: if either fails, neither is kept. */ + createWithEffects(data: CreateEventData, onCreated: (event: CalendarEvent) => void): CalendarEvent { + return this.db + .transaction((): CalendarEvent => { + const event = this.create(data); + onCreated(event); + return event; + }) + .immediate(); + } + + /** + * Creates the event unless the calendar already holds it. Candidates are the live, non-recurring events + * of the same calendar (the user's own primary-calendar events, or the same group) with the same title + * (trimmed, whitespace collapsed, case folded) and the same all-day flag: + * - `duplicate`: same start and end (all-day: the same dates) and the same details the caller stated, + * created within `windowMs`. That is what a repeated message, a retry after a lost reply or a second + * overlapping run produces; the existing row comes back and nothing is written. + * - `same_day`: any other candidate on the same local day (all-day: the same first date), whatever its + * age: a second copy, or the old half of a move done by creating (#520). The caller decides whether the + * user really wants it; nothing is written. + * - `created`: otherwise. `onCreated` runs in the same transaction as the insert, so an event never + * persists without the effects that must accompany it. + * The check and the insert are one immediate transaction, so two writers (connections, or a process + * restarted in between) cannot both miss each other's row. A recurring event is always created: a series + * never stands in for another. + */ + createUnlessDuplicate( + data: CreateEventData, + windowMs: number, + onCreated?: (event: CalendarEvent) => void, + ): { event: CalendarEvent; outcome: CreateOutcome } { + return this.db + .transaction((): { event: CalendarEvent; outcome: CreateOutcome } => { + const match = data.recurrence_rule ? null : this.findSameEvent(data, windowMs); + if (match) return { event: this.findCreated(match.id, data), outcome: match.outcome }; + const event = this.create(data); + onCreated?.(event); + return { event, outcome: 'created' }; + }) + .immediate(); + } + + private findSameEvent( + data: CreateEventData, + windowMs: number, + ): { id: number; outcome: Exclude } | null { + assertValidEventTimestamps(data); + const isGroup = data.owner_type === 'group' && data.group_id != null; + const candidates = this.db + .prepare( + `SELECT id, title, description, location, start_at, end_at, timezone, reminder_overrides, + created_at >= datetime('now', ?) AS recent + FROM events + WHERE is_deleted = 0 AND is_cancelled = 0 + AND recurrence_rule IS NULL AND parent_event_id IS NULL + AND ${ + isGroup + ? "owner_type = 'group' AND group_id = ?" + : `owner_type = 'user' AND user_id = ? + AND (google_calendar_id IS NULL OR google_calendar_id = 'primary' + OR google_calendar_id IN (SELECT google_calendar_id FROM google_calendars WHERE user_id = ? AND is_primary = 1))` + } + AND all_day = ? + AND ABS(julianday(start_at) - julianday(?)) < 3 + ORDER BY id ASC`, + ) + .all( + `-${Math.ceil(windowMs / 1000)} seconds`, + ...(isGroup ? [data.group_id!] : [data.user_id, data.user_id]), + data.all_day ? 1 : 0, + data.start_at, + ) as { + id: number; + title: string; + description: string | null; + location: string | null; + start_at: string; + end_at: string | null; + timezone: string; + reminder_overrides: string | null; + recent: number; + }[]; + const wanted = normalizedTitle(data.title); + const sameTitle = candidates.filter((c) => normalizedTitle(c.title) === wanted); + if (sameTitle.length === 0) return null; + + const sameSpan = data.all_day ? this.sameAllDaySpan(data) : sameInstants(data); + const sameDay = data.all_day + ? (c: { start_at: string; timezone: string }) => + allDayDates(c.start_at, null, c.timezone).first === allDayDates(data.start_at, null, data.timezone).first + : (c: { start_at: string }) => localDay(c.start_at, data.timezone) === localDay(data.start_at, data.timezone); + const duplicate = sameTitle.find((c) => c.recent === 1 && sameSpan(c) && sameDetails(data, c)); + if (duplicate) return { id: duplicate.id, outcome: 'duplicate' }; + const sameDayEvent = sameTitle.find(sameDay); + return sameDayEvent ? { id: sameDayEvent.id, outcome: 'same_day' } : null; + } + + /** + * All-day values are floating dates (a bare date, UTC midnight or local midnight in some zone): compare the + * dates they mean, each read in the zone it was saved with (a group event saved from Belgrade is read the + * same way by a member in Los Angeles). + */ + private sameAllDaySpan(data: CreateEventData) { + const wanted = allDayDates(data.start_at, data.end_at ?? null, data.timezone); + return (c: { start_at: string; end_at: string | null; timezone: string }) => { + const have = allDayDates(c.start_at, c.end_at, c.timezone); + return have.first === wanted.first && have.endExclusive === wanted.endExclusive; + }; + } + /** Load event by ID without ownership checks. For internal service use only (sync workers, etc.). */ findByIdUnfiltered( id: number, diff --git a/src/services/ai/agent.ts b/src/services/ai/agent.ts index 85b98b11f..26bbfc9d2 100644 --- a/src/services/ai/agent.ts +++ b/src/services/ai/agent.ts @@ -17,17 +17,20 @@ import { regroupToolCallBlocks, sanitizeMessages, } from './agent/message-sanitizer.ts'; +import { announceExpiredQueueLease, prepareQueuedTurn, showTypingWhileQueued, turnIds } from './agent/queued-turn.ts'; import { settleFinalResponse } from './agent/response-validation.ts'; import { type RetryOutcome, storeRetryAfterFailure } from './agent/retry-store.ts'; import { runRoundLoop } from './agent/round-loop.ts'; import { toolDefinitionsFor } from './agent/round-support.ts'; -import { AGENT_DRAIN_SETTLE_MS, RETRY_STORE_TIMEOUT_MS, TIMEOUT_MS } from './agent/run-limits.ts'; +import { AGENT_DRAIN_SETTLE_MS, CHAT_RUN_GRACE_MS, RETRY_STORE_TIMEOUT_MS, TIMEOUT_MS } from './agent/run-limits.ts'; import type { AgentRunResult, AgentToolCallRecord, AgentToolResultRecord } from './agent/run-types.ts'; import { toolCallKey, toolResultContent } from './agent/tool-call-key.ts'; import type { ToolRunLedger } from './agent/tool-round.ts'; import { finishTurn } from './agent/turn-finish.ts'; import { createTurnState, type RunFailure, type TurnEnv, type TurnState } from './agent/turn-state.ts'; import { createTurnWriter, senderForTurn } from './agent/turn-writer.ts'; +import { ChatRunGate, type ChatRunSlot, type ChatTurnReservation } from './chat-run-gate.ts'; +import { currentChatTurn } from './chat-turn-context.ts'; import { resolveTurnDayReferences } from './day-reference-guard.ts'; import type { AiDebugLogger, AiDebugRunContext } from './debug-logger.ts'; import type { HistorySummarizer } from './history-summarizer.ts'; @@ -67,11 +70,19 @@ export class CalendarBotAgent { private readonly shutdown = new AbortController(); /** Runs not yet settled, including their debug-log flush — what a shutdown drain waits for. */ private readonly inFlight = new Set>(); + /** One run at a time per chat; queued runs are in `inFlight` too, so a drain waits for them. */ + private readonly chatRuns: ChatRunGate; constructor( config: AgentConfig, sender: TelegramSender, - opts?: { streamImpl?: typeof aiStreamRound; requestTimeoutMs?: number; retryStoreTimeoutMs?: number }, + opts?: { + streamImpl?: typeof aiStreamRound; + requestTimeoutMs?: number; + retryStoreTimeoutMs?: number; + /** Replaces the default gate; tests use it to bound a hung run without real time. */ + chatRunGate?: ChatRunGate; + }, ) { this.toolSchemaMode = config.toolSchemaMode ?? 'full'; this.sender = sender; @@ -80,6 +91,7 @@ export class CalendarBotAgent { this.summarizer = config.summarizer; this.requestTimeoutMs = opts?.requestTimeoutMs ?? TIMEOUT_MS; this.retryStoreTimeoutMs = opts?.retryStoreTimeoutMs ?? RETRY_STORE_TIMEOUT_MS; + this.chatRuns = opts?.chatRunGate ?? new ChatRunGate({ leaseMs: this.requestTimeoutMs + CHAT_RUN_GRACE_MS }); if (!Number.isSafeInteger(this.requestTimeoutMs) || this.requestTimeoutMs < 1 || this.requestTimeoutMs > TIMEOUT_MS) throw new Error(`requestTimeoutMs must be between 1 and ${TIMEOUT_MS}`); } @@ -170,22 +182,45 @@ export class CalendarBotAgent { // Debug logging is evidence, not a dependency: the turn still runs without it. aiLogger.warn({ err, userId: ctx.user.telegram_id }, 'AI debug log unavailable for this turn'); } - const turn = this.runTurn(ctx, dbg).finally(() => { - this.inFlight.delete(turn); - // Every exit leaves the turn's evidence, including one that threw after its failure path. - dbg?.flush(); - }); + // One run at a time per chat: a message that arrives while an earlier run is still working waits for it + // and is then handled on the history that run wrote. The wait ends at shutdown (the run then fails fast). + const turn = this.chatRuns + .run( + ctx.chatId, + { + reservation: currentChatTurn(), + abort: this.shutdown.signal, + whileWaiting: () => showTypingWhileQueued(ctx, this.sender), + }, + (slot) => this.runTurn(ctx, dbg, slot), + ) + .finally(() => { + this.inFlight.delete(turn); + // Every exit leaves the turn's evidence, including one that threw after its failure path. + dbg?.flush(); + }); this.inFlight.add(turn); return turn; } + /** + * Holds the next place in `chatId`'s line for the user message just saved as `rowId`, before its handler can + * wait on anything: the run that answers this row takes the place over, and later messages queue behind it + * whatever order their handlers reach `run`. The caller releases it when the update's handling ends. + */ + reserveChatTurn(chatId: number, rowId: number): ChatTurnReservation { + return this.chatRuns.reserve(chatId, rowId); + } + /** * Loads what the model is asked to continue from: the history with this turn's day references * resolved, condensed by the summarizer, behind the system prompt for the request's tool mode. */ private async startConversation(env: TurnEnv, state: TurnState, summaryStream: typeof aiStreamRound): Promise { const { ctx, dbg, requestSignal } = env; - const history = ctx.chatHistory.getRecent(ctx.user.telegram_id, 30); + // Turns queued behind this run are left out in the query, before the limit, and resolved + // now: a message that joined the queue while this run was starting is excluded too. + const history = ctx.chatHistory.getRecent(ctx.user.telegram_id, 30, ctx.deferredTurnRowIds?.()); // A bare "Да" answers the ask_user of the conversation it was saved to: the group's own. const turnHistory = relevantHistoryRows(ctx, history, 30); ctx.dayReferences = resolveTurnDayReferences(ctx.messageText, turnHistory, new Date(), ctx.user.timezone); @@ -206,7 +241,9 @@ export class CalendarBotAgent { state.currentMessages = [{ role: 'system', content: activePrompt }, ...historyMessages]; } - private async runTurn(ctx: AgentContext, dbg: AiDebugRunContext | null): Promise { + private async runTurn(ctx: AgentContext, dbg: AiDebugRunContext | null, slot: ChatRunSlot): Promise { + // The request budget below counts from the moment this turn is given the chat, not from arrival. + prepareQueuedTurn(ctx, slot); const startTime = Date.now(); const requestSignal = AbortSignal.any([AbortSignal.timeout(this.requestTimeoutMs), this.shutdown.signal]); const requestMetrics = new AgentRequestMetrics(); @@ -216,6 +253,7 @@ export class CalendarBotAgent { requestId, userId: ctx.user.telegram_id, chatId: ctx.chatId, + ...turnIds(ctx), supplementMode: !!ctx.supplementMode, msg: ctx.messageText.slice(0, 100), }, @@ -269,6 +307,9 @@ export class CalendarBotAgent { const failure: RunFailure = { failed: false, error: undefined, retry: 'not_stored' }; try { + // Inside the try and under the request signal: a stalled notice fails this turn into the normal failure + // path (stored retry, user notice) instead of outliving the shutdown drain. + await waitForAbort(() => announceExpiredQueueLease(ctx, slot, this.sender), requestSignal); await writer.init(); await this.startConversation(env, state, summaryStream); await runRoundLoop(env, state); diff --git a/src/services/ai/agent/conversation.ts b/src/services/ai/agent/conversation.ts index 534f05f2c..221d5d875 100644 --- a/src/services/ai/agent/conversation.ts +++ b/src/services/ai/agent/conversation.ts @@ -7,13 +7,20 @@ import type { AgentContext } from '../types.ts'; import { parseHistoryRow, tagSender } from './history-rows.ts'; import { isToolMessage, type MessageParam } from './message-sanitizer.ts'; -/** In a group the conversation is the group's own, whoever sent each row. */ +/** + * The rows the model reads. In a group the conversation is the group's own, whoever sent each row. + * Turns queued behind this run in its chat (ctx.deferredTurnRowIds) are left out by the query itself, before + * the limit: they arrived after the message being answered and get their own run once this one has replied. A private chat's `history` was loaded that way + * by the caller. + */ export function relevantHistoryRows( ctx: AgentContext, history: ChatHistoryMessage[], limit: number, ): ChatHistoryMessage[] { - return ctx.isGroup && ctx.groupChatId ? ctx.chatHistory.getRecentByChat(ctx.groupChatId, limit) : history; + return ctx.isGroup && ctx.groupChatId + ? ctx.chatHistory.getRecentByChat(ctx.groupChatId, limit, ctx.deferredTurnRowIds?.()) + : history; } async function condenseToolResult( diff --git a/src/services/ai/agent/queued-turn.ts b/src/services/ai/agent/queued-turn.ts new file mode 100644 index 000000000..8ed919d12 --- /dev/null +++ b/src/services/ai/agent/queued-turn.ts @@ -0,0 +1,98 @@ +// What a run does when its turn in the chat comes after another run's: it was built when its message +// arrived, so the snapshot it carries is older than the run it waited for, and its user row sits +// before everything that run wrote. This brings the snapshot up to date and reads the message after +// that output (same row, same id), before the model reads anything. +import { t, toLang } from '../../../config/constants.ts'; +import type { EventOccurrence } from '../../../database/types.ts'; +import { logger } from '../../../utils/logger.ts'; +import type { ChatRunSlot } from '../chat-run-gate.ts'; +import type { AgentContext, TelegramSender } from '../types.ts'; + +const queuedTurnLogger = logger.child({ module: 'ai-agent' }); + +const SCHEDULE_WINDOW_MS = 14 * 24 * 60 * 60 * 1000; + +/** The Telegram ids that tell a redelivered update (same ids) from a second message (new ids), for log lines. */ +export function turnIds(ctx: Pick): { + updateId?: number; + messageId?: number; +} { + return { + ...(ctx.updateId !== undefined && { updateId: ctx.updateId }), + ...(ctx.incomingMessageId !== undefined && { messageId: ctx.incomingMessageId }), + }; +} + +const QUEUE_TYPING_INTERVAL_MS = 4_000; + +/** + * Keeps the chat's typing indicator alive while a message waits behind an earlier run, so the user sees + * that it was received. Returns what stops it. A supplement is optional text and shows nothing. + */ +export function showTypingWhileQueued(ctx: AgentContext, sender: TelegramSender): () => void { + if (ctx.supplementMode || !sender.sendChatAction) return () => {}; + const send = () => void sender.sendChatAction?.(ctx.chatId, 'typing').catch(() => {}); + send(); + const timer = setInterval(send, QUEUE_TYPING_INTERVAL_MS); + return () => clearInterval(timer); +} + +/** + * The wait for the earlier run ended because it held the chat past its lease, not because it finished: + * this run goes on beside it, and the user is told so instead of being left to guess (best effort). + */ +export async function announceExpiredQueueLease( + ctx: AgentContext, + slot: ChatRunSlot, + sender: TelegramSender, +): Promise { + // The notice says the user's message is being handled: a run without a message of theirs (scheduled) has none. + if (!slot.leaseExpired || ctx.supplementMode || slot.ownRowId === undefined) return; + try { + await sender.sendMessage(ctx.chatId, t(toLang(ctx.user.language)).agent_queue_lease_expired); + } catch (err) { + queuedTurnLogger.warn( + { err, userId: ctx.user.telegram_id }, + 'Could not tell the user their message left the queue', + ); + } +} + +/** Events two weeks either side of now: the nearby schedule the system prompt shows the model. */ +export function scheduleWindowAround( + eventService: AgentContext['eventService'], + userId: number, + now = new Date(), +): EventOccurrence[] { + const start = new Date(now.getTime() - SCHEDULE_WINDOW_MS).toISOString(); + const end = new Date(now.getTime() + SCHEDULE_WINDOW_MS).toISOString(); + return eventService.getEventsInRange(userId, start, end); +} + +/** + * Sets up a run that has just been given its turn. Always: the turns queued behind it (reserved at the moment + * they were saved, so a message whose handler is still busy elsewhere counts too) are kept out of its history, + * and the message it answers is read after what earlier runs wrote (same row, same id). After a wait: the user + * and the schedule window are read again (the earlier run may have changed either). A step that fails leaves + * the run on what it had; this is a refinement, never a reason to fail the run. + */ +export function prepareQueuedTurn(ctx: AgentContext, slot: ChatRunSlot): void { + ctx.deferredTurnRowIds = () => slot.queuedTurnRowIds(); + const userId = ctx.user.telegram_id; + if (slot.waited) { + try { + ctx.user = ctx.userRepo.findByTelegramId(userId) ?? ctx.user; + if (ctx.recentEventsWindow) ctx.recentEventsWindow = scheduleWindowAround(ctx.eventService, userId); + } catch (err) { + queuedTurnLogger.warn({ err, userId }, 'Could not refresh the context of a run that waited for its turn'); + } + } + // Only a run's own message is placed: a scheduled or synthetic run borrows the user's latest row for the + // action log, and a supplement answers a message the main run already placed. + if (slot.ownRowId === undefined || ctx.supplementMode) return; + try { + ctx.chatHistory.placeUserTurnAfterLatest(slot.ownRowId, userId); + } catch (err) { + queuedTurnLogger.warn({ err, userId }, 'Could not place the message of a run behind the earlier turn'); + } +} diff --git a/src/services/ai/agent/run-limits.ts b/src/services/ai/agent/run-limits.ts index 125701cd9..0451646f7 100644 --- a/src/services/ai/agent/run-limits.ts +++ b/src/services/ai/agent/run-limits.ts @@ -2,6 +2,11 @@ export const MAX_ROUNDS = 15; export const TIMEOUT_MS = 90_000; +/** + * How long past its request budget a run may still hold its chat (its failure notice, retry store and + * history write come after the budget) before the run queued behind it stops waiting for it. + */ +export const CHAT_RUN_GRACE_MS = 30_000; /** * How long a failed turn waits to learn whether its retry was stored before it answers anyway. * Kept inside the first shutdown drain below, so a hung store never makes that drain abandon the diff --git a/src/services/ai/agent/turn-finish.ts b/src/services/ai/agent/turn-finish.ts index 2fd345dbe..63eb2a093 100644 --- a/src/services/ai/agent/turn-finish.ts +++ b/src/services/ai/agent/turn-finish.ts @@ -9,6 +9,7 @@ import type { TelegramStreamWriter } from '../telegram-stream.ts'; import type { AgentContext } from '../types.ts'; import { aiFailureNotices } from './failure-notices.ts'; import type { MessageParam } from './message-sanitizer.ts'; +import { turnIds } from './queued-turn.ts'; import { correctUtcClockTimesInReply } from './reply-correction.ts'; import type { RetryOutcome } from './retry-store.ts'; import { isSkipText } from './round-support.ts'; @@ -211,6 +212,7 @@ function recordTurnEnd(env: TurnEnv, state: TurnState, deps: FinishDeps): string requestId: env.requestId, userId: ctx.user.telegram_id, chatId: ctx.chatId, + ...turnIds(ctx), toolCount: ledger.allToolCalls.length, supplementMode: !!ctx.supplementMode, termination: state.termination, diff --git a/src/services/ai/chat-run-gate.ts b/src/services/ai/chat-run-gate.ts new file mode 100644 index 000000000..98bf60c41 --- /dev/null +++ b/src/services/ai/chat-run-gate.ts @@ -0,0 +1,330 @@ +// src/services/ai/chat-run-gate.ts +// +// One agent run at a time per chat, in arrival order; different chats stay parallel. +// +// GramIO hands every Telegram update to its handler without waiting for the previous one (webhook +// and long polling alike), so two messages of one chat used to run two agents on overlapping +// history. On 2026-11-01 two quick confirmations did exactly that and every event was created +// twice. A run now starts only after the runs ahead of it in the same chat have finished and +// written their turn, so it reads the history they produced. +// +// Arrival order is the order the messages were SAVED in, not the order their handlers reach the agent +// (a handler may wait on a scene or retry lookup first). So the place in line is reserved at the moment +// the message is saved (`reserve`), and the run that answers that message takes the place over (`run` +// with that reservation; see chat-turn-context.ts for how a run finds it). A reservation the handler never turns into a run (an intent answered it, a +// command, a failed update) is released when the update ends or after `reservationMs`. +// +// A run that never finishes must not freeze its chat: the run behind stops waiting once the run +// ahead has held the chat for `leaseMs` (the request budget plus a grace), says so in the log and +// goes on. Shutdown also ends the wait, so queued runs take their normal fail-fast path. + +import { logger } from '../../utils/logger.ts'; + +const gateLogger = logger.child({ module: 'ai-chat-gate' }); + +/** What a run learns when its turn in the chat comes. */ +export interface ChatRunSlot { + /** True when the run had to wait behind an earlier run of the same chat. */ + readonly waited: boolean; + /** + * True when the run starts beside an earlier run of the chat that is still unfinished because it held the + * chat past its lease and no earlier run has reported that yet (a hung run is announced once). Never true + * when the wait ended at shutdown. + */ + readonly leaseExpired: boolean; + readonly waitedMs: number; + /** + * The chat_history row of the message this run answers, when the run took over the place reserved for + * that message; undefined for a run that has no message of its own (scheduled, synthetic). + */ + readonly ownRowId: number | undefined; + /** + * chat_history rows of the turns queued behind this run right now (reserved or waiting). They arrived + * after this run's message, so its model must not read them as already said. + */ + queuedTurnRowIds(): ReadonlySet; +} + +export interface ChatRunGateOptions { + /** How long a run holds its chat before the run behind it stops waiting for it. */ + leaseMs: number; + /** How long a reserved place may stay unclaimed before it is given up. Default 60 s. */ + reservationMs?: number; + now?: () => number; + /** Starts a timer and returns its cancel function; replaceable so tests need no real time. */ + schedule?: (callback: () => void, ms: number) => () => void; +} + +export interface ChatTurn { + /** + * The place reserved for the message this run answers, when there is one: it is taken over. Without it the + * run joins the end of the line and has no message of its own (scheduled, synthetic). + */ + reservation?: ChatTurnReservation; + /** Aborts the wait: used at shutdown, when every queued run should fail fast rather than queue. */ + abort?: AbortSignal; + /** Called when this run has to wait; the function it returns is called when the wait ends (typing, say). */ + whileWaiting?: () => () => void; +} + +/** A reserved place in a chat's line. `release` gives it up; harmless after the run took it over. */ +export interface ChatTurnReservation { + readonly chatId: number; + /** The chat_history row of the message the place is held for. */ + readonly rowId: number; + release(): void; +} + +interface QueuedRun { + /** The message this entry stands for; undefined for a run without a message of its own. */ + rowId: number | undefined; + /** True from `reserve` until a run takes the place over. */ + reserved: boolean; + cancelExpiry: (() => void) | undefined; + started: PromiseWithResolvers; + /** Set before `started` settles. */ + startedAt: number; + finished: PromiseWithResolvers; + settled: boolean; + /** True once a later run has reported giving up on this entry: a hung run is announced once, not per message. */ + leaseAnnounced: boolean; +} + +function newEntry(rowId: number | undefined, reserved: boolean): QueuedRun { + return { + rowId, + reserved, + cancelExpiry: undefined, + started: Promise.withResolvers(), + startedAt: 0, + finished: Promise.withResolvers(), + settled: false, + leaseAnnounced: false, + }; +} + +function scheduleWithTimeout(callback: () => void, ms: number): () => void { + const timer = setTimeout(callback, ms); + return () => clearTimeout(timer); +} + +const DEFAULT_RESERVATION_MS = 60_000; + +export class ChatRunGate { + private readonly leaseMs: number; + private readonly reservationMs: number; + private readonly now: () => number; + private readonly schedule: (callback: () => void, ms: number) => () => void; + /** Per chat: the places not yet finished, oldest first. A chat with none has no entry. */ + private readonly lines = new Map(); + private readonly reservedEntries = new WeakMap(); + /** Reservations a run has claimed the message of, so a second run of the same update gets no row of its own. */ + private readonly claimedReservations = new WeakSet(); + + constructor(options: ChatRunGateOptions) { + this.leaseMs = options.leaseMs; + this.reservationMs = options.reservationMs ?? DEFAULT_RESERVATION_MS; + this.now = options.now ?? Date.now; + this.schedule = options.schedule ?? scheduleWithTimeout; + } + + /** Chats that have a run in flight or queued; zero once everything has finished. */ + get busyChats(): number { + return this.lines.size; + } + + /** True while a run of the chat is in flight or queued: a run arriving now would have to wait. */ + isBusy(chatId: number): boolean { + return this.lines.has(chatId); + } + + /** + * Holds the next place in the chat's line for the message saved as `rowId`, synchronously, before the + * message's handler can wait on anything. Runs behind this place wait for it; the run for the same + * row takes it over. + */ + reserve(chatId: number, rowId: number): ChatTurnReservation { + const line = this.lines.get(chatId) ?? []; + this.lines.set(chatId, line); + const entry = newEntry(rowId, true); + line.push(entry); + entry.cancelExpiry = this.schedule(() => { + if (!entry.reserved) return; + gateLogger.warn( + { chatId, rowId, reservationMs: this.reservationMs }, + 'A reserved place in the chat was never used', + ); + this.release(chatId, entry); + }, this.reservationMs); + const reservation: ChatTurnReservation = { + chatId, + rowId, + release: () => { + if (entry.reserved) this.release(chatId, entry); + }, + }; + this.reservedEntries.set(reservation, entry); + return reservation; + } + + /** + * Runs `task` once every earlier run of `chatId` has finished (or held the chat past its lease). + * With nothing ahead the task starts synchronously, so an idle chat sees no change at all. + */ + run(chatId: number, turn: ChatTurn, task: (slot: ChatRunSlot) => Promise): Promise { + const line = this.lines.get(chatId) ?? []; + this.lines.set(chatId, line); + const held = + turn.reservation && turn.reservation.chatId === chatId ? this.reservedEntries.get(turn.reservation) : undefined; + const reserved = held?.reserved ? held : undefined; + const entry = reserved ?? newEntry(undefined, false); + if (reserved) { + reserved.reserved = false; + reserved.cancelExpiry?.(); + } else line.push(entry); + // The message stays the run's own even when its place expired before the run got here: its row must + // still be placed after what the runs in between wrote. + const own = + turn.reservation && turn.reservation.chatId === chatId && !this.claimedReservations.has(turn.reservation) + ? turn.reservation + : undefined; + if (own) this.claimedReservations.add(own); + const ownRowId = own?.rowId; + const hasAhead = line.indexOf(entry) > 0; + const enqueuedAt = this.now(); + + const start = (leaseExpired: boolean): Promise => { + entry.startedAt = this.now(); + entry.started.resolve(); + const slot: ChatRunSlot = { + waited: hasAhead, + leaseExpired, + waitedMs: entry.startedAt - enqueuedAt, + ownRowId, + queuedTurnRowIds: () => this.rowsQueuedBehind(chatId, entry), + }; + if (slot.waited) { + gateLogger.info( + { chatId, waitedMs: slot.waitedMs, queuedBehind: this.queuedBehind(chatId, entry).length }, + 'Agent run started after waiting for the previous run in this chat', + ); + } + let running: Promise; + try { + running = task(slot); + } catch (err) { + running = Promise.reject(err); + } + return running.finally(() => this.release(chatId, entry)); + }; + + // With nothing ahead the task starts on the caller's stack, as an ungated run always did. + if (!hasAhead) return start(false); + let stopWaitingSignal: (() => void) | undefined; + try { + stopWaitingSignal = turn.whileWaiting?.(); + } catch (err) { + // A typing indicator is a courtesy: it must not stop the run or leave its place in line unused. + gateLogger.warn({ err, chatId }, 'Could not show that a message is waiting for its turn'); + } + return this.untilTurn(chatId, entry, turn.abort, ownRowId !== undefined) + .finally(() => stopWaitingSignal?.()) + .then(start); + } + + private queuedBehind(chatId: number, entry: QueuedRun): QueuedRun[] { + const line = this.lines.get(chatId) ?? []; + const index = line.indexOf(entry); + return index === -1 ? [] : line.slice(index + 1); + } + + private rowsQueuedBehind(chatId: number, entry: QueuedRun): ReadonlySet { + const rows = new Set(); + for (const queued of this.queuedBehind(chatId, entry)) { + if (queued.rowId !== undefined && queued.rowId !== entry.rowId) rows.add(queued.rowId); + } + return rows; + } + + private release(chatId: number, entry: QueuedRun): void { + entry.settled = true; + entry.reserved = false; + entry.cancelExpiry?.(); + entry.finished.resolve(); + const line = this.lines.get(chatId); + if (!line) return; + const index = line.indexOf(entry); + if (index !== -1) line.splice(index, 1); + if (line.length === 0) this.lines.delete(chatId); + } + + /** The unfinished place nearest ahead of `entry` that has not already been given up on. */ + private nearestAhead(chatId: number, entry: QueuedRun, givenUp: ReadonlySet): QueuedRun | undefined { + const line = this.lines.get(chatId) ?? []; + for (let index = line.indexOf(entry) - 1; index >= 0; index--) { + const candidate = line[index]!; + if (!candidate.settled && !givenUp.has(candidate)) return candidate; + } + return undefined; + } + + /** + * Resolves when every place ahead of `entry` has finished, has held the chat for the lease, or shutdown + * began; true when the run starts beside an unfinished earlier run (the lease), false otherwise. A place + * ahead may itself still be queued or reserved, so its lease starts counting once it began. + */ + private async untilTurn( + chatId: number, + entry: QueuedRun, + abort: AbortSignal | undefined, + announces: boolean, + ): Promise { + const givenUp = new Set(); + for (;;) { + if (abort?.aborted) return false; + const ahead = this.nearestAhead(chatId, entry, givenUp); + if (!ahead) { + const fresh = [...givenUp].filter((given) => !given.settled && !given.leaseAnnounced); + // A run with no message of its own cannot show the notice, so it must not use it up for the next. + if (announces) for (const given of fresh) given.leaseAnnounced = true; + return fresh.length > 0; + } + if (await this.waitFor(chatId, ahead, abort)) givenUp.add(ahead); + else if (abort?.aborted) return false; + } + } + + /** Waits for one place: false when it finished or shutdown began, true when its lease ran out first. */ + private async waitFor(chatId: number, ahead: QueuedRun, abort: AbortSignal | undefined): Promise { + const done = Promise.withResolvers(); + let cancelLease: (() => void) | undefined; + let over = false; + const onAbort = () => done.resolve(); + if (abort?.aborted) done.resolve(); + else abort?.addEventListener('abort', onAbort, { once: true }); + void ahead.finished.promise.then(() => done.resolve()); + void ahead.started.promise.then(() => { + if (over) return; + const remaining = ahead.startedAt + this.leaseMs - this.now(); + if (remaining <= 0) done.resolve(); + else cancelLease = this.schedule(() => done.resolve(), remaining); + }); + await done.promise; + over = true; + cancelLease?.(); + abort?.removeEventListener('abort', onAbort); + const leaseRanOut = !ahead.settled && abort?.aborted !== true; + if (!ahead.settled) { + gateLogger.warn( + { + chatId, + heldMs: ahead.startedAt === 0 ? 0 : this.now() - ahead.startedAt, + leaseMs: this.leaseMs, + shuttingDown: abort?.aborted === true, + }, + 'Not waiting any longer for the earlier agent run in this chat', + ); + } + return leaseRanOut; + } +} diff --git a/src/services/ai/chat-turn-context.ts b/src/services/ai/chat-turn-context.ts new file mode 100644 index 000000000..34afa661a --- /dev/null +++ b/src/services/ai/chat-turn-context.ts @@ -0,0 +1,19 @@ +// src/services/ai/chat-turn-context.ts +// +// Which reserved place in its chat's line the code running right now is answering. The chat-logging +// middleware reserves the place when it saves a user message and runs the rest of that update's handling +// inside `withChatTurn`; whatever agent run the update starts (the text pipeline, /cal, a button) finds it with +// `currentChatTurn` and takes it over, with no id passed along. A run started elsewhere (scheduled, retry, +// supplement of another update) finds none and joins the end of the line without a message of its own. +import { AsyncLocalStorage } from 'node:async_hooks'; +import type { ChatTurnReservation } from './chat-run-gate.ts'; + +const current = new AsyncLocalStorage(); + +export function withChatTurn(reservation: ChatTurnReservation, work: () => T): T { + return current.run(reservation, work); +} + +export function currentChatTurn(): ChatTurnReservation | undefined { + return current.getStore(); +} diff --git a/src/services/ai/tool-handlers/events/create.ts b/src/services/ai/tool-handlers/events/create.ts index 641be4caa..d6c0e5ddc 100644 --- a/src/services/ai/tool-handlers/events/create.ts +++ b/src/services/ai/tool-handlers/events/create.ts @@ -1,11 +1,12 @@ // create_event: secretary access and timestamp validation, the past-event gate, then the write and // its side effects (group notifications, Google push, background location verification). import { t } from '../../../../config/constants.ts'; +import type { CalendarEvent } from '../../../../database/types.ts'; import { eventTimestampError } from '../../../../utils/event-timestamps.ts'; import { logger } from '../../../../utils/logger.ts'; import type { AgentContext, ToolResult } from '../../types.ts'; import { checkSecretaryAccess } from '../secretary-access.ts'; -import { resolveScope } from '../shared.ts'; +import { resolveScope, type Scope } from '../shared.ts'; import { enqueueGroupNotifications } from './group-notifications.ts'; import { eventToSummary, locationParts, recordChangedDay, timeParts } from './presentation.ts'; import type { CreateEventInput } from './types.ts'; @@ -46,13 +47,79 @@ export async function handleCreateEvent(ctx: AgentContext, input: CreateEventInp return executeCreateEvent(ctx, input, userId); } +/** The receipt for a create that found its event already saved: the same event, reported as created once. */ +function alreadyExistsReceipt(ctx: AgentContext, event: CalendarEvent): ToolResult { + const parts = [`id: ${event.id}`, `title: ${event.title}`]; + parts.push(...timeParts(event.start_at, event.end_at, event.all_day === 1, ctx.user.timezone)); + if (event.description) parts.push(`description: ${event.description}`); + parts.push(...locationParts(event)); + recordChangedDay(ctx, event); + return { + success: true, + // The call succeeded and the event exists, but this run wrote nothing: an interrupted run must not + // report a creation or hold back its retry on account of it. + mutationState: 'not_applied', + alreadyExists: { entity: 'event', id: event.id }, + output: t(ctx.user.language).aiTools.events.eventAlreadyExists(parts.join(', ')), + data: eventToSummary(event, ctx.user.timezone), + agentHint: + 'This event was already saved moments ago (same title, start and end); nothing new was created. ' + + 'Do NOT call create_event again for it: tell the user it is in the calendar, once. ' + + 'Only if the user explicitly asked for a second identical event, call create_event again with allow_duplicate: true.', + }; +} + +/** Same title on the same day as an event that is not this request's repeat: nothing was written, the model must choose. */ +function sameDayConflict(ctx: AgentContext, event: CalendarEvent): ToolResult { + const when = timeParts(event.start_at, event.end_at, event.all_day === 1, ctx.user.timezone).join(', '); + return { + success: false, + mutationState: 'not_applied', + // Written for the person: a workflow that cannot choose shows it as it is. + error: t(ctx.user.language).aiTools.events.eventSameDay(event.title, event.id, when), + agentHint: + 'SAME_DAY_EVENT: to move or change that event call update_event with its id (do not create a new event ' + + 'and leave the old one). If the user really wants a second event with this title that day, ask them, and ' + + 'on a clear yes call create_event again with allow_duplicate: true.', + }; +} + +function groupSavedHint(notificationsQueued: number): string { + return notificationsQueued > 0 + ? `The group event is saved and ${notificationsQueued} member notification(s) have been queued for delivery. Do NOT call create_event again for this event.` + : 'The group event is saved but no member notifications were queued (no registered members or broadcast queue unavailable). Do NOT call create_event again for this event.'; +} + +/** Calendar push and background location check of a created event; neither may fail the create. */ +async function afterCreateEffects( + ctx: AgentContext, + input: CreateEventInput, + event: CalendarEvent, + scope: Scope, + userId: number, +): Promise { + if (scope !== 'group' && ctx.google?.schedulePush) { + try { + await ctx.google.schedulePush(userId, event.id, 'create'); + } catch (err) { + logger.error({ err }, 'schedulePush failed'); + } + } + // Trigger background location verification if event has a concrete location + if (event.location && ctx.locationVerification && !input.location_abstract) { + ctx.locationVerification + .verifyEventLocation(event, ctx.user) + .catch((err) => logger.error({ err, eventId: event.id }, 'Background location verification failed')); + } +} + async function executeCreateEvent(ctx: AgentContext, input: CreateEventInput, userId: number): Promise { try { const scope = resolveScope(input, ctx); if (scope === 'group' && ctx.groupChatId === undefined) { return { success: false, mutationState: 'not_applied', error: 'Group context required for group scope' }; } - const event = ctx.eventService.createEvent({ + const data = { user_id: userId, title: input.title, start_at: input.start_at, @@ -68,10 +135,16 @@ async function executeCreateEvent(ctx: AgentContext, input: CreateEventInput, us group_id: ctx.groupChatId!, created_by: ctx.user.telegram_id, }), - }); + }; + // A repeat of a request saved moments ago (a second run of the same chat, a retry after a lost reply) + // is answered with the saved event; only the model's explicit allow_duplicate makes a second copy. + const { event, outcome } = input.allow_duplicate + ? { event: ctx.eventService.createEvent(data), outcome: 'created' as const } + : ctx.eventService.createEventIfNew(data); + if (outcome === 'duplicate') return alreadyExistsReceipt(ctx, event); + if (outcome === 'same_day') return sameDayConflict(ctx, event); ctx.createdEventIds ??= new Set(); ctx.createdEventIds.add(event.id); - const parts = [`id: ${event.id}`, `title: ${event.title}`]; parts.push(...timeParts(event.start_at, event.end_at, event.all_day === 1, ctx.user.timezone)); if (event.description) parts.push(`description: ${event.description}`); @@ -79,27 +152,9 @@ async function executeCreateEvent(ctx: AgentContext, input: CreateEventInput, us const groupNotificationsQueued = scope === 'group' ? await enqueueGroupNotifications(ctx, event, 'created') : 0; - if (scope !== 'group' && ctx.google?.schedulePush) { - try { - await ctx.google.schedulePush(userId, event.id, 'create'); - } catch (err) { - logger.error({ err }, 'schedulePush failed'); - } - } - - // Trigger background location verification if event has a concrete location - if (event.location && ctx.locationVerification && !input.location_abstract) { - ctx.locationVerification - .verifyEventLocation(event, ctx.user) - .catch((err) => logger.error({ err, eventId: event.id }, 'Background location verification failed')); - } + await afterCreateEffects(ctx, input, event, scope, userId); - const groupHint = - scope === 'group' - ? groupNotificationsQueued > 0 - ? `The group event is saved and ${groupNotificationsQueued} member notification(s) have been queued for delivery. Do NOT call create_event again for this event.` - : 'The group event is saved but no member notifications were queued (no registered members or broadcast queue unavailable). Do NOT call create_event again for this event.' - : undefined; + const groupHint = scope === 'group' ? groupSavedHint(groupNotificationsQueued) : undefined; recordChangedDay(ctx, event); return { diff --git a/src/services/ai/tool-handlers/events/types.ts b/src/services/ai/tool-handlers/events/types.ts index be47ee5f3..bbfda52d7 100644 --- a/src/services/ai/tool-handlers/events/types.ts +++ b/src/services/ai/tool-handlers/events/types.ts @@ -23,6 +23,7 @@ export interface CreateEventInput { recurrence_rule?: string; reminder_minutes?: number[]; force?: boolean; + allow_duplicate?: boolean; scope?: Scope; owner_id?: number; } diff --git a/src/services/ai/tool-handlers/history.ts b/src/services/ai/tool-handlers/history.ts index a4e067d90..6d2f75b53 100644 --- a/src/services/ai/tool-handlers/history.ts +++ b/src/services/ai/tool-handlers/history.ts @@ -59,6 +59,7 @@ export function handleGetHistory(ctx: AgentContext, input: GetHistoryInput): Too const messages = ctx.chatHistory.searchByChat(ctx.groupChatId, { limit, search: input.search, + excludeIds: ctx.deferredTurnRowIds?.(), }); if (messages.length === 0) return { @@ -78,6 +79,7 @@ export function handleGetHistory(ctx: AgentContext, input: GetHistoryInput): Too search: input.search, before, after, + excludeIds: ctx.deferredTurnRowIds?.(), }); if (messages.length === 0) { diff --git a/src/services/ai/tool-handlers/shared.ts b/src/services/ai/tool-handlers/shared.ts index 9df74b3c4..ddd236c40 100644 --- a/src/services/ai/tool-handlers/shared.ts +++ b/src/services/ai/tool-handlers/shared.ts @@ -1,4 +1,4 @@ -type Scope = 'personal' | 'group'; +export type Scope = 'personal' | 'group'; export function resolveScope(input: { scope?: string }, ctx: { isGroup: boolean }): Scope { return (input.scope ?? (ctx.isGroup ? 'group' : 'personal')) as Scope; diff --git a/src/services/ai/tool-schemas.ts b/src/services/ai/tool-schemas.ts index 7707511c5..337212968 100644 --- a/src/services/ai/tool-schemas.ts +++ b/src/services/ai/tool-schemas.ts @@ -40,6 +40,7 @@ const createEventSchema = z recurrence_rule: z.string().optional(), reminder_minutes: z.array(z.number()).optional(), force: toolBoolean.optional(), + allow_duplicate: toolBoolean.optional(), scope: scopeField, owner_id: numericId.optional(), }) diff --git a/src/services/ai/tools.ts b/src/services/ai/tools.ts index b1d17e21b..124bde0de 100644 --- a/src/services/ai/tools.ts +++ b/src/services/ai/tools.ts @@ -86,6 +86,8 @@ const toolDefinitions: ToolDefinition[] = [ name: 'create_event', description: 'Create a new calendar event. Returns the created event. ' + + 'A repeat of an event saved minutes ago returns it as "already exists": do not call again. ' + + 'Same title on the same day as an existing event is rejected with SAME_DAY_EVENT: to move or change that event use update_event. ' + 'If the event time is in the past, the tool will reject with PAST_EVENT error — ' + 'you must ask the user to confirm via ask_user, then retry with force: true.', input_schema: { @@ -115,6 +117,10 @@ const toolDefinitions: ToolDefinition[] = [ type: 'boolean', description: 'Set to true to create event in the past after user confirmed. Do NOT set without asking first.', }, + allow_duplicate: { + type: 'boolean', + description: 'True only if the user explicitly confirmed a second event with the same title on the same day.', + }, scope: scopeProperty, owner_id: ownerIdProperty, }, diff --git a/src/services/ai/types.ts b/src/services/ai/types.ts index a4e89fa30..ae5fa2dd8 100644 --- a/src/services/ai/types.ts +++ b/src/services/ai/types.ts @@ -172,6 +172,14 @@ export interface AgentContext { messageText: string; /** chat_history row ID of the incoming user message. Used to link action log → conversation. */ chatHistoryId?: number; + /** Telegram update_id of the message this run answers; only for logs (a redelivery repeats it). */ + updateId?: number; + /** + * chat_history rows of the turns queued behind this run in its chat, read when the history is loaded (and by + * the history tool). They arrived after this run's message, so the model reads the conversation without + * them; their own runs answer them. + */ + deferredTurnRowIds?: () => ReadonlySet; isGroup: boolean; eventService: EventService; holidayService: HolidayService; @@ -363,6 +371,12 @@ export interface ToolResult { | { kind: 'invitation'; delivery: 'delivered' | 'manual_forward' | 'failed' } | { kind: 'attendance_declined' } | { kind: 'event_deleted' }; + /** + * A create that wrote nothing because the calendar already held that event (a repeat of the same + * request). Distinct from `mutationState`: the call succeeded, but no new event exists, so a receipt or + * an intent answer must say "already in the calendar", not "created". + */ + alreadyExists?: { entity: 'event'; id: number }; /** * Agent-only instruction appended to the tool result seen by the AI. diff --git a/src/services/event/event-service.ts b/src/services/event/event-service.ts index 78306c291..bae94a664 100644 --- a/src/services/event/event-service.ts +++ b/src/services/event/event-service.ts @@ -1,7 +1,7 @@ // src/services/event/event-service.ts import type { AgendaRepository } from '../../database/repositories/agenda.repository.ts'; -import type { EventRepository } from '../../database/repositories/event.repository.ts'; +import type { CreateOutcome, EventRepository } from '../../database/repositories/event.repository.ts'; import type { GroupMemberRepository } from '../../database/repositories/group-member.repository.ts'; import type { ParticipantRepository } from '../../database/repositories/participant.repository.ts'; import type { CalendarEvent, CreateEventData, EventOccurrence, UpdateEventData } from '../../database/types.ts'; @@ -19,6 +19,14 @@ import { } from './free-slots.ts'; import { expandRecurrence } from './recurrence.ts'; +/** + * How recently an identical event (same calendar, title, start, end) must have been created for a + * repeated create to be taken as the same request. Covers a retry after a lost reply and a slow second + * run (a model round trip takes a minute on a loaded host). An older same-title event on the same day is + * not a repeat but is still not saved silently (EventRepository.createUnlessDuplicate, `same_day`). + */ +export const RECENT_DUPLICATE_WINDOW_MS = 10 * 60 * 1000; + export interface FreeSlot { start: string; end: string; @@ -59,40 +67,59 @@ export class EventService { } createEvent(data: CreateEventData): CalendarEvent { - const event = this.eventRepo.create(data); - if (this.materializer) { - // Explicit reminder intervals are persisted on creation and reused after edits/restarts. - const overrides = data.reminder_minutes - ? JSON.stringify(data.reminder_minutes) - : (event.reminder_overrides ?? null); - this.materializer.materialize( - { - id: event.id, - start_at: event.start_at, - reminder_overrides: overrides, - all_day: event.all_day, - user_timezone: event.timezone, - }, - event.user_id, - ); - // For recurring events, also materialize reminders for upcoming occurrences - if (event.recurrence_rule) { - this.materializeRecurringOccurrences(event, overrides); - } + const event = this.eventRepo.createWithEffects(data, (created) => this.materializeReminders(data, created)); + this.announceCreated(event); + return event; + } + + /** + * createEvent for a request that may be a repeat (see EventRepository.createUnlessDuplicate): a repeat + * (`duplicate`) or a same-title event on the same day (`same_day`) returns the existing event and + * writes nothing: no second reminder set, no second "new event" signal. The reminders are written in the + * insert's own transaction, so a failure leaves no event behind and a retry starts clean. + */ + createEventIfNew(data: CreateEventData): { event: CalendarEvent; outcome: CreateOutcome } { + const result = this.eventRepo.createUnlessDuplicate(data, RECENT_DUPLICATE_WINDOW_MS, (event) => + this.materializeReminders(data, event), + ); + if (result.outcome === 'created') this.announceCreated(result.event); + return result; + } + + private materializeReminders(data: CreateEventData, event: CalendarEvent): void { + if (!this.materializer) return; + // Explicit reminder intervals are persisted on creation and reused after edits/restarts. + const overrides = data.reminder_minutes + ? JSON.stringify(data.reminder_minutes) + : (event.reminder_overrides ?? null); + this.materializer.materialize( + { + id: event.id, + start_at: event.start_at, + reminder_overrides: overrides, + all_day: event.all_day, + user_timezone: event.timezone, + }, + event.user_id, + ); + // For recurring events, also materialize reminders for upcoming occurrences + if (event.recurrence_rule) { + this.materializeRecurringOccurrences(event, overrides); } - if (this.domainEvents) { - if (event.owner_type === 'group' && event.group_id) { - this.domainEvents.emit('myGroup.newEvent', { - userId: event.created_by ?? event.user_id, - groupChatId: event.group_id, - newEvent: event, - createdBy: event.created_by ?? event.user_id, - }); - } else { - this.domainEvents.emit('myCalendar.newEvent', { userId: event.user_id, newEvent: event }); - } + } + + private announceCreated(event: CalendarEvent): void { + if (!this.domainEvents) return; + if (event.owner_type === 'group' && event.group_id) { + this.domainEvents.emit('myGroup.newEvent', { + userId: event.created_by ?? event.user_id, + groupChatId: event.group_id, + newEvent: event, + createdBy: event.created_by ?? event.user_id, + }); + } else { + this.domainEvents.emit('myCalendar.newEvent', { userId: event.user_id, newEvent: event }); } - return event; } updateEvent( diff --git a/src/services/intent/answer-renderers.ts b/src/services/intent/answer-renderers.ts index 9719f56e0..181734040 100644 --- a/src/services/intent/answer-renderers.ts +++ b/src/services/intent/answer-renderers.ts @@ -31,6 +31,8 @@ export interface ToolAnswer { output: string; data?: ToolResultData; effect?: ToolResult['effect']; + /** The create wrote nothing: the event was already saved. */ + alreadyExists?: ToolResult['alreadyExists']; } export interface RenderContext extends HumanTimeContext { @@ -140,6 +142,7 @@ function renderConfirmation(answer: ToolAnswer, context: RenderContext): Rendere const write: ConfirmedWrite = answer.tool === 'create_event' ? 'created' : answer.tool === 'update_event' ? 'updated' : 'deleted'; const when = `${humanDay(event.date, context)}, ${eventTimeLabel(event, context, words)}`; + if (answer.alreadyExists?.entity === 'event') return rendered(words.alreadyExists(event.title, when)); return rendered(words.confirmed(write, event.title, when)); } diff --git a/src/services/intent/answer-wording.ts b/src/services/intent/answer-wording.ts index de2bc3131..c36916c3b 100644 --- a/src/services/intent/answer-wording.ts +++ b/src/services/intent/answer-wording.ts @@ -27,6 +27,8 @@ export interface AnswerWording { freeNowhere: (day: HumanDayParts, calendar: string) => string; noFreeWindowsInline: string; confirmed: (write: ConfirmedWrite, title: string, when: string) => string; + /** A create that found the same event already saved: nothing new was written. */ + alreadyExists: (title: string, when: string) => string; eventCard: { place: string; description: string; @@ -105,6 +107,7 @@ const ruWords: AnswerWording = { noFreeWindowsInline: 'свободных окон нет', confirmed: (write, title, when) => `${write === 'created' ? 'Событие создано' : write === 'updated' ? 'Событие обновлено' : 'Событие удалено'}: «${title}» — ${when}.`, + alreadyExists: (title, when) => `Это событие уже есть в календаре, второй раз не создавал: «${title}» — ${when}.`, eventCard: { place: 'Место', description: 'Описание', @@ -202,6 +205,8 @@ const enWords: AnswerWording = { `For ${day.relative ? `${day.relative}, ` : ''}${day.calendar}, there are no free windows ${calendar}: the day is fully booked.`, noFreeWindowsInline: 'no free windows', confirmed: (write, title, when) => `Event ${write}: “${title}” — ${when}.`, + alreadyExists: (title, when) => + `This event is already in the calendar, so I did not create it again: “${title}” — ${when}.`, eventCard: { place: 'Where', description: 'Notes', diff --git a/src/services/intent/intent-executor.ts b/src/services/intent/intent-executor.ts index 038a5bf8b..11136e5f2 100644 --- a/src/services/intent/intent-executor.ts +++ b/src/services/intent/intent-executor.ts @@ -6,7 +6,7 @@ import { cmdLogger } from '../../utils/logger.ts'; import { isEventSummary } from '../ai/event-summaries.ts'; import { isMutationTool, isReadOnlyCall } from '../ai/tool-executor.ts'; import type { ToolResult, ToolResultData } from '../ai/types.ts'; -import { quotableText, type ToolAnswer } from './answer-renderers.ts'; +import { quotableText, renderToolAnswer, type ToolAnswer } from './answer-renderers.ts'; import { confirmationChoice } from './confirm-words.ts'; import { evaluate } from './expression-evaluator.ts'; import { applyFilters, parseFilterChain } from './filter-parser.ts'; @@ -198,6 +198,7 @@ function toolAnswer(tool: string, input: unknown, result: ToolResult): ToolAnswe output: result.output, ...(result.data !== undefined ? { data: result.data } : {}), ...(result.effect !== undefined ? { effect: result.effect } : {}), + ...(result.alreadyExists !== undefined ? { alreadyExists: result.alreadyExists } : {}), }; } @@ -326,11 +327,18 @@ interface Level2Env { } /** Values the step loop carries from one executed tool step to the next and into the final result. */ +type WriteReport = { existing: ToolAnswer } | { text: string }; + interface Level2LoopState { mentionedEventId: number | undefined; lastToolOutput: string | undefined; lastAnswer: ToolAnswer | undefined; completeResponse: boolean; + /** + * What the confirmed write steps of this run reported, in order: a create that found its event already saved + * (a closing «created» sentence would be untrue) or the tool's own words for a write. + */ + writeReports: WriteReport[]; } function resolveStepInput(env: Level2Env, name: string, template: unknown, strict = env.strict): unknown { @@ -439,7 +447,19 @@ function applyResumeAnswer(steps: Level2Step[], resumeState: ResumeState, env: L } /** `respond:` field — answer with text and stop. */ -function respondWithText(template: string, env: Level2Env, mentionedEventId: number | undefined): ExecutorResult { +function respondWithText( + template: string, + env: Level2Env, + mentionedEventId: number | undefined, + writeReports: WriteReport[], +): ExecutorResult { + if (writeReports.some((report) => 'existing' in report)) { + const { timezone, language } = env.userCtx; + const lines = writeReports.map((report) => + 'existing' in report ? renderToolAnswer(report.existing, { now: new Date(), timezone, language }).text : report.text, + ); + return { success: true, response: lines.join('\n'), mentionedEventId }; + } const text = resolveVariables(template, env.captures, env.userCtx, env.stepResults, env.i18n, { strict: env.strict, }); @@ -566,6 +586,8 @@ async function runToolStep( state.lastToolOutput = result.output; const answer = toolAnswer(call, resolvedInput, result); state.lastAnswer = answer; + if (answer?.alreadyExists) state.writeReports.push({ existing: answer }); + else if (result.mutationState === 'confirmed' && result.output) state.writeReports.push({ text: result.output }); state.completeResponse = result.completeResponse === true; // If result carries structured event data, update last_mentioned_event in-workflow @@ -597,7 +619,8 @@ async function runLevel2Step( if (step.abstain === true) return { success: false, errorCode: 'ABSTAIN' }; // Respond with text and optionally stop - if (step.respond !== undefined) return respondWithText(step.respond, env, state.mentionedEventId); + if (step.respond !== undefined) + return respondWithText(step.respond, env, state.mentionedEventId, state.writeReports); // No call — nothing to execute in this step if (step.call === undefined) return null; @@ -637,6 +660,7 @@ async function runLevel2( mentionedEventId: undefined, lastToolOutput: undefined, lastAnswer: undefined, + writeReports: [], completeResponse: false, }; diff --git a/test/bot/handlers/message.handler.test.ts b/test/bot/handlers/message.handler.test.ts index a955b92d8..3c8d7718a 100644 --- a/test/bot/handlers/message.handler.test.ts +++ b/test/bot/handlers/message.handler.test.ts @@ -10,6 +10,15 @@ describe('createMessageHandler', () => { expect(deps.agent.run).toHaveBeenCalledTimes(1); }); + test('links the run to the row saved for this very message, not to the user’s latest row', async () => { + // A newer message of the same user already replaced the per-user entry (row 99) while this one was handled. + const deps = makeDeps({ chatHistoryIds: new Map([[100, 99]]) }); + const handler = createMessageHandler(deps as never); + await handler({ ...makeCtx(), chatHistoryId: 41 } as never); + const call = (deps.agent.run as ReturnType).mock.calls[0]![0] as { chatHistoryId: number }; + expect(call.chatHistoryId).toBe(41); + }); + test('ignores messages without text', async () => { const deps = makeDeps(); const handler = createMessageHandler(deps as never); diff --git a/test/bot/middleware/chat-logging.test.ts b/test/bot/middleware/chat-logging.test.ts index fe9aa0dd6..733786552 100644 --- a/test/bot/middleware/chat-logging.test.ts +++ b/test/bot/middleware/chat-logging.test.ts @@ -4,6 +4,7 @@ import { createChatLogging } from '../../../src/bot/middleware/chat-logging.ts'; import { createConnectWizardGuard } from '../../../src/bot/middleware/connect-wizard-guard.ts'; import { CONNECT_WIZARD_REDACTION } from '../../../src/bot/scenes/connect-telegram.scene.ts'; import { DatabaseService } from '../../../src/database/index.ts'; +import { currentChatTurn } from '../../../src/services/ai/chat-turn-context.ts'; import { ConversationLogger } from '../../../src/services/conversation-logger.ts'; describe('chat logging when the connect-wizard state cannot be read (GH-519)', () => { @@ -23,6 +24,7 @@ describe('chat logging when the connect-wizard state cannot be read (GH-519)', ( conversationLogger: new ConversationLogger(db.chatHistory), actionLog: db.actionLog, chatHistoryIds: new Map(), + reserveChatTurn: (chatId, rowId) => ({ chatId, rowId, release() {} }), isConnectWizardInput: guard.isConnectWizardInput, }); const update: TelegramUpdate = { @@ -58,3 +60,87 @@ describe('chat logging when the connect-wizard state cannot be read (GH-519)', ( db.db.close(); }); }); + +describe('the row saved for each update', () => { + function textUpdate(updateId: number, userId: number, text: string): TelegramUpdate { + return { + update_id: updateId, + message: { + message_id: updateId, + date: 1, + chat: { id: userId, type: 'private' }, + from: { id: userId, is_bot: false, first_name: 'Synthetic' }, + text, + }, + }; + } + + test('each update’s context names its own row even after a later message replaced the user’s latest row', async () => { + const db = new DatabaseService(':memory:'); + const user = db.users.create({ telegram_id: 719_901, language: 'ru', timezone: 'UTC' }); + const chatHistoryIds = new Map(); + const logging = createChatLogging({ + conversationLogger: new ConversationLogger(db.chatHistory), + actionLog: db.actionLog, + chatHistoryIds, + reserveChatTurn: (chatId, rowId) => ({ chatId, rowId, release() {} }), + isConnectWizardInput: () => false, + }); + const first: { dbUser: typeof user; update: TelegramUpdate; chatHistoryId?: number } = { + dbUser: user, + update: textUpdate(1, user.telegram_id, 'first'), + }; + const second: typeof first = { dbUser: user, update: textUpdate(2, user.telegram_id, 'second') }; + const command: typeof first = { dbUser: user, update: textUpdate(3, user.telegram_id, '/today') }; + + await logging(first, async () => {}); + await logging(second, async () => {}); + await logging(command, async () => {}); + + const rows = db.db + .query<{ id: number; content: string }, []>( + "SELECT id, content FROM chat_history WHERE role = 'user' ORDER BY id", + ) + .all(); + expect(rows.map((row) => row.content).slice(0, 2)).toEqual(['first', 'second']); + expect(first.chatHistoryId).toBe(rows[0]?.id); + expect(second.chatHistoryId).toBe(rows[1]?.id); + expect(chatHistoryIds.get(user.telegram_id)).toBe(rows[1]?.id); // the per-user entry only knows the latest + expect(command.chatHistoryId).toBeUndefined(); // a command is logged as a command, not as a turn to answer + db.db.close(); + }); + + test('a saved turn holds its place in the chat at once and gives it up when its handling ends', async () => { + const db = new DatabaseService(':memory:'); + const user = db.users.create({ telegram_id: 719_902, language: 'ru', timezone: 'UTC' }); + const events: string[] = []; + const logging = createChatLogging({ + conversationLogger: new ConversationLogger(db.chatHistory), + actionLog: db.actionLog, + chatHistoryIds: new Map(), + reserveChatTurn: (chatId, rowId) => { + events.push(`reserve ${chatId}:${rowId}`); + return { chatId, rowId, release: () => events.push(`release ${rowId}`) }; + }, + isConnectWizardInput: () => false, + }); + const message: { dbUser: typeof user; update: TelegramUpdate; chatHistoryId?: number } = { + dbUser: user, + update: textUpdate(1, user.telegram_id, 'Yes'), + }; + const command: typeof message = { dbUser: user, update: textUpdate(2, user.telegram_id, '/today') }; + + let seenInHandler: number | undefined; + await logging(message, async () => { + events.push('handler'); + seenInHandler = currentChatTurn()?.rowId; // what an agent run started by this update finds to take over + }); + await logging(command, async () => {}); + + const rowId = message.chatHistoryId; + expect(seenInHandler).toBe(rowId); + expect(currentChatTurn()).toBeUndefined(); // and nothing leaks out of the update + expect(events).toEqual([`reserve ${user.telegram_id}:${rowId}`, 'handler', `release ${rowId}`]); // none for the command + db.db.close(); + }); +}); diff --git a/test/bot/pipeline/ai-agent-layer-overlap.test.ts b/test/bot/pipeline/ai-agent-layer-overlap.test.ts new file mode 100644 index 000000000..23b200e86 --- /dev/null +++ b/test/bot/pipeline/ai-agent-layer-overlap.test.ts @@ -0,0 +1,265 @@ +// The 2026-11-01 incident, replayed through the real AI agent layer and the real CalendarBotAgent: +// quick confirmations in one private chat while the first run is still busy on a slow host. Only +// the model is scripted (a function of what it is shown); the layer, the agent, the tools, the +// 5 s repeated-call throttle and the database are real. +import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test'; +import type { AgentLayerDeps } from '../../../src/bot/pipeline/ai-agent-layer.ts'; +import { createAiAgentLayer } from '../../../src/bot/pipeline/ai-agent-layer.ts'; +import type { PipelineLayer } from '../../../src/bot/pipeline/types.ts'; +import type { BotCommandContext } from '../../../src/bot/types.ts'; +import { ActionLogRepository } from '../../../src/database/repositories/action-log.repository.ts'; +import { CalendarBotAgent } from '../../../src/services/ai/agent.ts'; +import type { ChatTurnReservation } from '../../../src/services/ai/chat-run-gate.ts'; +import { withChatTurn } from '../../../src/services/ai/chat-turn-context.ts'; +import { executeTool } from '../../../src/services/ai/tool-executor.ts'; +import type { AgentContext } from '../../../src/services/ai/types.ts'; +import { cmdLogger } from '../../../src/utils/logger.ts'; +import { + type ModelRequest, + REVIEW, + recordDelivery, + SlowHost, + STANDUP, + settle, + titlesInCalendar, + userTurn, +} from '../../helpers/overlap-fixtures.ts'; +import { type AgentRunFixture, createAgentRunFixture, USER_ID } from '../../services/ai/agent-run-helpers.ts'; + +describe('overlapping confirmations in one private chat (2026-11-01)', () => { + let fixture: AgentRunFixture; + let host: SlowHost; + let chatHistoryIds: Map; + let nextUpdateId: number; + let agentInUse: CalendarBotAgent; + const reservations = new Map(); + + beforeEach(() => { + fixture = createAgentRunFixture(); + host = new SlowHost(); + chatHistoryIds = new Map(); + nextUpdateId = 900; + const { ctx } = fixture; + ctx.conversationLogger.logUserMessage(USER_ID, 'Plan the Synthetic Standup and the Synthetic Review'); + ctx.conversationLogger.logAiTurn(USER_ID, { role: 'assistant', content: 'Shall I add both events?' }); + }); + + afterEach(() => host.restore()); + + /** What the chat-logging middleware does on receipt: the message is saved, its row id remembered, its place reserved. */ + function receive(text: string): number { + const chatHistoryId = fixture.ctx.conversationLogger.logUserMessage(USER_ID, text); + chatHistoryIds.set(USER_ID, chatHistoryId); + reservations.set(chatHistoryId, agentInUse.reserveChatTurn(USER_ID, chatHistoryId)); + return chatHistoryId; + } + + /** What the message handler does with a received message: hands it to the layer. */ + function deliver(layer: PipelineLayer, text: string, chatHistoryId: number) { + const { ctx } = fixture; + const updateId = nextUpdateId++; + const incoming = { + dbUser: ctx.user, + chatId: USER_ID, + id: updateId + 500, + updateId, + chatHistoryId, + send: async () => {}, + } as unknown as BotCommandContext; + const run = () => layer(incoming, text, { incomingMessageId: updateId + 500, chatHistoryId }); + const reservation = reservations.get(chatHistoryId); + return reservation ? withChatTurn(reservation, run) : run(); + } + + function arrive(layer: PipelineLayer, text: string) { + return deliver(layer, text, receive(text)); + } + + function layerFor(agent: CalendarBotAgent) { + agentInUse = agent; + const build: AgentLayerDeps['agentContextBuilder'] = (user, chatId, messageText, _group, incomingMessageId) => + ({ + ...fixture.ctx, + user, + chatId, + messageText, + incomingMessageId, + chatHistoryId: chatHistoryIds.get(user.telegram_id), + wasExplicitInvocation: true, + }) satisfies AgentContext; + return createAiAgentLayer({ agent, agentContextBuilder: build }); + } + + test('the second confirmation waits, is handled on the first run’s history and creates nothing', async () => { + const shown = recordDelivery(fixture.sender); + const layer = layerFor(new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: host.model })); + + const first = arrive(layer, 'Yes'); + await host.reached; + const second = arrive(layer, 'Yes'); + await settle(); + host.release(); + await Promise.all([first, second]); + + // One set of rows: the second run saw the first run's creates and did not repeat them. + expect(titlesInCalendar(fixture)).toEqual([STANDUP.title, REVIEW.title]); + + // The second run's transcript: the created events come first, its own «Yes» is the newest turn. + const secondRun = host.requests.find( + (request) => userTurn(request.messages.at(-1)) === 'Yes' && request.messages.some((m) => m.role === 'tool'), + ); + expect(secondRun).toBeDefined(); + const roles = secondRun?.messages.map((m) => m.role) ?? []; + expect(roles.lastIndexOf('tool')).toBeLessThan(roles.lastIndexOf('user')); + expect(roles.at(-1)).toBe('user'); + + // Ordered replies: the first run's report is out before the second run says anything. + const firstReport = shown.findIndex((text) => text.includes('Done:')); + const secondReport = shown.findIndex((text) => text.includes('already exist')); + expect(firstReport).toBeGreaterThan(-1); + expect(secondReport).toBeGreaterThan(firstReport); + }); + + test('a third message waits for the second, and each run is shown only the turns up to its own', async () => { + const layer = layerFor(new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: host.model })); + + const pending = [arrive(layer, 'Yes')]; + await host.reached; + pending.push(arrive(layer, 'Yes, both')); + pending.push(arrive(layer, 'Thanks')); + await settle(); + host.release(); + await Promise.all(pending); + + expect(titlesInCalendar(fixture)).toEqual([STANDUP.title, REVIEW.title]); + + const userTexts = (request: ModelRequest) => request.messages.filter((m) => m.role === 'user').map(userTurn); + const lastUserOf = (request: ModelRequest) => userTexts(request).at(-1); + // First requests of the three runs come in arrival order. + const order = host.requests.map(lastUserOf).filter((text, index, all) => all.indexOf(text) === index); + expect(order).toEqual(['Yes', 'Yes, both', 'Thanks']); + // A run is not shown the messages queued behind it. + for (const request of host.requests) { + const last = lastUserOf(request); + if (last === 'Yes') expect(userTexts(request)).not.toContain('Yes, both'); + if (last !== 'Thanks') expect(userTexts(request)).not.toContain('Thanks'); + } + // The last run reads the conversation in the order things happened: the first confirmation, what the + // first run did, the second confirmation, and only then its own message. + const lastRun = host.requests.filter((request) => lastUserOf(request) === 'Thanks').at(-1); + const seen = lastRun?.messages.map((m) => (m.role === 'user' ? userTurn(m) : m.role)) ?? []; + const at = (needle: string) => seen.indexOf(needle); + expect(at('Yes')).toBeGreaterThan(-1); + expect(at('tool')).toBeGreaterThan(at('Yes')); + expect(at('Yes, both')).toBeGreaterThan(at('tool')); + expect(at('Thanks')).toBe(seen.length - 1); + }); + + test('a message received but still parked before the agent is not read by the run ahead of it', async () => { + const layer = layerFor(new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: host.model })); + + const firstRow = receive('Yes'); + const parkedText = 'actually cancel everything'; + const parkedRow = receive(parkedText); // saved, but its handler is delayed before it reaches the agent + const first = deliver(layer, 'Yes', firstRow); + await host.reached; + host.release(); + await first; + + const everyUserTurnShown = host.requests.flatMap((request) => request.messages.filter((m) => m.role === 'user')); + expect(everyUserTurnShown.map(userTurn)).not.toContain(parkedText); + + // The parked message is then handled in its own run, which does read it. + await deliver(layer, parkedText, parkedRow); + expect(host.requests.at(-1)?.messages.map((m) => (m.role === 'user' ? userTurn(m) : m.role))).toContain(parkedText); + }); + + test('a message whose handler reaches the agent late is still answered in the order the messages were saved', async () => { + const layer = layerFor(new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: host.model })); + + const firstRow = receive('Yes'); // saved first, its handler is slow (a scene or retry lookup) + const secondRow = receive('Thanks'); // saved second, its handler is quick + const second = deliver(layer, 'Thanks', secondRow); + await settle(); + expect(host.requests).toHaveLength(0); // the quick one waits for the place saved before it + const first = deliver(layer, 'Yes', firstRow); + await host.reached; + host.release(); + await Promise.all([first, second]); + + const userTexts = (request: ModelRequest) => request.messages.filter((m) => m.role === 'user').map(userTurn); + const order = host.requests + .map((request) => userTexts(request).at(-1)) + .filter((text, i, all) => all.indexOf(text) === i); + expect(order).toEqual(['Yes', 'Thanks']); + expect(userTexts(host.requests[0]!)).not.toContain('Thanks'); + expect(titlesInCalendar(fixture)).toEqual([STANDUP.title, REVIEW.title]); + }); + + test('ids held by the pipeline stay valid after a message waited and was placed after the earlier run', async () => { + const layer = layerFor(new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: host.model })); + + const first = arrive(layer, 'Yes'); + await host.reached; + const secondRow = receive('Yes, both'); + const second = deliver(layer, 'Yes, both', secondRow); + await settle(); + host.release(); + await Promise.all([first, second]); + + // A scheduled run or button built from the per-user map after the move writes an audit row against it. + const held = chatHistoryIds.get(USER_ID); + expect(held).toBe(secondRow); + const audited = await executeTool( + { ...fixture.ctx, actionLogRepo: new ActionLogRepository(fixture.db), chatHistoryId: held, messageText: '' }, + 'create_event', + { + title: 'Synthetic Scheduled', + start_at: '2031-01-01T10:00:00Z', + allow_duplicate: true, + }, + ); + expect(audited.success).toBe(true); + expect( + fixture.db.prepare('SELECT chat_history_id FROM user_action_log WHERE action_name = ?').all('create_event'), + ).toContainEqual({ chat_history_id: secondRow }); + }); + + describe('the order of runs alone, with every copy asked for deliberately', () => { + // allow_duplicate switches the calendar's own repeat check off, so the one set of rows below can only + // come from the second run being shown what the first one did and not creating again. + beforeEach(() => { + host.restore(); + host = new SlowHost({ allowDuplicates: true }); + }); + + test('a queued confirmation creates nothing the first run already created', async () => { + const layer = layerFor(new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: host.model })); + + const first = arrive(layer, 'Yes'); + await host.reached; + const second = arrive(layer, 'Yes'); + await settle(); + host.release(); + await Promise.all([first, second]); + + expect(titlesInCalendar(fixture)).toEqual([STANDUP.title, REVIEW.title]); + }); + }); + + test('the routing log line carries the Telegram update id and message id next to the message text it already had', async () => { + const info = spyOn(cmdLogger, 'info').mockImplementation(() => {}); + try { + const layer = layerFor(new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: host.model })); + const done = arrive(layer, 'Yes'); + await host.reached; + host.release(); + await done; + const routing = info.mock.calls.find((call) => call[1] === 'Routing to AI agent'); + expect(routing).toBeDefined(); + expect(routing?.[0]).toMatchObject({ userId: USER_ID, messageText: 'Yes', updateId: 900, messageId: 1400 }); + } finally { + info.mockRestore(); + } + }); +}); diff --git a/test/bot/scenes/connect-telegram.secrets.test.ts b/test/bot/scenes/connect-telegram.secrets.test.ts index 9954f6071..724029b58 100644 --- a/test/bot/scenes/connect-telegram.secrets.test.ts +++ b/test/bot/scenes/connect-telegram.secrets.test.ts @@ -206,6 +206,7 @@ function makeRuntime(options: { messagesPerMinute?: number } = {}) { conversationLogger, actionLog: db.actionLog, chatHistoryIds: new Map(), + reserveChatTurn: (chatId, rowId) => ({ chatId, rowId, release() {} }), isConnectWizardInput: guard.isConnectWizardInput, }), ) diff --git a/test/database/repositories/chat-history.repository.test.ts b/test/database/repositories/chat-history.repository.test.ts index 72d9735ba..2054494be 100644 --- a/test/database/repositories/chat-history.repository.test.ts +++ b/test/database/repositories/chat-history.repository.test.ts @@ -248,3 +248,110 @@ describe('ChatHistoryRepository', () => { }); }); }); + +describe('ChatHistoryRepository queued-turn handling', () => { + const USER_ID = 321; + let db: Database; + let repo: ChatHistoryRepository; + + beforeEach(() => { + db = createTestDb(); + new UserRepository(db).create({ telegram_id: USER_ID }); + repo = new ChatHistoryRepository(db); + }); + + test('excluded rows are left out before the limit, in private history, group history and the history search', () => { + const rows = Array.from({ length: 6 }, (_, index) => repo.save(USER_ID, 'user', `m${index}`)); + const newest = new Set([rows[4]!, rows[5]!]); + expect(repo.getRecent(USER_ID, 3, newest).map((row) => row.content)).toEqual(['m1', 'm2', 'm3']); + expect(repo.getRecent(USER_ID, 3).map((row) => row.content)).toEqual(['m3', 'm4', 'm5']); + expect(repo.search(USER_ID, { excludeIds: newest }).map((row) => row.content)).toEqual(['m0', 'm1', 'm2', 'm3']); + repo.save(USER_ID, 'user', 'g1', -5); + const group = repo.save(USER_ID, 'user', 'g2', -5); + expect(repo.getRecentByChat(-5, 10, new Set([group])).map((row) => row.content)).toEqual(['g1']); + expect(repo.searchByChat(-5, { excludeIds: new Set([group]) }).map((row) => row.content)).toEqual(['g1']); + }); + + describe('placeUserTurnAfterLatest', () => { + const roles = () => repo.getRecent(USER_ID, 20).map((row) => `${row.role}:${row.content}`); + + test('reads the turn after what was saved since, under the same id', () => { + const confirm = repo.save(USER_ID, 'user', 'Yes'); + repo.save(USER_ID, 'assistant', 'working on the earlier request'); + repo.save(USER_ID, 'tool', '[]'); + + expect(repo.placeUserTurnAfterLatest(confirm, USER_ID)).toBe(true); + + expect(roles()).toEqual(['assistant:working on the earlier request', 'tool:[]', 'user:Yes']); + expect(repo.getRecent(USER_ID, 20).at(-1)?.id).toBe(confirm); + }); + + test('rows saved afterwards, in the same second, still read after the moved turn', () => { + const confirm = repo.save(USER_ID, 'user', 'Yes'); + repo.save(USER_ID, 'assistant', 'earlier output'); + repo.placeUserTurnAfterLatest(confirm, USER_ID); + repo.save(USER_ID, 'assistant', 'answer to the moved turn'); + expect(roles()).toEqual(['assistant:earlier output', 'user:Yes', 'assistant:answer to the moved turn']); + }); + + test('nothing moves while no bot row follows the turn', () => { + const first = repo.save(USER_ID, 'user', 'Yes'); + const queued = repo.save(USER_ID, 'user', 'Thanks'); + expect(repo.placeUserTurnAfterLatest(first, USER_ID)).toBe(false); + expect(roles()).toEqual(['user:Yes', 'user:Thanks']); + expect(queued).toBeGreaterThan(first); + }); + + test('a group turn stays in its group', () => { + const row = repo.save(USER_ID, 'user', 'book it', -1001); + repo.save(USER_ID, 'assistant', 'earlier answer', -1001); + expect(repo.placeUserTurnAfterLatest(row, USER_ID)).toBe(true); + expect(repo.getRecentByChat(-1001, 10).map((entry) => entry.content)).toEqual(['earlier answer', 'book it']); + }); + + test('in a group, another member’s run output counts as what the turn must be read after', () => { + new UserRepository(db).create({ telegram_id: 456 }); + const memberAMessage = repo.save(USER_ID, 'user', 'A: add the review', -1001); + repo.save(456, 'assistant', 'answer to member B', -1001); // the active run was triggered by member B + expect(repo.placeUserTurnAfterLatest(memberAMessage, USER_ID)).toBe(true); + expect(repo.getRecentByChat(-1001, 10).map((entry) => entry.content)).toEqual([ + 'answer to member B', + 'A: add the review', + ]); + }); + + test('a private turn is not moved by bot rows of another conversation', () => { + new UserRepository(db).create({ telegram_id: 456 }); + const privateTurn = repo.save(USER_ID, 'user', 'Yes'); + repo.save(USER_ID, 'assistant', 'group answer', -1001); + repo.save(456, 'assistant', 'someone else', undefined); + expect(repo.placeUserTurnAfterLatest(privateTurn, USER_ID)).toBe(false); + }); + + test('tool actions logged against the turn keep pointing at it', () => { + const turn = repo.save(USER_ID, 'user', 'Yes'); + repo.save(USER_ID, 'assistant', 'earlier run output'); + repo.placeUserTurnAfterLatest(turn, USER_ID); + db.prepare( + "INSERT INTO user_action_log (user_id, chat_id, action_type, action_name, chat_history_id, success) VALUES (?, ?, 'tool', 'create_event', ?, 1)", + ).run(USER_ID, USER_ID, turn); + expect(db.prepare('SELECT chat_history_id FROM user_action_log').get()).toEqual({ chat_history_id: turn }); + }); + + test.each([ + ['another user’s row', 456, 'user'], + ['a bot row', USER_ID, 'assistant'], + ] as const)('leaves %s where it is', (_name, owner, role) => { + new UserRepository(db).create({ telegram_id: 456 }); + const row = repo.save(owner, role, 'Yes'); + repo.save(USER_ID, 'assistant', 'later'); + expect(repo.placeUserTurnAfterLatest(row, USER_ID)).toBe(false); + }); + + test('a row that no longer exists is reported, nothing changes', () => { + repo.save(USER_ID, 'user', 'Yes'); + expect(repo.placeUserTurnAfterLatest(9_999, USER_ID)).toBe(false); + expect(roles()).toEqual(['user:Yes']); + }); + }); +}); diff --git a/test/helpers/manual-time.ts b/test/helpers/manual-time.ts new file mode 100644 index 000000000..897c2bc84 --- /dev/null +++ b/test/helpers/manual-time.ts @@ -0,0 +1,27 @@ +// A clock and timers a test advances by hand, for code that takes `now` and `schedule` as options. + +export class ManualTime { + nowMs = 1_000_000; + private timers: { at: number; callback: () => void }[] = []; + + now = () => this.nowMs; + + schedule = (callback: () => void, ms: number): (() => void) => { + const timer = { at: this.nowMs + ms, callback }; + this.timers.push(timer); + return () => { + this.timers = this.timers.filter((candidate) => candidate !== timer); + }; + }; + + advance(ms: number): void { + this.nowMs += ms; + const due = this.timers.filter((timer) => timer.at <= this.nowMs); + this.timers = this.timers.filter((timer) => timer.at > this.nowMs); + for (const timer of due) timer.callback(); + } + + get pendingTimers(): number { + return this.timers.length; + } +} diff --git a/test/helpers/overlap-fixtures.ts b/test/helpers/overlap-fixtures.ts new file mode 100644 index 000000000..519305636 --- /dev/null +++ b/test/helpers/overlap-fixtures.ts @@ -0,0 +1,152 @@ +// Shared fixtures for the suites that replay the 2026-11-01 overlap: a slow host (a clock the test +// moves and one parked model request) and a model that is a function of what it is shown. Only the +// model is scripted; agents, tools, the 5 s repeated-call throttle and the database are real. +import { spyOn } from 'bun:test'; +import type OpenAI from 'openai'; +import type { StreamCallbacks, StreamRoundOptions, StreamRoundResult } from '../../src/services/ai/streaming.ts'; +import type { TelegramSender } from '../../src/services/ai/types.ts'; +import { type AgentRunFixture, fakeMetrics, isValidatorCall, USER_ID } from '../services/ai/agent-run-helpers.ts'; + +export const STANDUP = { title: 'Synthetic Standup', start_at: '2030-03-05T10:00:00Z', end_at: '2030-03-05T10:30:00Z' }; +export const REVIEW = { title: 'Synthetic Review', start_at: '2030-03-05T11:00:00Z', end_at: '2030-03-05T12:00:00Z' }; +/** How long a held model request is held: longer than the 5 s identical-call throttle, as on the slow host. */ +export const HOST_DELAY_MS = 10_000; + +export interface ModelRequest { + messages: OpenAI.ChatCompletionMessageParam[]; +} + +export function textOf(message: OpenAI.ChatCompletionMessageParam | undefined): string { + const content = message?.content; + return typeof content === 'string' ? content : JSON.stringify(content ?? ''); +} + +/** A user turn as the model is shown it, without the timestamp the history builder prefixes. */ +export function userTurn(message: OpenAI.ChatCompletionMessageParam | undefined): string { + return textOf(message).replace(/^\[[^\]]*\] /, ''); +} + +function textRound(text: string): StreamRoundResult { + return { + text, + toolCalls: [], + finishReason: 'stop', + assistantMessage: { role: 'assistant', content: text }, + providerUsed: 'mock', + metrics: fakeMetrics('zai', 100, 20), + }; +} + +function createBothRound(turn: number, allowDuplicate: boolean): StreamRoundResult { + const calls = [STANDUP, REVIEW].map((event, index) => ({ + id: `create-${turn}-${index}`, + name: 'create_event', + arguments: JSON.stringify(allowDuplicate ? { ...event, allow_duplicate: true } : event), + })); + return { + text: '', + toolCalls: calls, + finishReason: 'tool_calls', + assistantMessage: { + role: 'assistant', + content: null, + tool_calls: calls.map((call) => ({ + id: call.id, + type: 'function' as const, + function: { name: call.name, arguments: call.arguments }, + })), + }, + providerUsed: 'mock', + }; +} + +/** The host: a clock the test moves and one model request it can park, the way a loaded odroid does. */ +export class SlowHost { + readonly requests: ModelRequest[] = []; + /** + * `allowDuplicates`: the model asks for every copy deliberately, so the calendar's own repeat check + * cannot hide a run that created again; only what the run was shown can keep it from doing so. + */ + constructor(private readonly options: { allowDuplicates?: boolean } = {}) {} + + private clockMs = Date.parse('2030-01-01T00:00:00Z'); + private readonly released = Promise.withResolvers(); + private readonly reachedFirst = Promise.withResolvers(); + private firstRequestSeen = false; + private readonly clock = spyOn(Date, 'now').mockImplementation(() => this.clockMs); + + /** Settles once the first model request is waiting. */ + get reached(): Promise { + return this.reachedFirst.promise; + } + + /** The parked request returns HOST_DELAY_MS later. */ + release(): void { + this.clockMs += HOST_DELAY_MS; + this.released.resolve(); + } + + restore(): void { + this.clock.mockRestore(); + } + + /** + * A model that behaves like a sensible one: asked to confirm, it creates both events; once the + * history it is shown already holds the created events, it only says so; after its own tool + * results it reports them. + */ + model = async (opts: StreamRoundOptions, callbacks: StreamCallbacks = {}): Promise => { + if (isValidatorCall(opts)) return { ...textRound('APPROVE'), metrics: fakeMetrics('hf', 10, 2) }; + this.requests.push({ messages: opts.messages }); + const turn = this.requests.length; + if (!this.firstRequestSeen) { + this.firstRequestSeen = true; + this.reachedFirst.resolve(); + await this.released.promise; + } + const messages = opts.messages; + const lastUser = messages.map((m) => m.role).lastIndexOf('user'); + const thisTurn = messages.slice(lastUser + 1); + const before = messages.slice(0, lastUser); + let round: StreamRoundResult; + if (thisTurn.some((m) => m.role === 'tool')) { + round = textRound(`Done: ${thisTurn.filter((m) => m.role === 'tool').length} results seen.`); + } else if (before.some((m) => m.role === 'tool' && /Event created/.test(textOf(m)))) { + round = textRound('Both events already exist.'); + } else { + round = createBothRound(turn, this.options.allowDuplicates === true); + } + callbacks.onTextDelta?.(round.text); + return round; + }; +} + +/** Lets every run that can make progress without a held request do so: two turns of the event loop. */ +export async function settle(): Promise { + for (let turn = 0; turn < 2; turn++) { + const { promise, resolve } = Promise.withResolvers(); + setImmediate(resolve); + await promise; + } +} + +export function titlesInCalendar(fixture: AgentRunFixture): string[] { + return ( + fixture.db.prepare('SELECT title FROM events WHERE user_id = ? AND is_deleted = 0 ORDER BY id').all(USER_ID) as { + title: string; + }[] + ).map((row) => row.title); +} + +/** What the chat showed, in order: sends and edits of every run. */ +export function recordDelivery(sender: TelegramSender): string[] { + const shown: string[] = []; + sender.sendMessage = async (_chatId, text) => { + shown.push(text); + return { message_id: 42 }; + }; + sender.editMessageText = async (_chatId, _messageId, text) => { + shown.push(text); + }; + return shown; +} diff --git a/test/regressions/canonical-basis-behavior.test.ts b/test/regressions/canonical-basis-behavior.test.ts index 07ea12e79..d77b1510b 100644 --- a/test/regressions/canonical-basis-behavior.test.ts +++ b/test/regressions/canonical-basis-behavior.test.ts @@ -250,6 +250,24 @@ describe('every example runs through the real executor and schema dispatch', () } }); + test('a confirmed create that finds the event already saved never says it was created', async () => { + const existing = (): ToolResult => ({ + success: true, + output: 'Event already exists: id: 21', + data: { ...STANDUP, id: 21 }, + mutationState: 'not_applied', + alreadyExists: { entity: 'event', id: 21 }, + }); + for (const name of ['basis.event.create', 'basis.event.create_invite']) { + const example = canonicalMetadata.find((meta) => meta.name === name)!.examples.synthetic[0]!; + const first = await runMessage(example, stubTools({ create_event: existing })); + const done = await answer(name, first, 'yes'); + expect(done.success, name).toBe(true); + expect(done.response, name).toContain('already in the calendar'); + expect(done.response, name).not.toMatch(/created|Done/); + } + }); + test('every private rule refuses or steps aside in a group before any tool runs', async () => { let checked = 0; for (const meta of canonicalMetadata) { diff --git a/test/services/ai/agent-chat-queue.test.ts b/test/services/ai/agent-chat-queue.test.ts new file mode 100644 index 000000000..212eed753 --- /dev/null +++ b/test/services/ai/agent-chat-queue.test.ts @@ -0,0 +1,328 @@ +// CalendarBotAgent.run() per chat: queued runs, a hung run, a run that waited, and the calendar's own +// repeat check standing alone when nothing orders the runs. Real agents, tools and database; the model is +// scripted (see helpers/overlap-fixtures.ts). +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { t } from '../../../src/config/constants.ts'; +import { turnIds } from '../../../src/services/ai/agent/queued-turn.ts'; +import { CalendarBotAgent } from '../../../src/services/ai/agent.ts'; +import { ChatRunGate, type ChatTurnReservation } from '../../../src/services/ai/chat-run-gate.ts'; +import { withChatTurn } from '../../../src/services/ai/chat-turn-context.ts'; +import type { StreamRoundOptions, StreamRoundResult } from '../../../src/services/ai/streaming.ts'; +import type { AgentContext } from '../../../src/services/ai/types.ts'; +import { ManualTime } from '../../helpers/manual-time.ts'; +import { + REVIEW, + recordDelivery, + SlowHost, + STANDUP, + settle, + textOf, + titlesInCalendar, +} from '../../helpers/overlap-fixtures.ts'; +import { + type AgentRunFixture, + createAgentRunFixture, + fakeMetrics, + isValidatorCall, + USER_ID, +} from './agent-run-helpers.ts'; + +describe('CalendarBotAgent.run() in one chat', () => { + let fixture: AgentRunFixture; + let host: SlowHost; + const reservations = new WeakMap(); + + beforeEach(() => { + fixture = createAgentRunFixture(); + host = new SlowHost(); + fixture.ctx.conversationLogger.logUserMessage(USER_ID, 'Plan the Synthetic Standup and the Synthetic Review'); + fixture.ctx.conversationLogger.logAiTurn(USER_ID, { role: 'assistant', content: 'Shall I add both events?' }); + }); + + afterEach(() => host.restore()); + + /** A turn as the chat-logging middleware leaves it: the user's row saved and its place in the chat reserved. */ + function turn(agent: CalendarBotAgent, text: string): AgentContext { + const chatHistoryId = fixture.ctx.conversationLogger.logUserMessage(USER_ID, text); + const context: AgentContext = { ...fixture.ctx, messageText: text, chatHistoryId }; + reservations.set(context, agent.reserveChatTurn(USER_ID, chatHistoryId)); + return context; + } + + /** What the middleware does around the handler: the run starts inside the update's reserved place. */ + function go(agent: CalendarBotAgent, context: AgentContext, source: AgentContext = context) { + const reservation = reservations.get(source); + return reservation ? withChatTurn(reservation, () => agent.run(context)) : agent.run(context); + } + + test('without any ordering between the runs, the calendar’s own repeat check still leaves one set of events', async () => { + // Two agents have two gates: nothing orders their runs, exactly as before the queue existed. + const slowAgent = new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: host.model }); + const otherAgent = new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: host.model }); + + const first = go(slowAgent, turn(slowAgent, 'Yes')); + await host.reached; + const second = go(otherAgent, turn(otherAgent, 'Yes')); + await settle(); + expect(titlesInCalendar(fixture)).toEqual([STANDUP.title, REVIEW.title]); // the unordered run created them + host.release(); + await Promise.all([first, second]); + + expect(titlesInCalendar(fixture)).toEqual([STANDUP.title, REVIEW.title]); + // The run that came second in time was told, per event, that it already existed. + const lateResults = host.requests + .flatMap((request) => request.messages) + .filter((message) => message.role === 'tool' && textOf(message).includes('already exists')); + expect(lateResults).toHaveLength(2); + }); + + test('a hung run holds the chat only until its lease; the late run still creates nothing twice', async () => { + const time = new ManualTime(); + const leaseMs = 120_000; + const agent = new CalendarBotAgent(fixture.config, fixture.sender, { + streamImpl: host.model, + chatRunGate: new ChatRunGate({ leaseMs, now: time.now, schedule: time.schedule }), + }); + + const hung = go(agent, turn(agent, 'Yes')); + await host.reached; // its model request is parked + const queued = go(agent, turn(agent, 'Yes again')); + await settle(); + expect(host.requests).toHaveLength(1); // the queued run has not reached the model: it is waiting + + time.advance(leaseMs); + await queued; // the wait ended; the run completed while the first is still parked + expect(titlesInCalendar(fixture)).toEqual([STANDUP.title, REVIEW.title]); + + host.release(); + await hung; + // The late run's creates found the events already there. + expect(titlesInCalendar(fixture)).toEqual([STANDUP.title, REVIEW.title]); + }); + + test('a run that waited reads the user and schedule as the earlier run left them', async () => { + const prompts: string[] = []; + const parked = Promise.withResolvers(); + const reached = Promise.withResolvers(); + let requests = 0; + const streamImpl = async (opts: StreamRoundOptions): Promise => { + const text = 'Noted.'; + const answer = { + text, + toolCalls: [], + finishReason: 'stop', + assistantMessage: { role: 'assistant' as const, content: text }, + providerUsed: 'mock', + metrics: fakeMetrics('zai', 10, 2), + }; + if (isValidatorCall(opts)) return { ...answer, text: 'APPROVE', metrics: fakeMetrics('hf', 10, 2) }; + prompts.push(textOf(opts.messages[0])); + requests++; + if (requests === 1) { + // While this run works the user changes their timezone (a settings tool, a button, an admin). + fixture.ctx.userRepo.update(USER_ID, { timezone: 'Pacific/Auckland' }); + reached.resolve(); + await parked.promise; + } + return answer; + }; + const agent = new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl }); + + const first = go(agent, turn(agent, 'first message')); + await reached.promise; + const second = go(agent, turn(agent, 'second message')); // its context still holds the old user + await settle(); + expect(prompts).toHaveLength(1); + parked.resolve(); + await Promise.all([first, second]); + + expect(prompts[0]).toContain('UTC'); + expect(prompts[0]).not.toContain('Pacific/Auckland'); + expect(prompts[1]).toContain('Pacific/Auckland'); + }); + + test('a queued message gets one answer of its own and its turn is saved once, after the earlier run', async () => { + const agent = new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: host.model }); + + const first = go(agent, turn(agent, 'Yes')); + await host.reached; + const second = go(agent, turn(agent, 'Yes, both')); + await settle(); + host.release(); + const [firstResult, secondResult] = await Promise.all([first, second]); + + expect(firstResult.responseText).toContain('Done: 2 results seen.'); + expect(secondResult.responseText).toContain('Both events already exist.'); + expect(secondResult.toolCalls).toEqual([]); + + const rows = fixture.ctx.chatHistory + .getRecent(USER_ID, 50) + .map((row) => (row.role === 'user' ? `user:${row.content}` : row.role)); + expect(rows.filter((row) => row === 'user:Yes')).toHaveLength(1); + expect(rows.filter((row) => row === 'user:Yes, both')).toHaveLength(1); + const at = rows.indexOf('user:Yes, both'); + expect(rows.slice(0, at)).toContain('tool'); // what the first run did comes before the second message + expect(rows.slice(at + 1)).toEqual(['assistant']); // and only the second run's own answer comes after it + }); + + describe('what the user sees while a message waits', () => { + function withTypingProbe() { + const actions: string[] = []; + fixture.sender.sendChatAction = async (chatId, action) => { + actions.push(`${chatId}:${action}`); + }; + return actions; + } + + test('a waiting message shows the typing indicator at once', async () => { + const actions = withTypingProbe(); + const agent = new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: host.model }); + + const first = go(agent, turn(agent, 'Yes')); + await host.reached; + const beforeQueued = actions.length; + const second = go(agent, turn(agent, 'Yes, both')); + expect(actions.length).toBe(beforeQueued + 1); + expect(actions.at(-1)).toBe(`${USER_ID}:typing`); + + host.release(); + await Promise.all([first, second]); + }); + + test('a supplement waits behind the main run like any run, silently: no typing, no text', async () => { + const actions = withTypingProbe(); + const shown = recordDelivery(fixture.sender); + const agent = new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: host.model }); + + const main = go(agent, turn(agent, 'Yes')); + await host.reached; + const beforeQueued = actions.length; + const supplementTurn = turn(agent, 'Yes, both'); + const supplement = go( + agent, + { ...supplementTurn, supplementMode: true, supplementAutoResponse: 'Done.' }, + supplementTurn, + ); + await settle(); + expect(actions.length).toBe(beforeQueued); // no typing for a supplement + expect(host.requests).toHaveLength(1); // and it has not started: it waits for the main run + + host.release(); + await Promise.all([main, supplement]); + expect(shown.some((text) => text.includes('already exist'))).toBe(false); // its answer never reached the chat + }); + + test('when the run ahead outlives the lease the user is told, once, and the run goes on', async () => { + const time = new ManualTime(); + const leaseMs = 90_000; + const shown = recordDelivery(fixture.sender); + const agent = new CalendarBotAgent(fixture.config, fixture.sender, { + streamImpl: host.model, + chatRunGate: new ChatRunGate({ leaseMs, now: time.now, schedule: time.schedule }), + }); + + const hung = go(agent, turn(agent, 'Yes')); + await host.reached; + const queued = go(agent, turn(agent, 'Yes, both')); + await settle(); + expect(shown).not.toContain(t('en').agent_queue_lease_expired); + + time.advance(leaseMs); + await queued; + expect(shown.filter((text) => text === t('en').agent_queue_lease_expired)).toHaveLength(1); + + host.release(); + await hung; + expect(shown.filter((text) => text === t('en').agent_queue_lease_expired)).toHaveLength(1); + }); + + test('a scheduled run (no message of the user behind it) is never told "handling your message"', async () => { + const time = new ManualTime(); + const leaseMs = 90_000; + const shown = recordDelivery(fixture.sender); + const agent = new CalendarBotAgent(fixture.config, fixture.sender, { + streamImpl: host.model, + chatRunGate: new ChatRunGate({ leaseMs, now: time.now, schedule: time.schedule }), + }); + const main = turn(agent, 'Yes'); + const hung = go(agent, main); + await host.reached; + // As the production context factory builds it: it borrows the user's latest row for the action log. + const scheduled = agent.run({ + ...fixture.ctx, + messageText: 'Scheduled check', + chatHistoryId: main.chatHistoryId, + }); + await settle(); + time.advance(leaseMs); + await scheduled; + expect(shown).not.toContain(t('en').agent_queue_lease_expired); + host.release(); + await hung; + // The borrowed row is still where it was saved: the earlier request is not made to look unanswered. + const roles = fixture.ctx.chatHistory + .getRecent(USER_ID, 50) + .map((row) => `${row.role}:${row.content.slice(0, 3)}`); + expect(roles.at(-1)).not.toBe('user:Yes'); + }); + + test('a message that waited and then ran after the earlier run finished gets no lease notice', async () => { + const shown = recordDelivery(fixture.sender); + const agent = new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: host.model }); + const first = go(agent, turn(agent, 'Yes')); + await host.reached; + const second = go(agent, turn(agent, 'Yes, both')); + await settle(); + host.release(); + await Promise.all([first, second]); + expect(shown).not.toContain(t('en').agent_queue_lease_expired); + }); + }); + + test('log ids name the Telegram update and message of the run', () => { + expect(turnIds({ updateId: 7, incomingMessageId: 8 })).toEqual({ updateId: 7, messageId: 8 }); + expect(turnIds({})).toEqual({}); + }); +}); + +describe('CalendarBotAgent shutdown with a stalled lease notice', () => { + test('a queued message whose notice never gets delivered is still drained and stored for retry', async () => { + const fixture = createAgentRunFixture(); + const host = new SlowHost(); + try { + const time = new ManualTime(); + const leaseMs = 90_000; + const retried: string[] = []; + fixture.ctx.retryEnqueue = async (message: string) => { + retried.push(message); + return true; + }; + // The notice is the only message this sender cannot deliver; everything else goes through. + const stalled = Promise.withResolvers<{ message_id: number }>(); + fixture.sender.sendMessage = async (_chatId, text) => + text === t('en').agent_queue_lease_expired ? stalled.promise : { message_id: 42 }; + const agent = new CalendarBotAgent(fixture.config, fixture.sender, { + streamImpl: host.model, + chatRunGate: new ChatRunGate({ leaseMs, now: time.now, schedule: time.schedule }), + }); + const runFor = (text: string) => { + const chatHistoryId = fixture.ctx.conversationLogger.logUserMessage(USER_ID, text); + const reservation = agent.reserveChatTurn(USER_ID, chatHistoryId); + return withChatTurn(reservation, () => agent.run({ ...fixture.ctx, messageText: text, chatHistoryId })); + }; + + const first = runFor('first request'); + await host.reached; + const second = runFor('second request'); + await settle(); + time.advance(leaseMs); // the second run leaves the queue and tries to tell the user + await settle(); + + await agent.drain(2_000); + await Promise.allSettled([first, second]); + expect([...retried].sort()).toEqual(['first request', 'second request']); + } finally { + host.restore(); + } + }); +}); diff --git a/test/services/ai/agent-shutdown-drain.test.ts b/test/services/ai/agent-shutdown-drain.test.ts index 3e6d3098a..ab1f0c168 100644 --- a/test/services/ai/agent-shutdown-drain.test.ts +++ b/test/services/ai/agent-shutdown-drain.test.ts @@ -232,6 +232,29 @@ describe('agent drain on shutdown', () => { expect(log).toContain('## END — no FINAL logged for this turn'); }); + test('a message queued behind a still-streaming run is released by the shutdown, drained and queued again', async () => { + const probe = makeSender(); + const { impl, secondRoundStarted } = stalledAfterTool({ + name: 'calculate', + input: { expression: '2026-09-28 20:30 Europe/Belgrade to UTC' }, + }); + const agent = new CalendarBotAgent({}, probe.sender, { streamImpl: impl }); + + const running = agent.run(ctx); + await secondRoundStarted; + // Same chat, so this run waits for the one above — it has not started and has said nothing. + const queued = agent.run({ ...ctx, messageText: 'And add the second one too' }); + + await agent.drain(DRAIN_BOUND_MS); + expect((await Promise.allSettled([running, queued])).map((outcome) => outcome.status)).toEqual([ + 'fulfilled', + 'fulfilled', + ]); + // Neither message is dropped: each is stored for a retry, and the user is told the bot is restarting. + expect([...enqueued].sort()).toEqual([REQUEST, 'And add the second one too'].sort()); + expect(probe.delivered.join('\n')).toContain(t('ru').agent_restarting(true)); + }); + test('a scheduled retry cut short by a restart is told so and queued again', async () => { ctx.retryAttempt = 1; const probe = makeSender(); diff --git a/test/services/ai/chat-run-gate.test.ts b/test/services/ai/chat-run-gate.test.ts new file mode 100644 index 000000000..f9d7183e5 --- /dev/null +++ b/test/services/ai/chat-run-gate.test.ts @@ -0,0 +1,510 @@ +// ChatRunGate: one run at a time per chat, in arrival order; other chats are not held back; a run that +// never finishes cannot freeze its chat. Time is a clock and a timer list the test moves, never real time. +import { beforeEach, describe, expect, test } from 'bun:test'; +import { + ChatRunGate, + type ChatRunSlot, + type ChatTurn, + type ChatTurnReservation, +} from '../../../src/services/ai/chat-run-gate.ts'; +import { ManualTime } from '../../helpers/manual-time.ts'; + +const LEASE_MS = 120_000; + +/** A run the test finishes by hand; `slot` is what the gate told it when it started. */ +function manualRun(gate: ChatRunGate, chatId: number, name: string, log: string[], turn: ChatTurn = {}) { + const finish = Promise.withResolvers(); + const started = Promise.withResolvers(); + const result = gate.run(chatId, turn, async (slot) => { + log.push(`start ${name}`); + started.resolve(slot); + const value = await finish.promise; + log.push(`end ${name}`); + return value; + }); + return { + result, + started: started.promise, + finish: (value = name) => finish.resolve(value), + fail: (error: Error) => finish.reject(error), + }; +} + +/** Lets every pending continuation run. */ +async function settle(): Promise { + const { promise, resolve } = Promise.withResolvers(); + setImmediate(resolve); + await promise; +} + +describe('ChatRunGate', () => { + let time: ManualTime; + let gate: ChatRunGate; + let log: string[]; + + beforeEach(() => { + time = new ManualTime(); + gate = new ChatRunGate({ leaseMs: LEASE_MS, now: time.now, schedule: time.schedule }); + log = []; + }); + + test('runs of one chat start one after another, in arrival order', async () => { + const first = manualRun(gate, 1, 'a', log); + const second = manualRun(gate, 1, 'b', log); + const third = manualRun(gate, 1, 'c', log); + await settle(); + expect(log).toEqual(['start a']); + + third.finish(); + second.finish(); + await settle(); + expect(log).toEqual(['start a']); // b and c still wait for a + + first.finish(); + await settle(); + expect(log).toEqual(['start a', 'end a', 'start b', 'end b', 'start c', 'end c']); + expect(await Promise.all([first.result, second.result, third.result])).toEqual(['a', 'b', 'c']); + }); + + test('another chat is not held back', async () => { + const slow = manualRun(gate, 1, 'slow', log); + const other = manualRun(gate, 2, 'other', log); + await settle(); + expect(log).toEqual(['start slow', 'start other']); + other.finish(); + await other.result; + slow.finish(); + await slow.result; + }); + + test('an idle chat starts a run synchronously and reports that it did not wait', async () => { + let startedSynchronously = false; + const result = gate.run(1, {}, async (slot) => { + startedSynchronously = true; + expect(slot.waited).toBe(false); + expect(slot.waitedMs).toBe(0); + return 'ok'; + }); + expect(startedSynchronously).toBe(true); + expect(await result).toBe('ok'); + }); + + test('a run that waited knows it and for how long', async () => { + const first = manualRun(gate, 1, 'a', log); + const second = manualRun(gate, 1, 'b', log); + await settle(); + time.advance(7_000); + first.finish(); + const slot = await second.started; + expect(slot.waited).toBe(true); + expect(slot.waitedMs).toBe(7_000); + second.finish(); + await second.result; + }); + + test('a failing run releases the chat to the next run and still rejects for its caller', async () => { + const first = manualRun(gate, 1, 'a', log); + const second = manualRun(gate, 1, 'b', log); + await settle(); + first.fail(new Error('model down')); + await expect(first.result).rejects.toThrow('model down'); + await settle(); + expect(log).toContain('start b'); + second.finish(); + expect(await second.result).toBe('b'); + }); + + test('a task that throws before returning a promise is a rejection and frees the chat', async () => { + const broken = gate.run(1, {}, () => { + throw new Error('broken task'); + }); + await expect(broken).rejects.toThrow('broken task'); + expect(gate.busyChats).toBe(0); + }); + + test('a run that never finishes holds its chat only for the lease', async () => { + const hung = manualRun(gate, 1, 'hung', log); + const next = manualRun(gate, 1, 'next', log); + const last = manualRun(gate, 1, 'last', log); + await settle(); + time.advance(LEASE_MS - 1); + await settle(); + expect(log).toEqual(['start hung']); + + time.advance(1); + await settle(); + expect(log).toEqual(['start hung', 'start next']); // next stopped waiting; last still waits for next + next.finish(); + await settle(); + expect(log).toContain('start last'); + + // The hung run finishing late changes nothing for the others and still resolves for its caller. + hung.finish(); + last.finish(); + expect(await Promise.all([hung.result, next.result, last.result])).toEqual(['hung', 'next', 'last']); + expect(gate.busyChats).toBe(0); + }); + + test('the lease of a run behind a hung one counts from when it began, not when it was queued', async () => { + const hung = manualRun(gate, 1, 'hung', log); + const second = manualRun(gate, 1, 'second', log); + const third = manualRun(gate, 1, 'third', log); + await settle(); + time.advance(LEASE_MS); + await settle(); + expect(log).toEqual(['start hung', 'start second']); + time.advance(LEASE_MS - 1); + await settle(); + expect(log).toEqual(['start hung', 'start second']); // third has waited long, but second began only a moment ago + time.advance(1); + await settle(); + expect(log).toEqual(['start hung', 'start second', 'start third']); + for (const run of [hung, second, third]) run.finish(); + await Promise.all([hung.result, second.result, third.result]); + }); + + test('waiting ends at once on shutdown, and a run queued after shutdown does not wait at all', async () => { + const shutdown = new AbortController(); + const running = manualRun(gate, 1, 'running', log); + const queued = gate.run(1, { abort: shutdown.signal }, async () => { + log.push('start queued'); + return 'queued'; + }); + await settle(); + expect(log).toEqual(['start running']); + shutdown.abort(new Error('Bot is shutting down')); + expect(await queued).toBe('queued'); + const late = gate.run(1, { abort: shutdown.signal }, async () => 'late'); + expect(await late).toBe('late'); + running.finish(); + await running.result; + expect(gate.busyChats).toBe(0); + }); + + test('a finished chat leaves nothing behind: no queue entry, no running timer', async () => { + const first = manualRun(gate, 1, 'a', log); + const second = manualRun(gate, 1, 'b', log); + await settle(); + expect(gate.busyChats).toBe(1); + first.finish(); + second.finish(); + await Promise.all([first.result, second.result]); + expect(gate.busyChats).toBe(0); + expect(time.pendingTimers).toBe(0); + }); + + test('a run is told which reserved turns are queued behind it, never its own', async () => { + const [r10, r11, r12] = [10, 11, 12].map((row) => gate.reserve(1, row)) as [ + ChatTurnReservation, + ChatTurnReservation, + ChatTurnReservation, + ]; + const r99 = gate.reserve(2, 99); + const first = manualRun(gate, 1, 'a', log, { reservation: r10 }); + const second = manualRun(gate, 1, 'b', log, { reservation: r11 }); + const third = manualRun(gate, 1, 'c', log, { reservation: r12 }); + const otherChat = manualRun(gate, 2, 'e', log, { reservation: r99 }); + const firstSlot = await first.started; + expect(firstSlot.ownRowId).toBe(10); + expect([...firstSlot.queuedTurnRowIds()].sort()).toEqual([11, 12]); + first.finish(); + const secondSlot = await second.started; + expect([...secondSlot.queuedTurnRowIds()]).toEqual([12]); + for (const run of [second, third, otherChat]) run.finish(); + await Promise.all([first.result, second.result, third.result, otherChat.result]); + }); + + test('a run without a reservation joins the end, owns no row and hides no one else’s', async () => { + const r10 = gate.reserve(1, 10); + const scheduled = manualRun(gate, 1, 'scheduled', log); // borrows a row id for its action log; the gate never sees it + await settle(); + expect(log).toEqual([]); // waits behind the place reserved for row 10 + const r11 = gate.reserve(1, 11); + const own = manualRun(gate, 1, 'own', log, { reservation: r10 }); + const ownSlot = await own.started; // the place of row 10 was ahead of the scheduled run + expect(ownSlot.ownRowId).toBe(10); + expect([...ownSlot.queuedTurnRowIds()]).toEqual([11]); + own.finish(); + await settle(); + expect(log).toEqual(['start own', 'end own', 'start scheduled']); // joined before row 11 was saved + r11.release(); + const slot = await scheduled.started; + expect(slot.ownRowId).toBeUndefined(); + scheduled.finish(); + await Promise.all([own.result, scheduled.result]); + }); + + test('a reservation made in another chat is not taken over', async () => { + const elsewhere = gate.reserve(2, 5); + const run = manualRun(gate, 1, 'a', log, { reservation: elsewhere }); + const slot = await run.started; + expect(slot.ownRowId).toBeUndefined(); + run.finish(); + await run.result; + elsewhere.release(); + }); + + describe('reserved places', () => { + test('a message whose handler reaches the gate late still runs in the order it was saved', async () => { + const first = gate.reserve(1, 1); + const second = gate.reserve(1, 2); + // The second message's handler is quicker: it reaches the gate first. + const later = manualRun(gate, 1, 'second', log, { reservation: second }); + await settle(); + expect(log).toEqual([]); // row 1 holds the first place + const earlier = manualRun(gate, 1, 'first', log, { reservation: first }); + await settle(); + expect(log).toEqual(['start first']); + earlier.finish(); + await settle(); + expect(log).toEqual(['start first', 'end first', 'start second']); + later.finish(); + await Promise.all([earlier.result, later.result]); + }); + + test('the run for a row sees a message saved after it as queued behind it, handler order notwithstanding', async () => { + const r1 = gate.reserve(1, 1); + const r2 = gate.reserve(1, 2); + const second = manualRun(gate, 1, 'second', log, { reservation: r2 }); + const first = manualRun(gate, 1, 'first', log, { reservation: r1 }); + const slot = await first.started; + expect([...slot.queuedTurnRowIds()]).toEqual([2]); + first.finish(); + second.finish(); + await Promise.all([first.result, second.result]); + }); + + test('a place released without a run (an intent answered, a command) lets the run behind it go', async () => { + const intent = gate.reserve(1, 1); + const r2 = gate.reserve(1, 2); + const agent = manualRun(gate, 1, 'agent', log, { reservation: r2 }); + await settle(); + expect(log).toEqual([]); + intent.release(); + await settle(); + expect(log).toEqual(['start agent']); + agent.finish(); + await agent.result; + }); + + test('a place nobody uses is given up after the reservation time', async () => { + gate.reserve(1, 1); + const behind = manualRun(gate, 1, 'behind', log); + await settle(); + expect(log).toEqual([]); + time.advance(60_000); + await settle(); + expect(log).toEqual(['start behind']); + behind.finish(); + await behind.result; + expect(gate.busyChats).toBe(0); + expect(time.pendingTimers).toBe(0); + }); + + test('releasing after the run took the place over changes nothing', async () => { + const reservation = gate.reserve(1, 1); + const run = manualRun(gate, 1, 'a', log, { reservation }); + const behind = manualRun(gate, 1, 'b', log); + await settle(); + reservation.release(); + await settle(); + expect(log).toEqual(['start a']); + run.finish(); + behind.finish(); + await Promise.all([run.result, behind.result]); + }); + + test('a second run of the same update does not take the place twice', async () => { + const reservation = gate.reserve(1, 1); + const main = manualRun(gate, 1, 'main', log, { reservation }); + const again = manualRun(gate, 1, 'again', log, { reservation }); + await settle(); + expect(log).toEqual(['start main']); // the second joins the end, behind the first + main.finish(); + const slot = await again.started; + expect(slot.ownRowId).toBeUndefined(); + again.finish(); + await Promise.all([main.result, again.result]); + }); + + test('a run whose place expired still owns its message and is placed behind the runs in between', async () => { + const time = new ManualTime(); + const expiring = new ChatRunGate({ leaseMs: LEASE_MS, reservationMs: 1000, now: time.now, schedule: time.schedule }); + const reservation = expiring.reserve(1, 10); + time.advance(1000); + const slot = await expiring.run(1, { reservation }, async (s) => s); + expect(slot.ownRowId).toBe(10); + const second = await expiring.run(1, { reservation }, async (s) => s); + expect(second.ownRowId).toBeUndefined(); + }); + + test('a released place does not let a run behind it overtake a run still working ahead of it', async () => { + const working = manualRun(gate, 1, 'working', log); + const intent = gate.reserve(1, 5); + const behind = manualRun(gate, 1, 'behind', log); + await settle(); + intent.release(); + await settle(); + expect(log).toEqual(['start working']); + working.finish(); + await settle(); + expect(log).toEqual(['start working', 'end working', 'start behind']); + behind.finish(); + await Promise.all([working.result, behind.result]); + }); + + test('a throwing waiting signal does not stop the run', async () => { + const working = manualRun(gate, 1, 'working', log); + const waiting = gate.run( + 1, + { + whileWaiting: () => { + throw new Error('synthetic typing failure'); + }, + }, + async () => 'ran', + ); + working.finish(); + expect(await waiting).toBe('ran'); + await working.result; + expect(gate.busyChats).toBe(0); + }); + }); +}); + +describe('ChatRunGate waiting signals', () => { + const LEASE = 60_000; + + function fixture() { + const time = new ManualTime(); + return { time, gate: new ChatRunGate({ leaseMs: LEASE, now: time.now, schedule: time.schedule }) }; + } + + test('whileWaiting starts only for a run that has to wait and stops when its wait ends', async () => { + const { gate } = fixture(); + const events: string[] = []; + const hold = Promise.withResolvers(); + const first = gate.run(1, { whileWaiting: () => () => events.push('first-stop') }, () => hold.promise); + const second = gate.run( + 1, + { + whileWaiting: () => { + events.push('second-wait'); + return () => events.push('second-stop'); + }, + }, + async () => { + events.push('second-run'); + }, + ); + expect(gate.isBusy(1)).toBe(true); + expect(gate.isBusy(2)).toBe(false); + expect(events).toEqual(['second-wait']); + hold.resolve(); + await Promise.all([first, second]); + expect(events).toEqual(['second-wait', 'second-stop', 'second-run']); + expect(gate.isBusy(1)).toBe(false); + }); + + test('the wait hook is stopped when the wait ends by shutdown too', async () => { + const { gate } = fixture(); + const shutdown = new AbortController(); + const hold = Promise.withResolvers(); + const stops: string[] = []; + void gate.run(1, {}, () => hold.promise); + const queued = gate.run( + 1, + { abort: shutdown.signal, whileWaiting: () => () => stops.push('stopped') }, + async () => 'ran', + ); + shutdown.abort(new Error('Bot is shutting down')); + expect(await queued).toBe('ran'); + expect(stops).toEqual(['stopped']); + hold.resolve(); + }); + + test('leaseExpired is true only when the run ahead was still holding the chat at the lease', async () => { + const { time, gate } = fixture(); + const hold = Promise.withResolvers(); + void gate.run(1, {}, () => hold.promise); + const slots: boolean[] = []; + const late = gate.run(1, {}, async (slot) => { + slots.push(slot.leaseExpired); + }); + time.advance(LEASE); + await late; + + const quick = Promise.withResolvers(); + const done = gate.run(2, {}, () => quick.promise); + const after = gate.run(2, {}, async (slot) => { + slots.push(slot.leaseExpired); + }); + quick.resolve(); + await Promise.all([done, after]); + const idle = await gate.run(3, {}, async (slot) => slot.leaseExpired); + expect([...slots, idle]).toEqual([true, false, false]); + hold.resolve(); + }); + + test('a run that starts beside a hung run of the chat knows it, however many runs sit between', async () => { + const { time, gate } = fixture(); + const hung = Promise.withResolvers(); + void gate.run(1, {}, () => hung.promise); + const bFinish = Promise.withResolvers(); + const flags: Record = {}; + const b = gate.run(1, {}, async (slot) => { + flags.b = slot.leaseExpired; + await bFinish.promise; + }); + const c = gate.run(1, {}, async (slot) => { + flags.c = slot.leaseExpired; + }); + time.advance(LEASE); + await settle(); + bFinish.resolve(); + await Promise.all([b, c]); + expect(flags).toEqual({ b: true, c: true }); + hung.resolve(); + }); + + test('a hung run is announced once, not on every later message', async () => { + const { time, gate } = fixture(); + const hung = Promise.withResolvers(); + void gate.run(1, {}, () => hung.promise); + const first = gate.run(1, { reservation: gate.reserve(1, 1) }, async (slot) => slot.leaseExpired); + time.advance(LEASE); + expect(await first).toBe(true); + const second = await gate.run(1, { reservation: gate.reserve(1, 2) }, async (slot) => slot.leaseExpired); + expect(second).toBe(false); + hung.resolve(); + }); + + test('a run without a message of its own does not use up the notice of a hung run', async () => { + const { time, gate } = fixture(); + const hung = Promise.withResolvers(); + void gate.run(1, {}, () => hung.promise); + const scheduled = gate.run(1, {}, async (slot) => slot.leaseExpired); + const reservation = gate.reserve(1, 7); + const user = gate.run(1, { reservation }, async (slot) => slot.leaseExpired); + time.advance(LEASE); + expect(await scheduled).toBe(true); + expect(await user).toBe(true); + hung.resolve(); + }); + + test('shutdown while runs queue behind a hung one leaves no lease timer running', async () => { + const { time, gate } = fixture(); + const shutdown = new AbortController(); + const hung = Promise.withResolvers(); + void gate.run(1, {}, () => hung.promise); + const bFinish = Promise.withResolvers(); + const b = gate.run(1, { abort: shutdown.signal }, () => bFinish.promise); + const c = gate.run(1, { abort: shutdown.signal }, async () => {}); + shutdown.abort(new Error('Bot is shutting down')); + await settle(); + expect(time.pendingTimers).toBe(0); + bFinish.resolve(); + hung.resolve(); + await Promise.all([b, c]); + }); +}); diff --git a/test/services/ai/tool-handlers/events-duplicate-create.test.ts b/test/services/ai/tool-handlers/events-duplicate-create.test.ts new file mode 100644 index 000000000..731236b17 --- /dev/null +++ b/test/services/ai/tool-handlers/events-duplicate-create.test.ts @@ -0,0 +1,595 @@ +// create_event is idempotent for a repeat of the same request: the same owner asking again, within +// minutes, for an event with the same title, start and end gets the event that already exists back as +// a successful receipt instead of a second row (#520, #929 — the 2026-11-01 duplicates). +import { Database } from 'bun:sqlite'; +import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { migrations } from '../../../../src/database/migrations.ts'; +import { ChatHistoryRepository } from '../../../../src/database/repositories/chat-history.repository.ts'; +import { EventRepository } from '../../../../src/database/repositories/event.repository.ts'; +import { EventReminderRepository } from '../../../../src/database/repositories/event-reminder.repository.ts'; +import { HolidayRepository } from '../../../../src/database/repositories/holiday.repository.ts'; +import { NotificationPreferencesRepository } from '../../../../src/database/repositories/notification-preferences.repository.ts'; +import { SecretaryRepository } from '../../../../src/database/repositories/secretary.repository.ts'; +import { UserRepository } from '../../../../src/database/repositories/user.repository.ts'; +import { runMigrations } from '../../../../src/database/schema.ts'; +import { _resetToolThrottleForTest, executeTool } from '../../../../src/services/ai/tool-executor.ts'; +import { handleCreateEvent } from '../../../../src/services/ai/tool-handlers/events.ts'; +import type { AgentContext, ToolResult } from '../../../../src/services/ai/types.ts'; +import { ConversationLogger } from '../../../../src/services/conversation-logger.ts'; +import { EventService } from '../../../../src/services/event/event-service.ts'; +import { HolidayService } from '../../../../src/services/holiday/holiday-service.ts'; +import { ReminderMaterializer } from '../../../../src/services/notification/materializer.ts'; +import { partialOf } from '../partial-stubs.ts'; +import { createEventToolFixture, type EventToolFixture, USER_ID } from './events-fixtures.ts'; + +/** The id of the event a create_event result describes. */ +function eventIdOf(result: ToolResult): number { + const data = result.data; + if (data && typeof data === 'object' && 'id' in data && typeof data.id === 'number') return data.id; + throw new Error('create_event result carries no event id'); +} + +const STANDUP = { title: 'Synthetic Standup', start_at: '2030-03-05T10:00:00Z', end_at: '2030-03-05T10:30:00Z' }; + +describe('create_event repeated within minutes', () => { + let db: EventToolFixture['db']; + let ctx: AgentContext; + + beforeEach(() => { + _resetToolThrottleForTest(); + ({ db, ctx } = createEventToolFixture()); + }); + + const rows = (userId = USER_ID) => + db.prepare('SELECT id, title FROM events WHERE user_id = ? AND is_deleted = 0 ORDER BY id').all(userId) as { + id: number; + title: string; + }[]; + + /** What a later run of the same chat looks like: its own context, sharing the calendar. */ + const laterRun = (): AgentContext => ({ ...ctx, createdEventIds: undefined }); + + const ageEvents = (minutes: number) => + db.prepare("UPDATE events SET created_at = datetime('now', ?)").run(`-${minutes} minutes`); + + test('returns the existing event as a successful receipt and keeps one row', async () => { + const first = await handleCreateEvent(ctx, { ...STANDUP }); + const second = await handleCreateEvent(laterRun(), { ...STANDUP }); + + expect(first.success).toBe(true); + expect(second.success).toBe(true); + expect(second.output).toContain('already exists'); + expect(second.data).toMatchObject({ title: STANDUP.title }); + expect(rows()).toHaveLength(1); + expect(eventIdOf(second)).toBe(eventIdOf(first)); + expect(rows().map((row) => row.id)).toEqual([eventIdOf(second)]); + expect(second.agentHint).toContain('Do NOT call create_event again'); + }); + + test('the receipt tells how to make a deliberate second copy, and that flag creates it', async () => { + await handleCreateEvent(ctx, { ...STANDUP }); + const repeat = await handleCreateEvent(laterRun(), { ...STANDUP }); + expect(repeat.agentHint).toContain('allow_duplicate'); + + const deliberate = await handleCreateEvent(laterRun(), { ...STANDUP, allow_duplicate: true }); + expect(deliberate.success).toBe(true); + expect(deliberate.output).not.toContain('already exists'); + expect(rows()).toHaveLength(2); + }); + + test('titles match after trimming, collapsing spaces and folding case, Cyrillic included', async () => { + await handleCreateEvent(ctx, { title: 'Встреча с Леной', start_at: '2030-03-06T09:00:00Z' }); + const again = await handleCreateEvent(laterRun(), { + title: ' встреча С леной ', + start_at: '2030-03-06T09:00:00Z', + }); + expect(again.output).toContain('already exists'); + expect(rows()).toHaveLength(1); + }); + + test('the same instant spelled with another offset is the same event', async () => { + await handleCreateEvent(ctx, { title: 'Offset', start_at: '2030-03-06T09:00:00Z', end_at: '2030-03-06T10:00:00Z' }); + const again = await handleCreateEvent(laterRun(), { + title: 'Offset', + start_at: '2030-03-06T11:00:00+02:00', + end_at: '2030-03-06T12:00:00+02:00', + }); + expect(again.output).toContain('already exists'); + expect(rows()).toHaveLength(1); + }); + + test.each([ + ['another title', { title: 'Synthetic Review' }], + ['another day', { start_at: '2030-03-06T10:00:00Z', end_at: '2030-03-06T10:30:00Z' }], + ])('%s is a different event and is created', async (_name, change) => { + await handleCreateEvent(ctx, { ...STANDUP }); + const other = await handleCreateEvent(laterRun(), { ...STANDUP, ...change }); + expect(other.success).toBe(true); + expect(other.output).not.toContain('already exists'); + expect(rows()).toHaveLength(2); + }); + + // #520: the same title on the same local day at another time is a move done by creating, or a second copy. + test.each([ + ['another start', { start_at: '2030-03-05T11:00:00Z', end_at: '2030-03-05T11:30:00Z' }], + ['another end', { end_at: '2030-03-05T11:00:00Z' }], + ['no end', { end_at: undefined }], + ])('%s on the same day is refused with a pointer to update_event', async (_name, change) => { + const first = await handleCreateEvent(ctx, { ...STANDUP }); + const other = await handleCreateEvent(laterRun(), { ...STANDUP, ...change }); + expect(other.success).toBe(false); + expect(other.mutationState).toBe('not_applied'); + expect(other.error).toContain(`id: ${eventIdOf(first)}`); + expect(other.error).toContain('nothing was created'); + expect(other.error).not.toContain('update_event'); // the message is for the person; the route is for the model + expect(other.agentHint).toContain('update_event'); + expect(other.agentHint).toContain('allow_duplicate'); + expect(rows()).toHaveLength(1); + }); + + test('allow_duplicate confirms a same-day second event', async () => { + await handleCreateEvent(ctx, { ...STANDUP }); + const second = await handleCreateEvent(laterRun(), { + ...STANDUP, + start_at: '2030-03-05T16:00:00Z', + end_at: '2030-03-05T16:30:00Z', + allow_duplicate: true, + }); + expect(second.success).toBe(true); + expect(rows()).toHaveLength(2); + }); + + test('the same day is the user’s local day, not the UTC date', async () => { + const la: AgentContext = { ...ctx, user: { ...ctx.user, timezone: 'America/Los_Angeles' } }; + // 2030-03-06T02:00Z is still 5 March in Los Angeles; 2030-03-05T10:00Z is 5 March there too. + await handleCreateEvent(la, { title: 'Late Call', start_at: '2030-03-06T02:00:00Z' }); + const sameLocalDay = await handleCreateEvent(la, { title: 'Late Call', start_at: '2030-03-05T18:00:00Z' }); + expect(sameLocalDay.success).toBe(false); + const nextLocalDay = await handleCreateEvent(la, { title: 'Late Call', start_at: '2030-03-06T18:00:00Z' }); + expect(nextLocalDay.success).toBe(true); + expect(rows()).toHaveLength(2); + }); + + test('a repeat that states different details is not reported as already existing', async () => { + await handleCreateEvent(ctx, { ...STANDUP }); + const withRoom = await handleCreateEvent(laterRun(), { ...STANDUP, location: 'Room 5' }); + expect(withRoom.success).toBe(false); + expect(withRoom.agentHint).toContain('update_event'); + const withReminder = await handleCreateEvent(laterRun(), { ...STANDUP, reminder_minutes: [60] }); + expect(withReminder.success).toBe(false); + expect(rows()).toHaveLength(1); + }); + + test('a repeat that states the same details is still the same request', async () => { + const detailed = { ...STANDUP, location: 'Room 5', description: 'Weekly', reminder_minutes: [15, 60] }; + await handleCreateEvent(ctx, detailed); + const again = await handleCreateEvent(laterRun(), { ...detailed, location: ' Room 5 ' }); + expect(again.success).toBe(true); + expect(again.output).toContain('already exists'); + expect(rows()).toHaveLength(1); + }); + + test('an all-day event and a timed one at the same moment are different events', async () => { + await handleCreateEvent(ctx, { title: 'Holiday', start_at: '2030-03-05', all_day: true }); + await handleCreateEvent(laterRun(), { title: 'Holiday', start_at: '2030-03-05T00:00:00Z' }); + expect(rows()).toHaveLength(2); + }); + + test('a deleted or cancelled event does not stand in for a new one', async () => { + const created = await handleCreateEvent(ctx, { ...STANDUP }); + db.prepare('UPDATE events SET is_deleted = 1 WHERE id = ?').run(eventIdOf(created)); + const again = await handleCreateEvent(laterRun(), { ...STANDUP }); + expect(again.output).not.toContain('already exists'); + db.prepare('UPDATE events SET is_cancelled = 1 WHERE id = ?').run(eventIdOf(again)); + const third = await handleCreateEvent(laterRun(), { ...STANDUP }); + expect(third.output).not.toContain('already exists'); + expect(rows()).toHaveLength(2); // the deleted one is hidden; the cancelled one and the new one remain + }); + + test('an identical event created longer ago than the window is not a repeat but still needs confirmation', async () => { + await handleCreateEvent(ctx, { ...STANDUP }); + ageEvents(11); + const again = await handleCreateEvent(laterRun(), { ...STANDUP }); + expect(again.success).toBe(false); + expect(again.agentHint).toContain('SAME_DAY_EVENT'); + expect(again.output).toBeUndefined(); + expect(rows()).toHaveLength(1); + const confirmed = await handleCreateEvent(laterRun(), { ...STANDUP, allow_duplicate: true }); + expect(confirmed.success).toBe(true); + expect(rows()).toHaveLength(2); + }); + + test('an event created a few minutes ago is still matched', async () => { + await handleCreateEvent(ctx, { ...STANDUP }); + ageEvents(4); + const again = await handleCreateEvent(laterRun(), { ...STANDUP }); + expect(again.output).toContain('already exists'); + expect(rows()).toHaveLength(1); + }); + + test('recurring events are never matched, whichever side is recurring', async () => { + await handleCreateEvent(ctx, { ...STANDUP, recurrence_rule: 'FREQ=WEEKLY' }); + await handleCreateEvent(laterRun(), { ...STANDUP, recurrence_rule: 'FREQ=WEEKLY' }); + await handleCreateEvent(laterRun(), { ...STANDUP }); + expect(rows()).toHaveLength(3); + const once = await handleCreateEvent(laterRun(), { ...STANDUP }); + expect(once.output).toContain('already exists'); // the one-off from the line above, not the recurring ones + expect(rows()).toHaveLength(3); + }); + + test('another owner’s identical event is not matched', async () => { + const other = 888; + ctx.userRepo.create({ telegram_id: other, timezone: 'UTC' }); + await handleCreateEvent(ctx, { ...STANDUP }); + const otherCtx: AgentContext = { ...laterRun(), user: ctx.userRepo.findByTelegramId(other)!, chatId: other }; + const result = await handleCreateEvent(otherCtx, { ...STANDUP }); + expect(result.output).not.toContain('already exists'); + expect(rows(USER_ID)).toHaveLength(1); + expect(rows(other)).toHaveLength(1); + }); + + test('a secretary creating for an owner is matched against the owner’s calendar', async () => { + const owner = 999; + ctx.userRepo.create({ telegram_id: owner, timezone: 'UTC' }); + const secretaryRepo = new SecretaryRepository(db); + const grant = secretaryRepo.upsert({ owner_id: owner, secretary_id: USER_ID, permission: 'write' }); + secretaryRepo.updateStatus(grant.id, 'active'); + const secretaryCtx: AgentContext = { + ...ctx, + secretary: partialOf>('secretary', { secretaryRepo }), + }; + await handleCreateEvent(secretaryCtx, { ...STANDUP, owner_id: owner }); + const again = await handleCreateEvent( + { ...secretaryCtx, createdEventIds: undefined }, + { ...STANDUP, owner_id: owner }, + ); + expect(again.output).toContain('already exists'); + expect(rows(owner)).toHaveLength(1); + expect(rows(USER_ID)).toHaveLength(0); + // the owner creating the same thing by themselves is the same calendar too + const ownerCtx: AgentContext = { ...ctx, user: ctx.userRepo.findByTelegramId(owner)!, chatId: owner }; + const byOwner = await handleCreateEvent(ownerCtx, { ...STANDUP }); + expect(byOwner.output).toContain('already exists'); + expect(rows(owner)).toHaveLength(1); + }); + + describe('in a group', () => { + const GROUP = -100777; + const groupCtx = (groupChatId: number): AgentContext => ({ + ...ctx, + isGroup: true, + groupChatId, + chatId: groupChatId, + groupTitle: 'Synthetic Group', + }); + + test('is compared inside the group only', async () => { + await handleCreateEvent(groupCtx(GROUP), { ...STANDUP, scope: 'group' }); + const sameGroup = await handleCreateEvent(groupCtx(GROUP), { ...STANDUP, scope: 'group' }); + expect(sameGroup.output).toContain('already exists'); + const otherGroup = await handleCreateEvent(groupCtx(GROUP - 1), { ...STANDUP, scope: 'group' }); + expect(otherGroup.output).not.toContain('already exists'); + const personal = await handleCreateEvent(ctx, { ...STANDUP }); + expect(personal.output).not.toContain('already exists'); + const groupRows = db.prepare("SELECT id FROM events WHERE owner_type = 'group' AND is_deleted = 0").all(); + expect(groupRows).toHaveLength(2); + }); + }); + + test('a repeat is not a second creation: no reminders, notifications, calendar push or delete rights', async () => { + const reminders = mock(() => {}); + const emitted = mock(() => {}); + const push = mock(async () => {}); + const eventRepo = new EventRepository(db); + const service = new EventService({ + eventRepo, + materializer: partialOf[0]['materializer']>>( + 'materializer', + { materialize: reminders }, + ), + domainEvents: partialOf[0]['domainEvents']>>('bus', { + emit: emitted, + }), + }); + const wired: AgentContext = { + ...ctx, + eventService: service, + google: partialOf>('google', { schedulePush: push }), + }; + await handleCreateEvent(wired, { ...STANDUP }); + expect([reminders.mock.calls.length, emitted.mock.calls.length, push.mock.calls.length]).toEqual([1, 1, 1]); + + const later = { ...wired, createdEventIds: undefined }; + const repeat = await handleCreateEvent(later, { ...STANDUP }); + expect(repeat.output).toContain('already exists'); + expect([reminders.mock.calls.length, emitted.mock.calls.length, push.mock.calls.length]).toEqual([1, 1, 1]); + expect(later.createdEventIds).toBeUndefined(); // the event is not this run's creation + }); + + test('the executor accepts allow_duplicate and rejects a non-boolean', async () => { + await executeTool(ctx, 'create_event', { ...STANDUP }); + _resetToolThrottleForTest(); + const repeat = await executeTool(laterRun(), 'create_event', { ...STANDUP }); + expect(repeat.success).toBe(true); + expect(repeat.mutationState).toBe('not_applied'); // nothing was written by this call + expect(repeat.output).toContain('already exists'); + _resetToolThrottleForTest(); + const deliberate = await executeTool(laterRun(), 'create_event', { ...STANDUP, allow_duplicate: 'true' }); + expect(deliberate.success).toBe(true); + expect(rows()).toHaveLength(2); + _resetToolThrottleForTest(); + const invalid = await executeTool(laterRun(), 'create_event', { ...STANDUP, allow_duplicate: 'maybe' }); + expect(invalid.success).toBe(false); + expect(rows()).toHaveLength(2); + }); +}); + +describe('create_event repeated across connections and restarts', () => { + let dir: string; + + beforeEach(() => { + _resetToolThrottleForTest(); + dir = mkdtempSync(path.join(tmpdir(), 'hcb-duplicate-create-')); + }); + + afterEach(() => rmSync(dir, { recursive: true, force: true })); + + /** A bot process: its own connection, repositories and services over the same database file. */ + function startProcess(file: string): { ctx: AgentContext; db: Database } { + const db = new Database(file); + db.exec('PRAGMA foreign_keys = ON'); + db.exec('PRAGMA journal_mode = WAL'); + runMigrations(db, migrations); + const userRepo = new UserRepository(db); + if (!userRepo.findByTelegramId(USER_ID)) userRepo.create({ telegram_id: USER_ID, timezone: 'UTC' }); + const eventRepo = new EventRepository(db); + const chatHistoryRepo = new ChatHistoryRepository(db); + const ctx: AgentContext = { + user: userRepo.findByTelegramId(USER_ID)!, + chatId: USER_ID, + messageText: '', + isGroup: false, + eventService: new EventService({ eventRepo }), + holidayService: new HolidayService(new HolidayRepository(db)), + chatHistory: chatHistoryRepo, + userRepo, + eventReminderRepo: new EventReminderRepository(db), + conversationLogger: new ConversationLogger(chatHistoryRepo), + }; + return { ctx, db }; + } + + test('a process that starts later still sees the event created by the one before it', async () => { + const file = path.join(dir, 'calendar.db'); + const before = startProcess(file); + await handleCreateEvent(before.ctx, { ...STANDUP }); + before.db.close(); + + const after = startProcess(file); + const repeat = await handleCreateEvent(after.ctx, { ...STANDUP }); + expect(repeat.output).toContain('already exists'); + expect(after.db.prepare('SELECT COUNT(*) AS n FROM events').get()).toEqual({ n: 1 }); + after.db.close(); + }); + + test('two connections creating the same event hold one row between them', async () => { + const file = path.join(dir, 'calendar.db'); + const a = startProcess(file); + const b = startProcess(file); + const results = await Promise.all([ + handleCreateEvent(a.ctx, { ...STANDUP }), + handleCreateEvent(b.ctx, { ...STANDUP }), + ]); + expect(results.every((result) => result.success)).toBe(true); + expect(results.filter((result) => result.output?.includes('already exists'))).toHaveLength(1); + expect(a.db.prepare('SELECT COUNT(*) AS n FROM events').get()).toEqual({ n: 1 }); + a.db.close(); + b.db.close(); + }); +}); + +describe('create_event all-day repeats', () => { + test('an all-day event is the same event whether its start came as local midnight or as a date', async () => { + _resetToolThrottleForTest(); + const { db, ctx } = createEventToolFixture(); + const belgrade: AgentContext = { ...ctx, user: { ...ctx.user, timezone: 'Europe/Belgrade' } }; + const first = await handleCreateEvent(belgrade, { + title: 'Synthetic Holiday', + start_at: '2030-03-04T23:00:00Z', + all_day: true, + }); + const again = await handleCreateEvent(belgrade, { + title: 'Synthetic Holiday', + start_at: '2030-03-05', + all_day: true, + }); + const other = await handleCreateEvent(belgrade, { + title: 'Synthetic Holiday', + start_at: '2030-03-06', + all_day: true, + }); + + expect(first.output).not.toContain('already exists'); + expect(again.output).toContain('already exists'); + expect(other.output).not.toContain('already exists'); + expect(db.prepare('SELECT COUNT(*) AS n FROM events').get()).toEqual({ n: 2 }); + }); + + const LA = (ctx: AgentContext): AgentContext => ({ ...ctx, user: { ...ctx.user, timezone: 'America/Los_Angeles' } }); + + test('a floating all-day date is the same day in a zone west of UTC', async () => { + _resetToolThrottleForTest(); + const { db, ctx } = createEventToolFixture(); + const la = LA(ctx); + // A UTC midnight is the floating date 2030-03-05 everywhere, not 4 March at 16:00 in Los Angeles. + await handleCreateEvent(la, { title: 'Floating Day', start_at: '2030-03-05T00:00:00Z', all_day: true }); + const sameDate = await handleCreateEvent(la, { title: 'Floating Day', start_at: '2030-03-05', all_day: true }); + expect(sameDate.success).toBe(true); + expect(sameDate.output).toContain('already exists'); + expect(db.prepare('SELECT COUNT(*) AS n FROM events').get()).toEqual({ n: 1 }); + + const dayBefore = await handleCreateEvent(la, { title: 'Floating Day', start_at: '2030-03-04', all_day: true }); + expect(dayBefore.success).toBe(true); + expect(dayBefore.output).not.toContain('already exists'); + expect(db.prepare('SELECT COUNT(*) AS n FROM events').get()).toEqual({ n: 2 }); + }); + + test('the other way round: a date-only event, then the same date as a UTC midnight', async () => { + _resetToolThrottleForTest(); + const { db, ctx } = createEventToolFixture(); + const la = LA(ctx); + await handleCreateEvent(la, { title: 'Floating Day', start_at: '2030-03-05', all_day: true }); + const sameDate = await handleCreateEvent(la, { + title: 'Floating Day', + start_at: '2030-03-05T00:00:00Z', + all_day: true, + }); + expect(sameDate.output).toContain('already exists'); + const dayBefore = await handleCreateEvent(la, { + title: 'Floating Day', + start_at: '2030-03-04T00:00:00Z', + all_day: true, + }); + expect(dayBefore.output).not.toContain('already exists'); + expect(db.prepare('SELECT COUNT(*) AS n FROM events').get()).toEqual({ n: 2 }); + }); +}); + +describe('create_event next to events imported from another Google calendar', () => { + test('an identical event of a secondary calendar does not stand in for a personal one', async () => { + _resetToolThrottleForTest(); + const { db, ctx } = createEventToolFixture(); + new EventRepository(db).insertSyncedEvent({ + user_id: USER_ID, + title: STANDUP.title, + description: null, + start_at: STANDUP.start_at, + end_at: STANDUP.end_at, + all_day: false, + timezone: 'UTC', + location: null, + recurrence_rule: null, + google_calendar_id: 'team@group.calendar.example', + google_event_id: 'synthetic-team-1', + google_etag: null, + is_cancelled: false, + }); + const personal = await handleCreateEvent(ctx, { ...STANDUP }); + expect(personal.success).toBe(true); + expect(personal.output).not.toContain('already exists'); + const rows = db.prepare('SELECT id, google_calendar_id FROM events ORDER BY id').all(); + expect(rows).toHaveLength(2); + expect(eventIdOf(personal)).toBe(2); + + const repeat = await handleCreateEvent({ ...ctx, createdEventIds: undefined }, { ...STANDUP }); + expect(repeat.output).toContain('already exists'); + expect(eventIdOf(repeat)).toBe(2); + }); +}); + +describe('create_event next to the user’s primary Google calendar', () => { + test('an event imported under the primary calendar’s concrete id is the same calendar', async () => { + _resetToolThrottleForTest(); + const { db, ctx } = createEventToolFixture(); + db.prepare( + "INSERT INTO google_calendars (user_id, google_calendar_id, calendar_name, is_primary, access_role) VALUES (?, 'owner@example.test', 'Synthetic', 1, 'owner')", + ).run(USER_ID); + const eventRepo = new EventRepository(db); + const importEvent = (calendar: string, id: string) => + eventRepo.insertSyncedEvent({ + user_id: USER_ID, + title: STANDUP.title, + description: null, + start_at: STANDUP.start_at, + end_at: STANDUP.end_at, + all_day: false, + timezone: 'UTC', + location: null, + recurrence_rule: null, + google_calendar_id: calendar, + google_event_id: id, + google_etag: null, + is_cancelled: false, + }); + importEvent('owner@example.test', 'synthetic-primary-1'); + const same = await handleCreateEvent(ctx, { ...STANDUP }); + expect(same.output).toContain('already exists'); + expect(db.prepare('SELECT COUNT(*) AS n FROM events').get()).toEqual({ n: 1 }); + }); +}); + +describe('create_event all-day dates read in the zone each event was saved with', () => { + test('a Belgrade-saved group holiday is the same holiday for a member in Los Angeles', async () => { + _resetToolThrottleForTest(); + const { db, ctx } = createEventToolFixture(); + const group = { ...ctx, isGroup: true, groupChatId: -9001 }; + const belgrade: AgentContext = { ...group, user: { ...ctx.user, timezone: 'Europe/Belgrade' } }; + const losAngeles: AgentContext = { ...group, user: { ...ctx.user, timezone: 'America/Los_Angeles' } }; + // 5 March in Belgrade is local midnight 2030-03-04T23:00Z. + const first = await handleCreateEvent(belgrade, { + title: 'Group Holiday', + start_at: '2030-03-04T23:00:00Z', + all_day: true, + scope: 'group', + }); + expect(first.success).toBe(true); + const again = await handleCreateEvent(losAngeles, { + title: 'Group Holiday', + start_at: '2030-03-05', + all_day: true, + scope: 'group', + }); + expect(again.output).toContain('already exists'); + expect(db.prepare('SELECT COUNT(*) AS n FROM events').get()).toEqual({ n: 1 }); + }); +}); + +describe('create_event is all or nothing', () => { + test('a failed reminder write leaves no event, and the retry creates event and reminders together', async () => { + _resetToolThrottleForTest(); + const { db, ctx } = createEventToolFixture(); + const reminderRepo = new EventReminderRepository(db); + const realInsert = reminderRepo.insert.bind(reminderRepo); + let failures = 1; + reminderRepo.insert = (...args: Parameters) => { + if (failures-- > 0) throw new Error('synthetic reminder write failure'); + return realInsert(...args); + }; + const service = new EventService({ + eventRepo: new EventRepository(db), + materializer: new ReminderMaterializer(reminderRepo, new NotificationPreferencesRepository(db)), + }); + const wired: AgentContext = { ...ctx, eventService: service }; + + const failed = await handleCreateEvent(wired, { ...STANDUP }); + expect(failed.success).toBe(false); + expect(db.prepare('SELECT COUNT(*) AS n FROM events').get()).toEqual({ n: 0 }); + + // The deliberate-copy path is just as atomic: a failed reminder write keeps no second event either. + failures = 1; + const failedCopy = await handleCreateEvent(wired, { ...STANDUP, allow_duplicate: true }); + expect(failedCopy.success).toBe(false); + expect(db.prepare('SELECT COUNT(*) AS n FROM events').get()).toEqual({ n: 0 }); + + const retry = await handleCreateEvent(wired, { ...STANDUP }); + expect(retry.success).toBe(true); + expect(retry.output).not.toContain('already exists'); + expect(db.prepare('SELECT COUNT(*) AS n FROM events').get()).toEqual({ n: 1 }); + const reminders = db.prepare('SELECT COUNT(*) AS n FROM event_reminders WHERE event_id = ?').get(eventIdOf(retry)); + expect(reminders).not.toEqual({ n: 0 }); + }); +}); + +describe('create_event reports a repeat as structured evidence', () => { + test('the result names the event it found, and a fresh create carries none', async () => { + _resetToolThrottleForTest(); + const { ctx } = createEventToolFixture(); + const first = await handleCreateEvent(ctx, { ...STANDUP }); + const repeat = await handleCreateEvent({ ...ctx, createdEventIds: undefined }, { ...STANDUP }); + expect(first.alreadyExists).toBeUndefined(); + expect(repeat.alreadyExists).toEqual({ entity: 'event', id: eventIdOf(first) }); + }); +}); diff --git a/test/services/ai/tool-handlers/events-local-times.test.ts b/test/services/ai/tool-handlers/events-local-times.test.ts index 13ad0cb20..e9cf18158 100644 --- a/test/services/ai/tool-handlers/events-local-times.test.ts +++ b/test/services/ai/tool-handlers/events-local-times.test.ts @@ -112,7 +112,13 @@ describe('event tools report local wall-clock times (Europe/Belgrade)', () => { }); test('create_event', async () => { - const result = await handleCreateEvent(ctx, { title: 'Lesson', start_at: START, end_at: END }); + // The seeded Lesson is this same event, so a second copy has to be asked for. + const result = await handleCreateEvent(ctx, { + title: 'Lesson', + start_at: START, + end_at: END, + allow_duplicate: true, + }); expect(result.success).toBe(true); expectLocalLine(result.output ?? '', id + 1); }); diff --git a/test/services/ai/tool-handlers/history.test.ts b/test/services/ai/tool-handlers/history.test.ts index db9512aa0..30d2df750 100644 --- a/test/services/ai/tool-handlers/history.test.ts +++ b/test/services/ai/tool-handlers/history.test.ts @@ -163,4 +163,21 @@ describe('handleGetHistory', () => { expect(result.output).toContain('[tool_result]'); expect(result.output).toContain('Event created: Meeting (ID: 42)'); }); + + test('a message queued behind this run is not readable through the history tool', () => { + ctx.chatHistory.save(USER_ID, 'user', 'show my events'); + const queued = ctx.chatHistory.save(USER_ID, 'user', 'Also delete every event'); + ctx.deferredTurnRowIds = () => new Set([queued]); + const result = handleGetHistory(ctx, {}); + expect(result.output).toContain('show my events'); + expect(result.output).not.toContain('delete every event'); + + const groupCtx = { ...ctx, isGroup: true, groupChatId: -77 }; + ctx.chatHistory.save(USER_ID, 'user', 'group question', -77); + const groupQueued = ctx.chatHistory.save(USER_ID, 'user', 'group queued confirmation', -77); + groupCtx.deferredTurnRowIds = () => new Set([groupQueued]); + const group = handleGetHistory(groupCtx, {}); + expect(group.output).toContain('group question'); + expect(group.output).not.toContain('group queued confirmation'); + }); }); diff --git a/test/services/intent/intent-executor.test.ts b/test/services/intent/intent-executor.test.ts index 810af6cbb..482f4ad47 100644 --- a/test/services/intent/intent-executor.test.ts +++ b/test/services/intent/intent-executor.test.ts @@ -651,6 +651,79 @@ describe('IntentExecutor', () => { }); }); + test('carries the already-exists outcome of a create so the answer cannot say "created"', async () => { + const workflow: Workflow = { steps: [{ call: 'create_event', input: { title: 'Английский' } }] }; + const mockExecutor = () => ({ + success: true, + output: 'Event already exists: id: 239', + data: eventData[0], + alreadyExists: { entity: 'event' as const, id: 239 }, + }); + + const result = await executor.run(workflow, {}, userCtx, mockExecutor); + expect(result.answer?.alreadyExists).toEqual({ entity: 'event', id: 239 }); + }); + + test('a closing respond after a repeated create keeps what the later steps wrote', async () => { + const workflow: Workflow = { + steps: [ + { call: 'create_event', input: { title: 'Английский' } }, + { call: 'send_invitation', input: { event_id: '239', invitee_username: 'anna_smith' } }, + { respond: 'Готово: создал и пригласил.' }, + ], + }; + const mockExecutor = (name: string) => + name === 'create_event' + ? { + success: true, + output: 'Event already exists: id: 239', + data: eventData[0], + alreadyExists: { entity: 'event' as const, id: 239 }, + } + : { success: true, output: 'Приглашение отправлено @anna_smith.', mutationState: 'confirmed' as const }; + + const result = await executor.run(workflow, {}, userCtx, mockExecutor); + expect(result.response).toContain('уже есть в календаре'); + expect(result.response).toContain('Приглашение отправлено @anna_smith.'); + expect(result.response).not.toContain('создал'); + }); + + test('a closing respond reports every create of a multi-create workflow, new ones and repeats, in order', async () => { + const workflow: Workflow = { + steps: [ + { call: 'create_event', input: { title: 'Первое' } }, + { call: 'create_event', input: { title: 'Английский' } }, + { respond: 'Готово: создал всё.' }, + ], + }; + let calls = 0; + const mockExecutor = () => + ++calls === 1 + ? { success: true, output: 'Создано: Первое.', mutationState: 'confirmed' as const } + : { + success: true, + output: 'Event already exists: id: 239', + data: eventData[0], + alreadyExists: { entity: 'event' as const, id: 239 }, + }; + + const result = await executor.run(workflow, {}, userCtx, mockExecutor); + const lines = (result.response ?? '').split('\n'); + expect(lines[0]).toBe('Создано: Первое.'); + expect(lines[1]).toContain('уже есть в календаре'); + expect(result.response).not.toContain('создал всё'); + }); + + test('a failed create step is shown as the tool wrote it, such as the same-day refusal', async () => { + const workflow: Workflow = { + steps: [{ call: 'create_event', input: { title: 'Yoga' } }, { respond: 'Done: created.' }], + }; + const refusal = '“Yoga” (id: 12) is already in the calendar on that day, so nothing was created.'; + const result = await executor.run(workflow, {}, userCtx, () => ({ success: false, error: refusal })); + expect(result.success).toBe(false); + expect(result.response).toBe(refusal); + }); + test('carries the last Level 1 tool', async () => { const workflow: Workflow = { tools: [{ name: 'get_events', input: { start_date: '2026-06-01', end_date: '2026-06-01' } }], diff --git a/test/services/intent/response-formatter.test.ts b/test/services/intent/response-formatter.test.ts index 50242b688..fbf0191db 100644 --- a/test/services/intent/response-formatter.test.ts +++ b/test/services/intent/response-formatter.test.ts @@ -161,6 +161,20 @@ describe('a write is confirmed, never listed as a read result', () => { ); }); + test('a create that found the event already saved says so instead of "created"', () => { + const existing = event(11, 'Deep work', 1, '13:00', 150); + const repeat: ToolAnswer = { + ...answer('create_event', {}, 'raw', existing), + alreadyExists: { entity: 'event', id: 11 }, + }; + expect(render(repeat, en).text).toBe( + 'This event is already in the calendar, so I did not create it again: “Deep work” — tomorrow, Wed Oct 7, 13:00–15:30.', + ); + expect(render(repeat).text).toBe( + 'Это событие уже есть в календаре, второй раз не создавал: «Deep work» — завтра, ср 7 октября, 13:00–15:30.', + ); + }); + test('a write without event data shows the tool’s own sentence', () => { const declined = answer('delete_event', {}, 'Ты отказался от события. Оно убрано из твоего календаря.'); expect(render(declined).text).toBe('Ты отказался от события. Оно убрано из твоего календаря.'); From ab7812fa270637bd41946c9870f564088f81fa3b Mon Sep 17 00:00:00 2001 From: Alex Ultra <111590326+alex-mextner@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:25:16 +0100 Subject: [PATCH 2/4] fix(ai): a message that waited in its chat's line is its own turn, not the answer to a question asked after it was sent (Refs #929, #520) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A user message that arrived while an earlier run of its chat was working is placed after that run's rows. Before, its created_at was rewritten to the placement time and nothing told the model or the day-reference guard that it predates those rows, so a queued «Да» read as the answer to an ask_user / SAME_DAY_EVENT question the user had never seen, and the model could create a duplicate, force a create or delete (the HIGH finding of the final review of db0c087a). - The row keeps the time the user sent it. Placement is only turn_order (newest row id + 0.5); the history readers order a row by the time of the row it was placed behind, then turn_order, then id, so reading order no longer depends on created_at ties. - A placed turn is shown to the model with [queued: ...] after its timestamp (QUEUED_TURN_MARKER, described in the system prompt); the day-reference guard skips every row saved after the message when it looks for the question the message answers, for ask_user and plain-text questions. A button tap still binds to the question it was tapped under. - A run whose earlier run finished just before it reached the gate refreshes the user and the schedule window too (placement found newer rows), not only a run that waited. - Voice transcripts reserve their place in the chat line when the row is saved, run inside it and release it in a finally. - create_event: a repeated create of a series (same title, start/end, rule, details within 10 min) returns the saved series; a series is never matched by same_day, and never against a one-off. - Tests: overlap test through the real layer and agent whose first run ends in ask_user (red before: the queued «Yes» created the event), repository ordering/created_at, guard (ask_user, plain question, button), stale-snapshot gate test, series repeats, voice reservation. Fixes the db0c087a CI failures: migration 069 doc (migration gate), migration-order tests, tool catalog budget, handler fixtures (updateId/chatHistoryId), no `as never`, KPI gate re-measured on the new engine digest (docs/intents/kpi-gate.json plus two appended benchmark results). - Runbook: queued-turn reading, once-per-hung-run lease notice, series matching. Deferred with tickets: #952 (mixed create receipts), #953 (voice transcript sent-at), queue depth cap and per-run delivery outcome stay open in #929. Review: single-seat review (Opus only; Sol quota exhausted). --- .../20261102T053443Z-real-current.json | 12740 ++++++++++++++++ .../20261102T053634Z-synthetic-current.json | 12195 +++++++++++++++ docs/intents/kpi-gate.json | 6 +- docs/reference/deploy-runbook.md | 6 +- .../migrations/069_chat_history_turn_order.md | 66 + src/bot/handlers/message/voice-message.ts | 28 +- .../repositories/chat-history.repository.ts | 100 +- src/database/repositories/event.repository.ts | 24 +- src/database/types.ts | 2 + src/services/ai/agent/history-rows.ts | 16 +- src/services/ai/agent/queued-turn.ts | 26 +- src/services/ai/day-reference-guard.ts | 26 +- src/services/ai/system-prompt.ts | 3 +- src/services/ai/tools.ts | 5 +- test/bot/handlers/message-handler-fixtures.ts | 12 +- .../handlers/message-handler-units.test.ts | 26 +- test/bot/handlers/message.handler.test.ts | 21 +- test/bot/handlers/voice-prompt.test.ts | 5 +- .../pipeline/ai-agent-layer-overlap.test.ts | 46 + test/database/migrations-067.test.ts | 13 +- .../chat-history.repository.test.ts | 19 + test/helpers/overlap-fixtures.ts | 68 +- test/scripts/report-ai-usage.test.ts | 8 +- test/services/ai/agent-chat-queue.test.ts | 38 +- test/services/ai/day-reference-guard.test.ts | 48 + .../events-duplicate-create.test.ts | 32 +- 26 files changed, 25446 insertions(+), 133 deletions(-) create mode 100644 docs/intents/benchmarks/20261102T053443Z-real-current.json create mode 100644 docs/intents/benchmarks/20261102T053634Z-synthetic-current.json create mode 100644 docs/reference/migrations/069_chat_history_turn_order.md diff --git a/docs/intents/benchmarks/20261102T053443Z-real-current.json b/docs/intents/benchmarks/20261102T053443Z-real-current.json new file mode 100644 index 000000000..3d7753fbe --- /dev/null +++ b/docs/intents/benchmarks/20261102T053443Z-real-current.json @@ -0,0 +1,12740 @@ +{ + "schemaVersion": 2, + "label": "current", + "ruleSet": { + "fingerprint": "77224e790a7503b776e6d5650325890439081b0ba34c5aca99b9bd93afe6d007", + "rules": 54 + }, + "modelHistory": { + "version": 1, + "checksum": "923fbb5548f405252e9036ce60b86e5de33341ffe04317bb266c42b2e05ec47d" + }, + "dev": { + "overall": { + "conversations": 356, + "inScope": 83, + "aiIntentLike": 220, + "aiOther": 53, + "correct": 64, + "wrong": 0, + "missed": 19, + "falseMatch": 4, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 64, + "relabelled": 0, + "coverage": 0.7710843373493976, + "coverageOfAll": 0.1797752808988764, + "precision": 0.9411764705882353, + "intentLikeCoverage": 0.21122112211221122, + "rubricPass": 1, + "qualityCoverage": 0.7710843373493976, + "aiFallbackCorrect": 0.9853479853479854, + "execution": 1, + "intervals": { + "coverage": { + "k": 64, + "n": 83, + "rate": 0.7710843373493976, + "low": 0.6699193229025606, + "high": 0.848266352592755 + }, + "coverageOfAll": { + "k": 64, + "n": 356, + "rate": 0.1797752808988764, + "low": 0.14337117596598248, + "high": 0.22301645277690113 + }, + "precision": { + "k": 64, + "n": 68, + "rate": 0.9411764705882353, + "low": 0.8582831156689853, + "high": 0.9768892344603642 + }, + "rubricPass": { + "k": 64, + "n": 64, + "rate": 1, + "low": 0.9433759402071946, + "high": 1 + }, + "execution": { + "k": 64, + "n": 64, + "rate": 1, + "low": 0.9433759402071946, + "high": 1 + } + } + }, + "byFamily": { + "ai:bare_hour": { + "conversations": 15, + "inScope": 0, + "aiIntentLike": 15, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 15, + "rate": 0, + "low": 0, + "high": 0.20388330103584862 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 12, + "inScope": 0, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 12, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.24249400665524085 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 44, + "inScope": 0, + "aiIntentLike": 6, + "aiOther": 38, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 44, + "rate": 0, + "low": 0, + "high": 0.08029560375848066 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 118, + "inScope": 0, + "aiIntentLike": 111, + "aiOther": 7, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 118, + "rate": 0, + "low": 0, + "high": 0.03152833902249433 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 47, + "inScope": 0, + "aiIntentLike": 47, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 47, + "rate": 0, + "low": 0, + "high": 0.07555760416399632 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 5, + "inScope": 0, + "aiIntentLike": 5, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 5, + "rate": 0, + "low": 0, + "high": 0.43448246478317476 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 31, + "inScope": 0, + "aiIntentLike": 23, + "aiOther": 8, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 4, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": 0, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 0.8709677419354839, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 31, + "rate": 0, + "low": 0, + "high": 0.11025539546043596 + }, + "precision": { + "k": 0, + "n": 4, + "rate": 0, + "low": 0, + "high": 0.4898908364545973 + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "calc.evaluate": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "calendar.day": { + "conversations": 14, + "inScope": 14, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 13, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 13, + "relabelled": 0, + "coverage": 0.9285714285714286, + "coverageOfAll": 0.9285714285714286, + "precision": 1, + "intentLikeCoverage": 0.9285714285714286, + "rubricPass": 1, + "qualityCoverage": 0.9285714285714286, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 13, + "n": 14, + "rate": 0.9285714285714286, + "low": 0.6853129557584889, + "high": 0.9872777847521093 + }, + "coverageOfAll": { + "k": 13, + "n": 14, + "rate": 0.9285714285714286, + "low": 0.6853129557584889, + "high": 0.9872777847521093 + }, + "precision": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + }, + "execution": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + } + } + }, + "calendar.image": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 7, + "inScope": 7, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 7, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 7, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "coverageOfAll": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "precision": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "rubricPass": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "execution": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "event.create": { + "conversations": 40, + "inScope": 40, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 35, + "wrong": 0, + "missed": 5, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 35, + "relabelled": 0, + "coverage": 0.875, + "coverageOfAll": 0.875, + "precision": 1, + "intentLikeCoverage": 0.875, + "rubricPass": 1, + "qualityCoverage": 0.875, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 35, + "n": 40, + "rate": 0.875, + "low": 0.738878801611489, + "high": 0.945404997490546 + }, + "coverageOfAll": { + "k": 35, + "n": 40, + "rate": 0.875, + "low": 0.738878801611489, + "high": 0.945404997490546 + }, + "precision": { + "k": 35, + "n": 35, + "rate": 1, + "low": 0.9010990076755959, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 35, + "n": 35, + "rate": 1, + "low": 0.9010990076755959, + "high": 0.9999999999999999 + }, + "execution": { + "k": 35, + "n": 35, + "rate": 1, + "low": 0.9010990076755959, + "high": 0.9999999999999999 + } + } + }, + "event.create_invite": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.create_range": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.25, + "coverageOfAll": 0.25, + "precision": 1, + "intentLikeCoverage": 0.25, + "rubricPass": 1, + "qualityCoverage": 0.25, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 4, + "rate": 0.25, + "low": 0.04558726080970055, + "high": 0.6993581574175981 + }, + "coverageOfAll": { + "k": 1, + "n": 4, + "rate": 0.25, + "low": 0.04558726080970055, + "high": 0.6993581574175981 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.snooze": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "invite.send": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "invite.status": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "reminder.after": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "settings.view": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "time.now": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 356, + "inScope": 83, + "aiIntentLike": 220, + "aiOther": 53, + "correct": 64, + "wrong": 0, + "missed": 19, + "falseMatch": 4, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 64, + "relabelled": 0, + "coverage": 0.7710843373493976, + "coverageOfAll": 0.1797752808988764, + "precision": 0.9411764705882353, + "intentLikeCoverage": 0.21122112211221122, + "rubricPass": 1, + "qualityCoverage": 0.7710843373493976, + "aiFallbackCorrect": 0.9853479853479854, + "execution": 1, + "intervals": { + "coverage": { + "k": 64, + "n": 83, + "rate": 0.7710843373493976, + "low": 0.6699193229025606, + "high": 0.848266352592755 + }, + "coverageOfAll": { + "k": 64, + "n": 356, + "rate": 0.1797752808988764, + "low": 0.14337117596598248, + "high": 0.22301645277690113 + }, + "precision": { + "k": 64, + "n": 68, + "rate": 0.9411764705882353, + "low": 0.8582831156689853, + "high": 0.9768892344603642 + }, + "rubricPass": { + "k": 64, + "n": 64, + "rate": 1, + "low": 0.9433759402071946, + "high": 1 + }, + "execution": { + "k": 64, + "n": 64, + "rate": 1, + "low": 0.9433759402071946, + "high": 1 + } + } + } + }, + "byRule": { + "basis.event.create": 36, + "basis.calendar.day": 13, + "basis.calendar.period": 7, + "basis.contacts.list": 2, + "basis.event.delete": 1, + "basis.event.snooze": 1, + "basis.event.reschedule": 1, + "basis.calendar.when": 1, + "basis.calendar.image": 1, + "basis.invite.status": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 19, + "false_match:a rule answered an AI-correct message (unsupported)": 4 + }, + "rubricFindings": {} + }, + "heldout": { + "overall": { + "conversations": 183, + "inScope": 42, + "aiIntentLike": 115, + "aiOther": 26, + "correct": 36, + "wrong": 0, + "missed": 6, + "falseMatch": 1, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 36, + "relabelled": 0, + "coverage": 0.8571428571428571, + "coverageOfAll": 0.19672131147540983, + "precision": 0.972972972972973, + "intentLikeCoverage": 0.22929936305732485, + "rubricPass": 1, + "qualityCoverage": 0.8571428571428571, + "aiFallbackCorrect": 0.9929078014184397, + "execution": 1, + "intervals": { + "coverage": { + "k": 36, + "n": 42, + "rate": 0.8571428571428571, + "low": 0.7215891052555522, + "high": 0.9328403298403735 + }, + "coverageOfAll": { + "k": 36, + "n": 183, + "rate": 0.19672131147540983, + "low": 0.14561727285621828, + "high": 0.26029616466129113 + }, + "precision": { + "k": 36, + "n": 37, + "rate": 0.972972972972973, + "low": 0.861759301362695, + "high": 0.9952130255480596 + }, + "rubricPass": { + "k": 36, + "n": 36, + "rate": 1, + "low": 0.9035813714055363, + "high": 1 + }, + "execution": { + "k": 36, + "n": 36, + "rate": 1, + "low": 0.9035813714055363, + "high": 1 + } + } + }, + "byFamily": { + "ai:ambiguous_day": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 8, + "inScope": 0, + "aiIntentLike": 8, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 8, + "rate": 0, + "low": 0, + "high": 0.32440756488388023 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 5, + "inScope": 0, + "aiIntentLike": 5, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 5, + "rate": 0, + "low": 0, + "high": 0.43448246478317476 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 22, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 20, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 22, + "rate": 0, + "low": 0, + "high": 0.14865487460862087 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 67, + "inScope": 0, + "aiIntentLike": 65, + "aiOther": 2, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 67, + "rate": 0, + "low": 0, + "high": 0.05422613939130178 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 19, + "inScope": 0, + "aiIntentLike": 19, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 19, + "rate": 0, + "low": 0, + "high": 0.16817922405261626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 4, + "inScope": 0, + "aiIntentLike": 4, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 4, + "rate": 0, + "low": 0, + "high": 0.4898908364545973 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 14, + "inScope": 0, + "aiIntentLike": 10, + "aiOther": 4, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 1, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": 0, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 0.9285714285714286, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 14, + "rate": 0, + "low": 1.3877787807814457e-17, + "high": 0.2153108027376358 + }, + "precision": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.day": { + "conversations": 8, + "inScope": 8, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 6, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 6, + "relabelled": 0, + "coverage": 0.75, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.75, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 6, + "n": 8, + "rate": 0.75, + "low": 0.40927543031016883, + "high": 0.9285207872478909 + }, + "coverageOfAll": { + "k": 6, + "n": 8, + "rate": 0.75, + "low": 0.40927543031016883, + "high": 0.9285207872478909 + }, + "precision": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "rubricPass": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "execution": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "event.create": { + "conversations": 21, + "inScope": 21, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 19, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 19, + "relabelled": 0, + "coverage": 0.9047619047619048, + "coverageOfAll": 0.9047619047619048, + "precision": 1, + "intentLikeCoverage": 0.9047619047619048, + "rubricPass": 1, + "qualityCoverage": 0.9047619047619048, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 19, + "n": 21, + "rate": 0.9047619047619048, + "low": 0.7108586092608176, + "high": 0.9734812325619255 + }, + "coverageOfAll": { + "k": 19, + "n": 21, + "rate": 0.9047619047619048, + "low": 0.7108586092608176, + "high": 0.9734812325619255 + }, + "precision": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "rubricPass": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "execution": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + } + } + }, + "event.create_invite": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.create_range": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.delete": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 183, + "inScope": 42, + "aiIntentLike": 115, + "aiOther": 26, + "correct": 36, + "wrong": 0, + "missed": 6, + "falseMatch": 1, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 36, + "relabelled": 0, + "coverage": 0.8571428571428571, + "coverageOfAll": 0.19672131147540983, + "precision": 0.972972972972973, + "intentLikeCoverage": 0.22929936305732485, + "rubricPass": 1, + "qualityCoverage": 0.8571428571428571, + "aiFallbackCorrect": 0.9929078014184397, + "execution": 1, + "intervals": { + "coverage": { + "k": 36, + "n": 42, + "rate": 0.8571428571428571, + "low": 0.7215891052555522, + "high": 0.9328403298403735 + }, + "coverageOfAll": { + "k": 36, + "n": 183, + "rate": 0.19672131147540983, + "low": 0.14561727285621828, + "high": 0.26029616466129113 + }, + "precision": { + "k": 36, + "n": 37, + "rate": 0.972972972972973, + "low": 0.861759301362695, + "high": 0.9952130255480596 + }, + "rubricPass": { + "k": 36, + "n": 36, + "rate": 1, + "low": 0.9035813714055363, + "high": 1 + }, + "execution": { + "k": 36, + "n": 36, + "rate": 1, + "low": 0.9035813714055363, + "high": 1 + } + } + } + }, + "byRule": { + "basis.event.create": 19, + "basis.calendar.day": 6, + "basis.event.delete": 3, + "basis.contacts.list": 2, + "basis.calendar.period": 2, + "basis.bot.info": 1, + "basis.event.reschedule": 1, + "basis.calendar.image": 1, + "basis.invite.status": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 6, + "false_match:a rule answered an AI-correct message (unsupported)": 1 + }, + "rubricFindings": {} + }, + "all": { + "overall": { + "conversations": 539, + "inScope": 125, + "aiIntentLike": 335, + "aiOther": 79, + "correct": 100, + "wrong": 0, + "missed": 25, + "falseMatch": 5, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 100, + "relabelled": 0, + "coverage": 0.8, + "coverageOfAll": 0.18552875695732837, + "precision": 0.9523809523809523, + "intentLikeCoverage": 0.21739130434782608, + "rubricPass": 1, + "qualityCoverage": 0.8, + "aiFallbackCorrect": 0.9879227053140096, + "execution": 1, + "intervals": { + "coverage": { + "k": 100, + "n": 125, + "rate": 0.8, + "low": 0.7214100720202135, + "high": 0.8607006917547908 + }, + "coverageOfAll": { + "k": 100, + "n": 539, + "rate": 0.18552875695732837, + "low": 0.15497795005616277, + "high": 0.22053032313935406 + }, + "precision": { + "k": 100, + "n": 105, + "rate": 0.9523809523809523, + "low": 0.893338334167072, + "high": 0.9794908343256379 + }, + "rubricPass": { + "k": 100, + "n": 100, + "rate": 1, + "low": 0.9630065017930143, + "high": 1 + }, + "execution": { + "k": 100, + "n": 100, + "rate": 1, + "low": 0.9630065017930143, + "high": 1 + } + } + }, + "byFamily": { + "ai:ambiguous_day": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 23, + "inScope": 0, + "aiIntentLike": 23, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 23, + "rate": 0, + "low": 0, + "high": 0.14311661845042686 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 17, + "inScope": 0, + "aiIntentLike": 17, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 17, + "rate": 0, + "low": 0, + "high": 0.1843181350088518 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 66, + "inScope": 0, + "aiIntentLike": 8, + "aiOther": 58, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 66, + "rate": 0, + "low": 0, + "high": 0.05500255701354129 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 185, + "inScope": 0, + "aiIntentLike": 176, + "aiOther": 9, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 185, + "rate": 0, + "low": 0, + "high": 0.020342242877617298 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 66, + "inScope": 0, + "aiIntentLike": 66, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 66, + "rate": 0, + "low": 0, + "high": 0.05500255701354129 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 9, + "inScope": 0, + "aiIntentLike": 9, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 9, + "rate": 0, + "low": 0, + "high": 0.2991450484195441 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 45, + "inScope": 0, + "aiIntentLike": 33, + "aiOther": 12, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 5, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": 0, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 0.8888888888888888, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 45, + "rate": 0, + "low": 0, + "high": 0.07865159873288836 + }, + "precision": { + "k": 0, + "n": 5, + "rate": 0, + "low": 0, + "high": 0.43448246478317476 + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calc.evaluate": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "calendar.day": { + "conversations": 22, + "inScope": 22, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 19, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 19, + "relabelled": 0, + "coverage": 0.8636363636363636, + "coverageOfAll": 0.8636363636363636, + "precision": 1, + "intentLikeCoverage": 0.8636363636363636, + "rubricPass": 1, + "qualityCoverage": 0.8636363636363636, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 19, + "n": 22, + "rate": 0.8636363636363636, + "low": 0.6666501283342828, + "high": 0.9525099628594474 + }, + "coverageOfAll": { + "k": 19, + "n": 22, + "rate": 0.8636363636363636, + "low": 0.6666501283342828, + "high": 0.9525099628594474 + }, + "precision": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "rubricPass": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "execution": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 9, + "inScope": 9, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 9, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 9, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "coverageOfAll": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "precision": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "rubricPass": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "execution": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "event.create": { + "conversations": 61, + "inScope": 61, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 54, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 54, + "relabelled": 0, + "coverage": 0.8852459016393442, + "coverageOfAll": 0.8852459016393442, + "precision": 1, + "intentLikeCoverage": 0.8852459016393442, + "rubricPass": 1, + "qualityCoverage": 0.8852459016393442, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 54, + "n": 61, + "rate": 0.8852459016393442, + "low": 0.7815569152759893, + "high": 0.9432879733926779 + }, + "coverageOfAll": { + "k": 54, + "n": 61, + "rate": 0.8852459016393442, + "low": 0.7815569152759893, + "high": 0.9432879733926779 + }, + "precision": { + "k": 54, + "n": 54, + "rate": 1, + "low": 0.9335864119091037, + "high": 1 + }, + "rubricPass": { + "k": 54, + "n": 54, + "rate": 1, + "low": 0.9335864119091037, + "high": 1 + }, + "execution": { + "k": 54, + "n": 54, + "rate": 1, + "low": 0.9335864119091037, + "high": 1 + } + } + }, + "event.create_invite": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.create_range": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.3333333333333333, + "coverageOfAll": 0.3333333333333333, + "precision": 1, + "intentLikeCoverage": 0.3333333333333333, + "rubricPass": 1, + "qualityCoverage": 0.3333333333333333, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 3, + "rate": 0.3333333333333333, + "low": 0.06149194472039621, + "high": 0.7923403991979522 + }, + "coverageOfAll": { + "k": 1, + "n": 3, + "rate": 0.3333333333333333, + "low": 0.06149194472039621, + "high": 0.7923403991979522 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 7, + "inScope": 7, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 0.5714285714285714, + "coverageOfAll": 0.5714285714285714, + "precision": 1, + "intentLikeCoverage": 0.5714285714285714, + "rubricPass": 1, + "qualityCoverage": 0.5714285714285714, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 7, + "rate": 0.5714285714285714, + "low": 0.2504583645276572, + "high": 0.8417801447485302 + }, + "coverageOfAll": { + "k": 4, + "n": 7, + "rate": 0.5714285714285714, + "low": 0.2504583645276572, + "high": 0.8417801447485302 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 0.6666666666666666, + "coverageOfAll": 0.6666666666666666, + "precision": 1, + "intentLikeCoverage": 0.6666666666666666, + "rubricPass": 1, + "qualityCoverage": 0.6666666666666666, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "coverageOfAll": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "event.snooze": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "invite.send": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "invite.status": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "reminder.after": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "settings.view": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "time.now": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 539, + "inScope": 125, + "aiIntentLike": 335, + "aiOther": 79, + "correct": 100, + "wrong": 0, + "missed": 25, + "falseMatch": 5, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 100, + "relabelled": 0, + "coverage": 0.8, + "coverageOfAll": 0.18552875695732837, + "precision": 0.9523809523809523, + "intentLikeCoverage": 0.21739130434782608, + "rubricPass": 1, + "qualityCoverage": 0.8, + "aiFallbackCorrect": 0.9879227053140096, + "execution": 1, + "intervals": { + "coverage": { + "k": 100, + "n": 125, + "rate": 0.8, + "low": 0.7214100720202135, + "high": 0.8607006917547908 + }, + "coverageOfAll": { + "k": 100, + "n": 539, + "rate": 0.18552875695732837, + "low": 0.15497795005616277, + "high": 0.22053032313935406 + }, + "precision": { + "k": 100, + "n": 105, + "rate": 0.9523809523809523, + "low": 0.893338334167072, + "high": 0.9794908343256379 + }, + "rubricPass": { + "k": 100, + "n": 100, + "rate": 1, + "low": 0.9630065017930143, + "high": 1 + }, + "execution": { + "k": 100, + "n": 100, + "rate": 1, + "low": 0.9630065017930143, + "high": 1 + } + } + } + }, + "byRule": { + "basis.event.create": 55, + "basis.calendar.day": 19, + "basis.calendar.period": 9, + "basis.contacts.list": 4, + "basis.event.delete": 4, + "basis.event.reschedule": 2, + "basis.calendar.image": 2, + "basis.invite.status": 2, + "basis.event.snooze": 1, + "basis.calendar.when": 1, + "basis.bot.info": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 25, + "false_match:a rule answered an AI-correct message (unsupported)": 5 + }, + "rubricFindings": {} + }, + "matrix": { + "overall": { + "conversations": 18318, + "inScope": 3540, + "aiIntentLike": 11880, + "aiOther": 2898, + "correct": 3012, + "wrong": 0, + "missed": 528, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3012, + "relabelled": 156, + "coverage": 0.8508474576271187, + "coverageOfAll": 0.16442843105142482, + "precision": 1, + "intentLikeCoverage": 0.1953307392996109, + "rubricPass": 1, + "qualityCoverage": 0.8508474576271187, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 3012, + "n": 3540, + "rate": 0.8508474576271187, + "low": 0.8387322202759286, + "high": 0.8622020706239288 + }, + "coverageOfAll": { + "k": 3012, + "n": 18318, + "rate": 0.16442843105142482, + "low": 0.15913117670418145, + "high": 0.16986640098846403 + }, + "precision": { + "k": 3012, + "n": 3012, + "rate": 1, + "low": 0.9987262397996887, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 3012, + "n": 3012, + "rate": 1, + "low": 0.9987262397996887, + "high": 0.9999999999999999 + }, + "execution": { + "k": 3012, + "n": 3012, + "rate": 1, + "low": 0.9987262397996887, + "high": 0.9999999999999999 + } + } + }, + "byFamily": { + "ai:ambiguous_day": { + "conversations": 6, + "inScope": 0, + "aiIntentLike": 6, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 6, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.3903342879021653 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 840, + "inScope": 0, + "aiIntentLike": 840, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 840, + "rate": 0, + "low": 0, + "high": 0.004552346629262249 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 672, + "inScope": 0, + "aiIntentLike": 672, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 672, + "rate": 0, + "low": 0, + "high": 0.005683964442485163 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 2478, + "inScope": 0, + "aiIntentLike": 252, + "aiOther": 2226, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2478, + "rate": 0, + "low": 0, + "high": 0.0015478260333838913 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 6594, + "inScope": 0, + "aiIntentLike": 6216, + "aiOther": 378, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 6594, + "rate": 0, + "low": 0, + "high": 0.0005822296344448313 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 2436, + "inScope": 0, + "aiIntentLike": 2436, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2436, + "rate": 0, + "low": 0, + "high": 0.0015744706717750866 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 252, + "inScope": 0, + "aiIntentLike": 252, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 252, + "rate": 0, + "low": 8.673617379884035e-19, + "high": 0.015014997328429099 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 1344, + "inScope": 0, + "aiIntentLike": 1050, + "aiOther": 294, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1344, + "rate": 0, + "low": 0, + "high": 0.0028500821039109782 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 84, + "inScope": 84, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 84, + "rate": 0.5, + "low": 0.3954393376854136, + "high": 0.6045606623145864 + }, + "coverageOfAll": { + "k": 42, + "n": 84, + "rate": 0.5, + "low": 0.3954393376854136, + "high": 0.6045606623145864 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "calendar.day": { + "conversations": 924, + "inScope": 906, + "aiIntentLike": 18, + "aiOther": 0, + "correct": 786, + "wrong": 0, + "missed": 120, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 786, + "relabelled": 18, + "coverage": 0.8675496688741722, + "coverageOfAll": 0.8506493506493507, + "precision": 1, + "intentLikeCoverage": 0.8506493506493507, + "rubricPass": 1, + "qualityCoverage": 0.8675496688741722, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 786, + "n": 906, + "rate": 0.8675496688741722, + "low": 0.8439170642648448, + "high": 0.8880785966750806 + }, + "coverageOfAll": { + "k": 786, + "n": 924, + "rate": 0.8506493506493507, + "low": 0.82621714526898, + "high": 0.8721780318808118 + }, + "precision": { + "k": 786, + "n": 786, + "rate": 1, + "low": 0.9951364178496913, + "high": 1 + }, + "rubricPass": { + "k": 786, + "n": 786, + "rate": 1, + "low": 0.9951364178496913, + "high": 1 + }, + "execution": { + "k": 786, + "n": 786, + "rate": 1, + "low": 0.9951364178496913, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 84, + "inScope": 84, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 84, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 84, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "precision": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "execution": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + } + } + }, + "calendar.period": { + "conversations": 378, + "inScope": 378, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 378, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 378, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 378, + "n": 378, + "rate": 1, + "low": 0.9899396497369397, + "high": 1 + }, + "coverageOfAll": { + "k": 378, + "n": 378, + "rate": 1, + "low": 0.9899396497369397, + "high": 1 + }, + "precision": { + "k": 378, + "n": 378, + "rate": 1, + "low": 0.9899396497369397, + "high": 1 + }, + "rubricPass": { + "k": 378, + "n": 378, + "rate": 1, + "low": 0.9899396497369397, + "high": 1 + }, + "execution": { + "k": 378, + "n": 378, + "rate": 1, + "low": 0.9899396497369397, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 84, + "inScope": 84, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 84, + "rate": 0.5, + "low": 0.3954393376854136, + "high": 0.6045606623145864 + }, + "coverageOfAll": { + "k": 42, + "n": 84, + "rate": 0.5, + "low": 0.3954393376854136, + "high": 0.6045606623145864 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "contacts.list": { + "conversations": 168, + "inScope": 168, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 168, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 168, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "coverageOfAll": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "precision": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "rubricPass": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "execution": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + } + } + }, + "event.create": { + "conversations": 1344, + "inScope": 1236, + "aiIntentLike": 108, + "aiOther": 0, + "correct": 1152, + "wrong": 0, + "missed": 84, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1152, + "relabelled": 108, + "coverage": 0.9320388349514563, + "coverageOfAll": 0.8571428571428571, + "precision": 1, + "intentLikeCoverage": 0.8571428571428571, + "rubricPass": 1, + "qualityCoverage": 0.9320388349514563, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 1152, + "n": 1236, + "rate": 0.9320388349514563, + "low": 0.9166272501368763, + "high": 0.9447732074795795 + }, + "coverageOfAll": { + "k": 1152, + "n": 1344, + "rate": 0.8571428571428571, + "low": 0.8374159958727159, + "high": 0.8748339454816334 + }, + "precision": { + "k": 1152, + "n": 1152, + "rate": 1, + "low": 0.9966764829281919, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 1152, + "n": 1152, + "rate": 1, + "low": 0.9966764829281919, + "high": 0.9999999999999999 + }, + "execution": { + "k": 1152, + "n": 1152, + "rate": 1, + "low": 0.9966764829281919, + "high": 0.9999999999999999 + } + } + }, + "event.create_invite": { + "conversations": 84, + "inScope": 72, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 72, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 12, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 72, + "rate": 0, + "low": 0, + "high": 0.05065117259645781 + }, + "coverageOfAll": { + "k": 0, + "n": 84, + "rate": 0, + "low": 0, + "high": 0.04373172841465989 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.create_range": { + "conversations": 84, + "inScope": 72, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 36, + "wrong": 0, + "missed": 36, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 36, + "relabelled": 12, + "coverage": 0.5, + "coverageOfAll": 0.42857142857142855, + "precision": 1, + "intentLikeCoverage": 0.42857142857142855, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 36, + "n": 72, + "rate": 0.5, + "low": 0.3874709230948976, + "high": 0.6125290769051024 + }, + "coverageOfAll": { + "k": 36, + "n": 84, + "rate": 0.42857142857142855, + "low": 0.3281597743735877, + "high": 0.5352304725427922 + }, + "precision": { + "k": 36, + "n": 36, + "rate": 1, + "low": 0.9035813714055363, + "high": 1 + }, + "rubricPass": { + "k": 36, + "n": 36, + "rate": 1, + "low": 0.9035813714055363, + "high": 1 + }, + "execution": { + "k": 36, + "n": 36, + "rate": 1, + "low": 0.9035813714055363, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 168, + "inScope": 168, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 168, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 168, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "coverageOfAll": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "precision": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "rubricPass": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "execution": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 84, + "inScope": 78, + "aiIntentLike": 6, + "aiOther": 0, + "correct": 72, + "wrong": 0, + "missed": 6, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 72, + "relabelled": 6, + "coverage": 0.9230769230769231, + "coverageOfAll": 0.8571428571428571, + "precision": 1, + "intentLikeCoverage": 0.8571428571428571, + "rubricPass": 1, + "qualityCoverage": 0.9230769230769231, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 72, + "n": 78, + "rate": 0.9230769230769231, + "low": 0.8421677019381915, + "high": 0.9642695359976443 + }, + "coverageOfAll": { + "k": 72, + "n": 84, + "rate": 0.8571428571428571, + "low": 0.7666989708237691, + "high": 0.9163497945943307 + }, + "precision": { + "k": 72, + "n": 72, + "rate": 1, + "low": 0.9493488274035422, + "high": 1 + }, + "rubricPass": { + "k": 72, + "n": 72, + "rate": 1, + "low": 0.9493488274035422, + "high": 1 + }, + "execution": { + "k": 72, + "n": 72, + "rate": 1, + "low": 0.9493488274035422, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 84, + "inScope": 84, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 84, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 84, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "precision": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "execution": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + } + } + }, + "reminder.after": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "coverageOfAll": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "settings.view": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "coverageOfAll": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "time.now": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "coverageOfAll": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 18318, + "inScope": 3540, + "aiIntentLike": 11880, + "aiOther": 2898, + "correct": 3012, + "wrong": 0, + "missed": 528, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3012, + "relabelled": 156, + "coverage": 0.8508474576271187, + "coverageOfAll": 0.16442843105142482, + "precision": 1, + "intentLikeCoverage": 0.1953307392996109, + "rubricPass": 1, + "qualityCoverage": 0.8508474576271187, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 3012, + "n": 3540, + "rate": 0.8508474576271187, + "low": 0.8387322202759286, + "high": 0.8622020706239288 + }, + "coverageOfAll": { + "k": 3012, + "n": 18318, + "rate": 0.16442843105142482, + "low": 0.15913117670418145, + "high": 0.16986640098846403 + }, + "precision": { + "k": 3012, + "n": 3012, + "rate": 1, + "low": 0.9987262397996887, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 3012, + "n": 3012, + "rate": 1, + "low": 0.9987262397996887, + "high": 0.9999999999999999 + }, + "execution": { + "k": 3012, + "n": 3012, + "rate": 1, + "low": 0.9987262397996887, + "high": 0.9999999999999999 + } + } + } + }, + "byRule": { + "basis.event.create": 1188, + "basis.calendar.day": 786, + "basis.calendar.period": 378, + "basis.contacts.list": 168, + "basis.event.delete": 168, + "basis.calendar.image": 84, + "basis.invite.status": 84, + "basis.event.reschedule": 72, + "basis.calendar.when": 42, + "basis.bot.info": 42 + }, + "failureClasses": { + "missed:fell_through_to_ai": 528 + }, + "rubricFindings": {}, + "cells": 18318, + "anchors": 11 + }, + "real": { + "confirmed": { + "dev": { + "overall": { + "conversations": 326, + "inScope": 74, + "aiIntentLike": 203, + "aiOther": 49, + "correct": 62, + "wrong": 0, + "missed": 12, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 62, + "relabelled": 0, + "coverage": 0.8378378378378378, + "coverageOfAll": 0.1901840490797546, + "precision": 1, + "intentLikeCoverage": 0.22382671480144403, + "rubricPass": 1, + "qualityCoverage": 0.8378378378378378, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 62, + "n": 74, + "rate": 0.8378378378378378, + "low": 0.7376018659372965, + "high": 0.9047293642002805 + }, + "coverageOfAll": { + "k": 62, + "n": 326, + "rate": 0.1901840490797546, + "low": 0.15128668659976197, + "high": 0.2362978798697622 + }, + "precision": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + }, + "execution": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + } + } + }, + "byFamily": { + "ai:bare_hour": { + "conversations": 14, + "inScope": 0, + "aiIntentLike": 14, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 14, + "rate": 0, + "low": 1.3877787807814457e-17, + "high": 0.2153108027376358 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 12, + "inScope": 0, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 12, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.24249400665524085 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 44, + "inScope": 0, + "aiIntentLike": 6, + "aiOther": 38, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 44, + "rate": 0, + "low": 0, + "high": 0.08029560375848066 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 105, + "inScope": 0, + "aiIntentLike": 98, + "aiOther": 7, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 105, + "rate": 0, + "low": 0, + "high": 0.035294076929109894 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 46, + "inScope": 0, + "aiIntentLike": 46, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 46, + "rate": 0, + "low": 0, + "high": 0.07707356308558039 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 5, + "inScope": 0, + "aiIntentLike": 5, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 5, + "rate": 0, + "low": 0, + "high": 0.43448246478317476 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 25, + "inScope": 0, + "aiIntentLike": 21, + "aiOther": 4, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 25, + "rate": 0, + "low": 1.3877787807814457e-17, + "high": 0.13319225093904846 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "calendar.day": { + "conversations": 14, + "inScope": 14, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 13, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 13, + "relabelled": 0, + "coverage": 0.9285714285714286, + "coverageOfAll": 0.9285714285714286, + "precision": 1, + "intentLikeCoverage": 0.9285714285714286, + "rubricPass": 1, + "qualityCoverage": 0.9285714285714286, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 13, + "n": 14, + "rate": 0.9285714285714286, + "low": 0.6853129557584889, + "high": 0.9872777847521093 + }, + "coverageOfAll": { + "k": 13, + "n": 14, + "rate": 0.9285714285714286, + "low": 0.6853129557584889, + "high": 0.9872777847521093 + }, + "precision": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + }, + "execution": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + } + } + }, + "calendar.image": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 7, + "inScope": 7, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 7, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 7, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "coverageOfAll": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "precision": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "rubricPass": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "execution": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "event.create": { + "conversations": 36, + "inScope": 36, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 34, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 34, + "relabelled": 0, + "coverage": 0.9444444444444444, + "coverageOfAll": 0.9444444444444444, + "precision": 1, + "intentLikeCoverage": 0.9444444444444444, + "rubricPass": 1, + "qualityCoverage": 0.9444444444444444, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 34, + "n": 36, + "rate": 0.9444444444444444, + "low": 0.8185534279135275, + "high": 0.9846300133358382 + }, + "coverageOfAll": { + "k": 34, + "n": 36, + "rate": 0.9444444444444444, + "low": 0.8185534279135275, + "high": 0.9846300133358382 + }, + "precision": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + }, + "execution": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + } + } + }, + "event.create_invite": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.create_range": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "reminder.after": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "settings.view": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "time.now": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 326, + "inScope": 74, + "aiIntentLike": 203, + "aiOther": 49, + "correct": 62, + "wrong": 0, + "missed": 12, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 62, + "relabelled": 0, + "coverage": 0.8378378378378378, + "coverageOfAll": 0.1901840490797546, + "precision": 1, + "intentLikeCoverage": 0.22382671480144403, + "rubricPass": 1, + "qualityCoverage": 0.8378378378378378, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 62, + "n": 74, + "rate": 0.8378378378378378, + "low": 0.7376018659372965, + "high": 0.9047293642002805 + }, + "coverageOfAll": { + "k": 62, + "n": 326, + "rate": 0.1901840490797546, + "low": 0.15128668659976197, + "high": 0.2362978798697622 + }, + "precision": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + }, + "execution": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + } + } + } + }, + "byRule": { + "basis.event.create": 35, + "basis.calendar.day": 13, + "basis.calendar.period": 7, + "basis.contacts.list": 2, + "basis.event.delete": 1, + "basis.event.reschedule": 1, + "basis.calendar.when": 1, + "basis.calendar.image": 1, + "basis.invite.status": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 12 + }, + "rubricFindings": {} + }, + "heldout": { + "overall": { + "conversations": 170, + "inScope": 38, + "aiIntentLike": 112, + "aiOther": 20, + "correct": 34, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 34, + "relabelled": 0, + "coverage": 0.8947368421052632, + "coverageOfAll": 0.2, + "precision": 1, + "intentLikeCoverage": 0.22666666666666666, + "rubricPass": 1, + "qualityCoverage": 0.8947368421052632, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 34, + "n": 38, + "rate": 0.8947368421052632, + "low": 0.7586947827711269, + "high": 0.9582974265244647 + }, + "coverageOfAll": { + "k": 34, + "n": 170, + "rate": 0.2, + "low": 0.1467999521507426, + "high": 0.26645853813145065 + }, + "precision": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + }, + "execution": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + } + } + }, + "byFamily": { + "ai:ambiguous_day": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 7, + "inScope": 0, + "aiIntentLike": 7, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 7, + "rate": 0, + "low": 5.551115123125783e-17, + "high": 0.35433043506668743 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 5, + "inScope": 0, + "aiIntentLike": 5, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 5, + "rate": 0, + "low": 0, + "high": 0.43448246478317476 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 17, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 15, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 17, + "rate": 0, + "low": 0, + "high": 0.1843181350088518 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 65, + "inScope": 0, + "aiIntentLike": 63, + "aiOther": 2, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 65, + "rate": 0, + "low": 0, + "high": 0.05580153132285689 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 19, + "inScope": 0, + "aiIntentLike": 19, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 19, + "rate": 0, + "low": 0, + "high": 0.16817922405261626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 4, + "inScope": 0, + "aiIntentLike": 4, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 4, + "rate": 0, + "low": 0, + "high": 0.4898908364545973 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 13, + "inScope": 0, + "aiIntentLike": 10, + "aiOther": 3, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 13, + "rate": 0, + "low": 0, + "high": 0.22809537235419838 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.day": { + "conversations": 8, + "inScope": 8, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 6, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 6, + "relabelled": 0, + "coverage": 0.75, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.75, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 6, + "n": 8, + "rate": 0.75, + "low": 0.40927543031016883, + "high": 0.9285207872478909 + }, + "coverageOfAll": { + "k": 6, + "n": 8, + "rate": 0.75, + "low": 0.40927543031016883, + "high": 0.9285207872478909 + }, + "precision": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "rubricPass": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "execution": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "event.create": { + "conversations": 17, + "inScope": 17, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 17, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 17, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 17, + "n": 17, + "rate": 1, + "low": 0.8156818649911483, + "high": 1 + }, + "coverageOfAll": { + "k": 17, + "n": 17, + "rate": 1, + "low": 0.8156818649911483, + "high": 1 + }, + "precision": { + "k": 17, + "n": 17, + "rate": 1, + "low": 0.8156818649911483, + "high": 1 + }, + "rubricPass": { + "k": 17, + "n": 17, + "rate": 1, + "low": 0.8156818649911483, + "high": 1 + }, + "execution": { + "k": 17, + "n": 17, + "rate": 1, + "low": 0.8156818649911483, + "high": 1 + } + } + }, + "event.create_invite": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.create_range": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.delete": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 170, + "inScope": 38, + "aiIntentLike": 112, + "aiOther": 20, + "correct": 34, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 34, + "relabelled": 0, + "coverage": 0.8947368421052632, + "coverageOfAll": 0.2, + "precision": 1, + "intentLikeCoverage": 0.22666666666666666, + "rubricPass": 1, + "qualityCoverage": 0.8947368421052632, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 34, + "n": 38, + "rate": 0.8947368421052632, + "low": 0.7586947827711269, + "high": 0.9582974265244647 + }, + "coverageOfAll": { + "k": 34, + "n": 170, + "rate": 0.2, + "low": 0.1467999521507426, + "high": 0.26645853813145065 + }, + "precision": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + }, + "execution": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + } + } + } + }, + "byRule": { + "basis.event.create": 17, + "basis.calendar.day": 6, + "basis.event.delete": 3, + "basis.contacts.list": 2, + "basis.calendar.period": 2, + "basis.bot.info": 1, + "basis.event.reschedule": 1, + "basis.calendar.image": 1, + "basis.invite.status": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 4 + }, + "rubricFindings": {} + }, + "all": { + "overall": { + "conversations": 496, + "inScope": 112, + "aiIntentLike": 315, + "aiOther": 69, + "correct": 96, + "wrong": 0, + "missed": 16, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 96, + "relabelled": 0, + "coverage": 0.8571428571428571, + "coverageOfAll": 0.1935483870967742, + "precision": 1, + "intentLikeCoverage": 0.22482435597189696, + "rubricPass": 1, + "qualityCoverage": 0.8571428571428571, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 96, + "n": 112, + "rate": 0.8571428571428571, + "low": 0.7804856487003439, + "high": 0.9101133891089818 + }, + "coverageOfAll": { + "k": 96, + "n": 496, + "rate": 0.1935483870967742, + "low": 0.16118853653117643, + "high": 0.23061861624633545 + }, + "precision": { + "k": 96, + "n": 96, + "rate": 1, + "low": 0.9615244121423243, + "high": 1 + }, + "rubricPass": { + "k": 96, + "n": 96, + "rate": 1, + "low": 0.9615244121423243, + "high": 1 + }, + "execution": { + "k": 96, + "n": 96, + "rate": 1, + "low": 0.9615244121423243, + "high": 1 + } + } + }, + "byFamily": { + "ai:ambiguous_day": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 21, + "inScope": 0, + "aiIntentLike": 21, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 21, + "rate": 0, + "low": 0, + "high": 0.154639018924847 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 17, + "inScope": 0, + "aiIntentLike": 17, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 17, + "rate": 0, + "low": 0, + "high": 0.1843181350088518 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 61, + "inScope": 0, + "aiIntentLike": 8, + "aiOther": 53, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 61, + "rate": 0, + "low": 0, + "high": 0.05924386789811283 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 170, + "inScope": 0, + "aiIntentLike": 161, + "aiOther": 9, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 170, + "rate": 0, + "low": 0, + "high": 0.022097483803655452 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 65, + "inScope": 0, + "aiIntentLike": 65, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 65, + "rate": 0, + "low": 0, + "high": 0.05580153132285689 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 9, + "inScope": 0, + "aiIntentLike": 9, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 9, + "rate": 0, + "low": 0, + "high": 0.2991450484195441 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 38, + "inScope": 0, + "aiIntentLike": 31, + "aiOther": 7, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 38, + "rate": 0, + "low": 0, + "high": 0.09180986822558396 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.day": { + "conversations": 22, + "inScope": 22, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 19, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 19, + "relabelled": 0, + "coverage": 0.8636363636363636, + "coverageOfAll": 0.8636363636363636, + "precision": 1, + "intentLikeCoverage": 0.8636363636363636, + "rubricPass": 1, + "qualityCoverage": 0.8636363636363636, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 19, + "n": 22, + "rate": 0.8636363636363636, + "low": 0.6666501283342828, + "high": 0.9525099628594474 + }, + "coverageOfAll": { + "k": 19, + "n": 22, + "rate": 0.8636363636363636, + "low": 0.6666501283342828, + "high": 0.9525099628594474 + }, + "precision": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "rubricPass": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "execution": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 9, + "inScope": 9, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 9, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 9, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "coverageOfAll": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "precision": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "rubricPass": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "execution": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "event.create": { + "conversations": 53, + "inScope": 53, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 51, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 51, + "relabelled": 0, + "coverage": 0.9622641509433962, + "coverageOfAll": 0.9622641509433962, + "precision": 1, + "intentLikeCoverage": 0.9622641509433962, + "rubricPass": 1, + "qualityCoverage": 0.9622641509433962, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 51, + "n": 53, + "rate": 0.9622641509433962, + "low": 0.8724571280258164, + "high": 0.9895897127241731 + }, + "coverageOfAll": { + "k": 51, + "n": 53, + "rate": 0.9622641509433962, + "low": 0.8724571280258164, + "high": 0.9895897127241731 + }, + "precision": { + "k": 51, + "n": 51, + "rate": 1, + "low": 0.9299533801014687, + "high": 1 + }, + "rubricPass": { + "k": 51, + "n": 51, + "rate": 1, + "low": 0.9299533801014687, + "high": 1 + }, + "execution": { + "k": 51, + "n": 51, + "rate": 1, + "low": 0.9299533801014687, + "high": 1 + } + } + }, + "event.create_invite": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.create_range": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.3333333333333333, + "coverageOfAll": 0.3333333333333333, + "precision": 1, + "intentLikeCoverage": 0.3333333333333333, + "rubricPass": 1, + "qualityCoverage": 0.3333333333333333, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 3, + "rate": 0.3333333333333333, + "low": 0.06149194472039621, + "high": 0.7923403991979522 + }, + "coverageOfAll": { + "k": 1, + "n": 3, + "rate": 0.3333333333333333, + "low": 0.06149194472039621, + "high": 0.7923403991979522 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 5, + "inScope": 5, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 0.8, + "coverageOfAll": 0.8, + "precision": 1, + "intentLikeCoverage": 0.8, + "rubricPass": 1, + "qualityCoverage": 0.8, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 5, + "rate": 0.8, + "low": 0.37553462976252533, + "high": 0.9637758913675698 + }, + "coverageOfAll": { + "k": 4, + "n": 5, + "rate": 0.8, + "low": 0.37553462976252533, + "high": 0.9637758913675698 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 0.6666666666666666, + "coverageOfAll": 0.6666666666666666, + "precision": 1, + "intentLikeCoverage": 0.6666666666666666, + "rubricPass": 1, + "qualityCoverage": 0.6666666666666666, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "coverageOfAll": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "reminder.after": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "settings.view": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "time.now": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 496, + "inScope": 112, + "aiIntentLike": 315, + "aiOther": 69, + "correct": 96, + "wrong": 0, + "missed": 16, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 96, + "relabelled": 0, + "coverage": 0.8571428571428571, + "coverageOfAll": 0.1935483870967742, + "precision": 1, + "intentLikeCoverage": 0.22482435597189696, + "rubricPass": 1, + "qualityCoverage": 0.8571428571428571, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 96, + "n": 112, + "rate": 0.8571428571428571, + "low": 0.7804856487003439, + "high": 0.9101133891089818 + }, + "coverageOfAll": { + "k": 96, + "n": 496, + "rate": 0.1935483870967742, + "low": 0.16118853653117643, + "high": 0.23061861624633545 + }, + "precision": { + "k": 96, + "n": 96, + "rate": 1, + "low": 0.9615244121423243, + "high": 1 + }, + "rubricPass": { + "k": 96, + "n": 96, + "rate": 1, + "low": 0.9615244121423243, + "high": 1 + }, + "execution": { + "k": 96, + "n": 96, + "rate": 1, + "low": 0.9615244121423243, + "high": 1 + } + } + } + }, + "byRule": { + "basis.event.create": 52, + "basis.calendar.day": 19, + "basis.calendar.period": 9, + "basis.contacts.list": 4, + "basis.event.delete": 4, + "basis.event.reschedule": 2, + "basis.calendar.image": 2, + "basis.invite.status": 2, + "basis.calendar.when": 1, + "basis.bot.info": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 16 + }, + "rubricFindings": {} + } + }, + "unclear": { + "dev": { + "conversations": 30, + "inScope": 9, + "aiIntentLike": 17, + "aiOther": 4, + "correct": 2, + "wrong": 0, + "missed": 7, + "falseMatch": 4, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 0.2222222222222222, + "coverageOfAll": 0.06666666666666667, + "precision": 0.3333333333333333, + "intentLikeCoverage": 0.07692307692307693, + "rubricPass": 1, + "qualityCoverage": 0.2222222222222222, + "aiFallbackCorrect": 0.8095238095238095, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 9, + "rate": 0.2222222222222222, + "low": 0.06322510711784671, + "high": 0.5474110308930111 + }, + "coverageOfAll": { + "k": 2, + "n": 30, + "rate": 0.06666666666666667, + "low": 0.018477023791270378, + "high": 0.2132345836261692 + }, + "precision": { + "k": 2, + "n": 6, + "rate": 0.3333333333333333, + "low": 0.09677141110578041, + "high": 0.700006684861608 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "heldout": { + "conversations": 13, + "inScope": 4, + "aiIntentLike": 3, + "aiOther": 6, + "correct": 2, + "wrong": 0, + "missed": 2, + "falseMatch": 1, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.15384615384615385, + "precision": 0.6666666666666666, + "intentLikeCoverage": 0.2857142857142857, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": 0.8888888888888888, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 4, + "rate": 0.5, + "low": 0.15003898915214947, + "high": 0.8499610108478506 + }, + "coverageOfAll": { + "k": 2, + "n": 13, + "rate": 0.15384615384615385, + "low": 0.043258178358081184, + "high": 0.4223463101948254 + }, + "precision": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + } + }, + "byUser": { + "u01": { + "conversations": 29, + "inScope": 0, + "aiIntentLike": 27, + "aiOther": 2, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 29, + "rate": 0, + "low": 0, + "high": 0.11696979849974075 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u02": { + "conversations": 11, + "inScope": 2, + "aiIntentLike": 8, + "aiOther": 1, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 0.18181818181818182, + "precision": 1, + "intentLikeCoverage": 0.2, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 11, + "rate": 0.18181818181818182, + "low": 0.05136768974608513, + "high": 0.4769805619608442 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "u03": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 1, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u04": { + "conversations": 4, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 4, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 4, + "rate": 0, + "low": 0, + "high": 0.4898908364545973 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u05": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 1, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u06": { + "conversations": 11, + "inScope": 0, + "aiIntentLike": 11, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 11, + "rate": 0, + "low": 0, + "high": 0.2588329669680317 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u07": { + "conversations": 188, + "inScope": 26, + "aiIntentLike": 120, + "aiOther": 42, + "correct": 19, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 19, + "relabelled": 0, + "coverage": 0.7307692307692307, + "coverageOfAll": 0.10106382978723404, + "precision": 1, + "intentLikeCoverage": 0.13013698630136986, + "rubricPass": 1, + "qualityCoverage": 0.7307692307692307, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 19, + "n": 26, + "rate": 0.7307692307692307, + "low": 0.5391695962082381, + "high": 0.8629555167798162 + }, + "coverageOfAll": { + "k": 19, + "n": 188, + "rate": 0.10106382978723404, + "low": 0.06565857689789162, + "high": 0.1524457842652578 + }, + "precision": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "rubricPass": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "execution": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + } + } + }, + "u08": { + "conversations": 6, + "inScope": 0, + "aiIntentLike": 5, + "aiOther": 1, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 6, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.3903342879021653 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u09": { + "conversations": 218, + "inScope": 84, + "aiIntentLike": 120, + "aiOther": 14, + "correct": 75, + "wrong": 0, + "missed": 9, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 75, + "relabelled": 0, + "coverage": 0.8928571428571429, + "coverageOfAll": 0.3440366972477064, + "precision": 1, + "intentLikeCoverage": 0.36764705882352944, + "rubricPass": 1, + "qualityCoverage": 0.8928571428571429, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 75, + "n": 84, + "rate": 0.8928571428571429, + "low": 0.8087538548365549, + "high": 0.9425997871233551 + }, + "coverageOfAll": { + "k": 75, + "n": 218, + "rate": 0.3440366972477064, + "low": 0.28416629211048217, + "high": 0.4093084967703384 + }, + "precision": { + "k": 75, + "n": 75, + "rate": 1, + "low": 0.9512761575171942, + "high": 1 + }, + "rubricPass": { + "k": 75, + "n": 75, + "rate": 1, + "low": 0.9512761575171942, + "high": 1 + }, + "execution": { + "k": 75, + "n": 75, + "rate": 1, + "low": 0.9512761575171942, + "high": 1 + } + } + }, + "u10": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 1, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u11": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 1, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u12": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 1, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u13": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u14": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u15": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u16": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u17": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u18": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u19": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u20": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u21": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u22": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u23": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u24": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u25": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u26": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u27": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u28": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u29": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u30": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + } + } + }, + "latency": { + "matcherMicrosP50": 37.17999999935273, + "matcherMicrosP95": 224.29100000590552, + "matcherMicrosMax": 3022.12999999756, + "layerMsP50": 14.574628999999959, + "layerMsP95": 21.798281999999745, + "samples": 1617 + }, + "historyKind": "real", + "realHistory": { + "corpusChecksum": "2f793c2471dc077084a074d650ea29306a25a0998544b8174e9377d65a1d2de9", + "labelsChecksum": "19ee899c60362252cd89a08b71fc5621f417f2ece970dbf58222792ffff05f7e", + "splitChecksum": "310707bb8c752d46692ceb9b50cc86bda217b681ac54867263ca3e8dda8354f0", + "excludedQaProbes": 13, + "heldoutShare": 0.3395176252319109 + }, + "generatedAt": "2026-11-02T05:34:43.283Z", + "source": { + "sha": "5169da3c5ac8a1b5d132ac45c05ea1d07625724c", + "dirty": true, + "engineDigest": "736154156269de819a0a775de5e6162f364a300052aca709ea53273c941d4d2f" + }, + "checksum": "3098ab6a172515d6aad228b6acdd6ece872555a665bb7ee3879f79edb8ceb405" +} diff --git a/docs/intents/benchmarks/20261102T053634Z-synthetic-current.json b/docs/intents/benchmarks/20261102T053634Z-synthetic-current.json new file mode 100644 index 000000000..7995bc44d --- /dev/null +++ b/docs/intents/benchmarks/20261102T053634Z-synthetic-current.json @@ -0,0 +1,12195 @@ +{ + "schemaVersion": 2, + "label": "current", + "ruleSet": { + "fingerprint": "77224e790a7503b776e6d5650325890439081b0ba34c5aca99b9bd93afe6d007", + "rules": 54 + }, + "modelHistory": { + "version": 1, + "checksum": "98447f88cd71fa57404321b1da6fad4df63a2784d8ace86de0aa3eb84d1f64f8" + }, + "dev": { + "overall": { + "conversations": 508, + "inScope": 358, + "aiIntentLike": 128, + "aiOther": 22, + "correct": 338, + "wrong": 0, + "missed": 20, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 336, + "relabelled": 1, + "coverage": 0.9441340782122905, + "coverageOfAll": 0.6653543307086615, + "precision": 1, + "intentLikeCoverage": 0.6954732510288066, + "rubricPass": 0.9940828402366864, + "qualityCoverage": 0.9385474860335196, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 338, + "n": 358, + "rate": 0.9441340782122905, + "low": 0.9152902660679959, + "high": 0.963547667640371 + }, + "coverageOfAll": { + "k": 338, + "n": 508, + "rate": 0.6653543307086615, + "low": 0.6232155603385077, + "high": 0.7050110752823731 + }, + "precision": { + "k": 338, + "n": 338, + "rate": 1, + "low": 0.9887624548703174, + "high": 1 + }, + "rubricPass": { + "k": 336, + "n": 338, + "rate": 0.9940828402366864, + "low": 0.9786853253275402, + "high": 0.9983757987159095 + }, + "execution": { + "k": 338, + "n": 338, + "rate": 1, + "low": 0.9887624548703174, + "high": 1 + } + } + }, + "byFamily": { + "ai:advice_or_analysis": { + "conversations": 12, + "inScope": 0, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 12, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.24249400665524085 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:ambiguous_day": { + "conversations": 6, + "inScope": 0, + "aiIntentLike": 6, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 6, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.3903342879021653 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 23, + "inScope": 0, + "aiIntentLike": 23, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 23, + "rate": 0, + "low": 0, + "high": 0.14311661845042686 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 11, + "inScope": 0, + "aiIntentLike": 11, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 11, + "rate": 0, + "low": 0, + "high": 0.2588329669680317 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 22, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 22, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 22, + "rate": 0, + "low": 0, + "high": 0.14865487460862087 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 21, + "inScope": 0, + "aiIntentLike": 21, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 21, + "rate": 0, + "low": 0, + "high": 0.154639018924847 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 13, + "inScope": 0, + "aiIntentLike": 13, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 13, + "rate": 0, + "low": 0, + "high": 0.22809537235419838 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:negation_or_quote": { + "conversations": 13, + "inScope": 0, + "aiIntentLike": 13, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 13, + "rate": 0, + "low": 0, + "high": 0.22809537235419838 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 8, + "inScope": 0, + "aiIntentLike": 8, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 8, + "rate": 0, + "low": 0, + "high": 0.32440756488388023 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 7, + "inScope": 0, + "aiIntentLike": 7, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 7, + "rate": 0, + "low": 5.551115123125783e-17, + "high": 0.35433043506668743 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 13, + "inScope": 0, + "aiIntentLike": 13, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 13, + "rate": 0, + "low": 0, + "high": 0.22809537235419838 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "calc.evaluate": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "calendar.count": { + "conversations": 5, + "inScope": 5, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "coverageOfAll": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "calendar.day": { + "conversations": 47, + "inScope": 47, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 46, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 46, + "relabelled": 0, + "coverage": 0.9787234042553191, + "coverageOfAll": 0.9787234042553191, + "precision": 1, + "intentLikeCoverage": 0.9787234042553191, + "rubricPass": 1, + "qualityCoverage": 0.9787234042553191, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 46, + "n": 47, + "rate": 0.9787234042553191, + "low": 0.8888701895837066, + "high": 0.996234231961403 + }, + "coverageOfAll": { + "k": 46, + "n": 47, + "rate": 0.9787234042553191, + "low": 0.8888701895837066, + "high": 0.996234231961403 + }, + "precision": { + "k": 46, + "n": 46, + "rate": 1, + "low": 0.9229264369144198, + "high": 1 + }, + "rubricPass": { + "k": 46, + "n": 46, + "rate": 1, + "low": 0.9229264369144198, + "high": 1 + }, + "execution": { + "k": 46, + "n": 46, + "rate": 1, + "low": 0.9229264369144198, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 0.5, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 4, + "rate": 0.5, + "low": 0.15003898915214947, + "high": 0.8499610108478506 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 14, + "inScope": 14, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 14, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 14, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "coverageOfAll": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "precision": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "rubricPass": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "execution": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + } + } + }, + "calendar.search": { + "conversations": 6, + "inScope": 6, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 6, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 6, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "coverageOfAll": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "precision": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "rubricPass": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "execution": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + } + } + }, + "calendar.upcoming": { + "conversations": 8, + "inScope": 8, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 8, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 8, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "coverageOfAll": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "precision": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "rubricPass": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "execution": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 10, + "inScope": 10, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 10, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 10, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "precision": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "execution": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + } + } + }, + "contacts.find": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 0.6666666666666666, + "coverageOfAll": 0.6666666666666666, + "precision": 1, + "intentLikeCoverage": 0.6666666666666666, + "rubricPass": 1, + "qualityCoverage": 0.6666666666666666, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "coverageOfAll": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "event.create": { + "conversations": 80, + "inScope": 80, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 73, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 73, + "relabelled": 0, + "coverage": 0.9125, + "coverageOfAll": 0.9125, + "precision": 1, + "intentLikeCoverage": 0.9125, + "rubricPass": 1, + "qualityCoverage": 0.9125, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 73, + "n": 80, + "rate": 0.9125, + "low": 0.8302320449152499, + "high": 0.9569679982179401 + }, + "coverageOfAll": { + "k": 73, + "n": 80, + "rate": 0.9125, + "low": 0.8302320449152499, + "high": 0.9569679982179401 + }, + "precision": { + "k": 73, + "n": 73, + "rate": 1, + "low": 0.9500079920442688, + "high": 0.9999999999999998 + }, + "rubricPass": { + "k": 73, + "n": 73, + "rate": 1, + "low": 0.9500079920442688, + "high": 0.9999999999999998 + }, + "execution": { + "k": 73, + "n": 73, + "rate": 1, + "low": 0.9500079920442688, + "high": 0.9999999999999998 + } + } + }, + "event.create_invite": { + "conversations": 26, + "inScope": 26, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 19, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 19, + "relabelled": 0, + "coverage": 0.7307692307692307, + "coverageOfAll": 0.7307692307692307, + "precision": 1, + "intentLikeCoverage": 0.7307692307692307, + "rubricPass": 1, + "qualityCoverage": 0.7307692307692307, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 19, + "n": 26, + "rate": 0.7307692307692307, + "low": 0.5391695962082381, + "high": 0.8629555167798162 + }, + "coverageOfAll": { + "k": 19, + "n": 26, + "rate": 0.7307692307692307, + "low": 0.5391695962082381, + "high": 0.8629555167798162 + }, + "precision": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "rubricPass": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "execution": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + } + } + }, + "event.create_range": { + "conversations": 10, + "inScope": 10, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 8, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 8, + "relabelled": 0, + "coverage": 0.8, + "coverageOfAll": 0.8, + "precision": 1, + "intentLikeCoverage": 0.8, + "rubricPass": 1, + "qualityCoverage": 0.8, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 8, + "n": 10, + "rate": 0.8, + "low": 0.49016247153664183, + "high": 0.9433178485456247 + }, + "coverageOfAll": { + "k": 8, + "n": 10, + "rate": 0.8, + "low": 0.49016247153664183, + "high": 0.9433178485456247 + }, + "precision": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "rubricPass": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "execution": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 32, + "inScope": 32, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 32, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 32, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 32, + "n": 32, + "rate": 1, + "low": 0.8928208017449293, + "high": 1 + }, + "coverageOfAll": { + "k": 32, + "n": 32, + "rate": 1, + "low": 0.8928208017449293, + "high": 1 + }, + "precision": { + "k": 32, + "n": 32, + "rate": 1, + "low": 0.8928208017449293, + "high": 1 + }, + "rubricPass": { + "k": 32, + "n": 32, + "rate": 1, + "low": 0.8928208017449293, + "high": 1 + }, + "execution": { + "k": 32, + "n": 32, + "rate": 1, + "low": 0.8928208017449293, + "high": 1 + } + } + }, + "event.hide": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "event.rename": { + "conversations": 5, + "inScope": 5, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "coverageOfAll": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 25, + "inScope": 24, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 24, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 24, + "relabelled": 1, + "coverage": 1, + "coverageOfAll": 0.96, + "precision": 1, + "intentLikeCoverage": 0.96, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 24, + "n": 24, + "rate": 1, + "low": 0.8620237953250198, + "high": 1 + }, + "coverageOfAll": { + "k": 24, + "n": 25, + "rate": 0.96, + "low": 0.8045593626380625, + "high": 0.9929037664980127 + }, + "precision": { + "k": 24, + "n": 24, + "rate": 1, + "low": 0.8620237953250198, + "high": 1 + }, + "rubricPass": { + "k": 24, + "n": 24, + "rate": 1, + "low": 0.8620237953250198, + "high": 1 + }, + "execution": { + "k": 24, + "n": 24, + "rate": 1, + "low": 0.8620237953250198, + "high": 1 + } + } + }, + "event.set_detail": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "event.show": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "event.snooze": { + "conversations": 6, + "inScope": 6, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 0.8333333333333334, + "coverageOfAll": 0.8333333333333334, + "precision": 1, + "intentLikeCoverage": 0.8333333333333334, + "rubricPass": 1, + "qualityCoverage": 0.8333333333333334, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 6, + "rate": 0.8333333333333334, + "low": 0.43649717781352987, + "high": 0.9699466302516935 + }, + "coverageOfAll": { + "k": 5, + "n": 6, + "rate": 0.8333333333333334, + "low": 0.43649717781352987, + "high": 0.9699466302516935 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "google.status": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "holidays.upcoming": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "invite.send": { + "conversations": 8, + "inScope": 8, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 8, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 8, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "coverageOfAll": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "precision": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "rubricPass": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "execution": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "reminder.after": { + "conversations": 10, + "inScope": 10, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 10, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 10, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "precision": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "execution": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + } + } + }, + "reminder.clear": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "reminder.list": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "reminder.set": { + "conversations": 7, + "inScope": 7, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 7, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 7, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "coverageOfAll": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "precision": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "rubricPass": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "execution": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + } + } + }, + "settings.view": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "slots.check_time": { + "conversations": 9, + "inScope": 9, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 8, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 8, + "relabelled": 0, + "coverage": 0.8888888888888888, + "coverageOfAll": 0.8888888888888888, + "precision": 1, + "intentLikeCoverage": 0.8888888888888888, + "rubricPass": 1, + "qualityCoverage": 0.8888888888888888, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 8, + "n": 9, + "rate": 0.8888888888888888, + "low": 0.5650002944233439, + "high": 0.9801091123614551 + }, + "coverageOfAll": { + "k": 8, + "n": 9, + "rate": 0.8888888888888888, + "low": 0.5650002944233439, + "high": 0.9801091123614551 + }, + "precision": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "rubricPass": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "execution": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + } + } + }, + "slots.day": { + "conversations": 8, + "inScope": 8, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 8, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 8, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "coverageOfAll": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "precision": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "rubricPass": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "execution": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + } + } + }, + "time.now": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + } + }, + "byStyle": { + "typo": { + "conversations": 29, + "inScope": 28, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 25, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 25, + "relabelled": 0, + "coverage": 0.8928571428571429, + "coverageOfAll": 0.8620689655172413, + "precision": 1, + "intentLikeCoverage": 0.8620689655172413, + "rubricPass": 1, + "qualityCoverage": 0.8928571428571429, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 25, + "n": 28, + "rate": 0.8928571428571429, + "low": 0.7280414845132785, + "high": 0.9628816331707 + }, + "coverageOfAll": { + "k": 25, + "n": 29, + "rate": 0.8620689655172413, + "low": 0.6944100627964798, + "high": 0.9450256003588804 + }, + "precision": { + "k": 25, + "n": 25, + "rate": 1, + "low": 0.8668077490609515, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 25, + "n": 25, + "rate": 1, + "low": 0.8668077490609515, + "high": 0.9999999999999999 + }, + "execution": { + "k": 25, + "n": 25, + "rate": 1, + "low": 0.8668077490609515, + "high": 0.9999999999999999 + } + } + }, + "plain": { + "conversations": 131, + "inScope": 69, + "aiIntentLike": 49, + "aiOther": 13, + "correct": 62, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 62, + "relabelled": 0, + "coverage": 0.8985507246376812, + "coverageOfAll": 0.4732824427480916, + "precision": 1, + "intentLikeCoverage": 0.5254237288135594, + "rubricPass": 1, + "qualityCoverage": 0.8985507246376812, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 62, + "n": 69, + "rate": 0.8985507246376812, + "low": 0.8050811424505202, + "high": 0.9499833618915401 + }, + "coverageOfAll": { + "k": 62, + "n": 131, + "rate": 0.4732824427480916, + "low": 0.3897677343642162, + "high": 0.5583194484561042 + }, + "precision": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + }, + "execution": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + } + } + }, + "no_punct": { + "conversations": 155, + "inScope": 124, + "aiIntentLike": 26, + "aiOther": 5, + "correct": 121, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 119, + "relabelled": 0, + "coverage": 0.9758064516129032, + "coverageOfAll": 0.7806451612903226, + "precision": 1, + "intentLikeCoverage": 0.8066666666666666, + "rubricPass": 0.9834710743801653, + "qualityCoverage": 0.9596774193548387, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 121, + "n": 124, + "rate": 0.9758064516129032, + "low": 0.9312798718154085, + "high": 0.9917383813281402 + }, + "coverageOfAll": { + "k": 121, + "n": 155, + "rate": 0.7806451612903226, + "low": 0.7091483707205343, + "high": 0.8385675760223666 + }, + "precision": { + "k": 121, + "n": 121, + "rate": 1, + "low": 0.9692293020525218, + "high": 1 + }, + "rubricPass": { + "k": 119, + "n": 121, + "rate": 0.9834710743801653, + "low": 0.9417332575142658, + "high": 0.9954554064538752 + }, + "execution": { + "k": 121, + "n": 121, + "rate": 1, + "low": 0.9692293020525218, + "high": 1 + } + } + }, + "caps": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "trailing_punct": { + "conversations": 57, + "inScope": 43, + "aiIntentLike": 10, + "aiOther": 4, + "correct": 42, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 1, + "coverage": 0.9767441860465116, + "coverageOfAll": 0.7368421052631579, + "precision": 1, + "intentLikeCoverage": 0.7924528301886793, + "rubricPass": 1, + "qualityCoverage": 0.9767441860465116, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 43, + "rate": 0.9767441860465116, + "low": 0.8794100655939282, + "high": 0.9958829127256662 + }, + "coverageOfAll": { + "k": 42, + "n": 57, + "rate": 0.7368421052631579, + "low": 0.6102342073651618, + "high": 0.8335421340202692 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "terse": { + "conversations": 148, + "inScope": 104, + "aiIntentLike": 41, + "aiOther": 3, + "correct": 103, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 102, + "relabelled": 0, + "coverage": 0.9903846153846154, + "coverageOfAll": 0.6959459459459459, + "precision": 1, + "intentLikeCoverage": 0.7103448275862069, + "rubricPass": 0.9902912621359223, + "qualityCoverage": 0.9807692307692307, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 103, + "n": 104, + "rate": 0.9903846153846154, + "low": 0.9475322752920542, + "high": 0.9983006271503486 + }, + "coverageOfAll": { + "k": 103, + "n": 148, + "rate": 0.6959459459459459, + "low": 0.6176538029488693, + "high": 0.7643235598245665 + }, + "precision": { + "k": 103, + "n": 103, + "rate": 1, + "low": 0.964045241771352, + "high": 1 + }, + "rubricPass": { + "k": 102, + "n": 103, + "rate": 0.9902912621359223, + "low": 0.9470418081674632, + "high": 0.99828410852095 + }, + "execution": { + "k": 103, + "n": 103, + "rate": 1, + "low": 0.964045241771352, + "high": 1 + } + } + }, + "verbless": { + "conversations": 53, + "inScope": 50, + "aiIntentLike": 3, + "aiOther": 0, + "correct": 48, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 47, + "relabelled": 0, + "coverage": 0.96, + "coverageOfAll": 0.9056603773584906, + "precision": 1, + "intentLikeCoverage": 0.9056603773584906, + "rubricPass": 0.9791666666666666, + "qualityCoverage": 0.94, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 48, + "n": 50, + "rate": 0.96, + "low": 0.8653990931249297, + "high": 0.9889611156723802 + }, + "coverageOfAll": { + "k": 48, + "n": 53, + "rate": 0.9056603773584906, + "low": 0.7974627262370373, + "high": 0.9590273585027496 + }, + "precision": { + "k": 48, + "n": 48, + "rate": 1, + "low": 0.9258998703338824, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 47, + "n": 48, + "rate": 0.9791666666666666, + "low": 0.8910078200474814, + "high": 0.9963128890224893 + }, + "execution": { + "k": 48, + "n": 48, + "rate": 1, + "low": 0.9258998703338824, + "high": 0.9999999999999999 + } + } + }, + "slang": { + "conversations": 26, + "inScope": 25, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 23, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 22, + "relabelled": 0, + "coverage": 0.92, + "coverageOfAll": 0.8846153846153846, + "precision": 1, + "intentLikeCoverage": 0.8846153846153846, + "rubricPass": 0.9565217391304348, + "qualityCoverage": 0.88, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 23, + "n": 25, + "rate": 0.92, + "low": 0.7503389104960468, + "high": 0.9777795987151524 + }, + "coverageOfAll": { + "k": 23, + "n": 26, + "rate": 0.8846153846153846, + "low": 0.7102409678828635, + "high": 0.9599675537638938 + }, + "precision": { + "k": 23, + "n": 23, + "rate": 1, + "low": 0.8568833815495731, + "high": 1 + }, + "rubricPass": { + "k": 22, + "n": 23, + "rate": 0.9565217391304348, + "low": 0.7900884492974114, + "high": 0.9922833338565469 + }, + "execution": { + "k": 23, + "n": 23, + "rate": 1, + "low": 0.8568833815495731, + "high": 1 + } + } + }, + "polite": { + "conversations": 24, + "inScope": 23, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 19, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 19, + "relabelled": 0, + "coverage": 0.8260869565217391, + "coverageOfAll": 0.7916666666666666, + "precision": 1, + "intentLikeCoverage": 0.7916666666666666, + "rubricPass": 1, + "qualityCoverage": 0.8260869565217391, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 19, + "n": 23, + "rate": 0.8260869565217391, + "low": 0.6286235235977385, + "high": 0.9302134643693744 + }, + "coverageOfAll": { + "k": 19, + "n": 24, + "rate": 0.7916666666666666, + "low": 0.5952954674681133, + "high": 0.9075517464714814 + }, + "precision": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "rubricPass": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "execution": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + } + } + }, + "long": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "mixed_lang": { + "conversations": 12, + "inScope": 11, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 10, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 10, + "relabelled": 0, + "coverage": 0.9090909090909091, + "coverageOfAll": 0.8333333333333334, + "precision": 1, + "intentLikeCoverage": 0.8333333333333334, + "rubricPass": 1, + "qualityCoverage": 0.9090909090909091, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 10, + "n": 11, + "rate": 0.9090909090909091, + "low": 0.6226415635484043, + "high": 0.9837678271141151 + }, + "coverageOfAll": { + "k": 10, + "n": 12, + "rate": 0.8333333333333334, + "low": 0.5519691377470266, + "high": 0.9530348578161463 + }, + "precision": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "execution": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + } + } + } + }, + "byRule": { + "basis.event.create": 81, + "basis.calendar.day": 46, + "basis.event.delete": 32, + "basis.event.reschedule": 24, + "basis.event.create_invite": 19, + "basis.calendar.period": 14, + "basis.reminder.after": 10, + "basis.calendar.when": 10, + "basis.slots.check_time": 8, + "basis.slots.day": 8, + "basis.invite.send": 8, + "basis.calendar.upcoming": 7, + "basis.reminder.set": 7, + "basis.calendar.search": 6, + "basis.event.rename": 5, + "basis.event.snooze": 5, + "basis.calendar.count": 5, + "basis.time.now": 4, + "basis.invite.status": 4, + "basis.event.set_detail": 4, + "basis.calendar.image": 4, + "basis.event.show": 3, + "basis.event.hide": 3, + "basis.holidays.upcoming": 3, + "basis.calc.evaluate": 3, + "basis.contacts.find": 2, + "basis.settings.view": 2, + "basis.bot.info": 2, + "basis.reminder.list": 2, + "basis.google.status": 2, + "basis.reminder.clear": 2, + "basis.contacts.list": 2, + "basis.calendar.next": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 20 + }, + "rubricFindings": { + "RAW_ISO_DATE": 2 + } + }, + "heldout": { + "overall": { + "conversations": 247, + "inScope": 185, + "aiIntentLike": 57, + "aiOther": 5, + "correct": 153, + "wrong": 0, + "missed": 32, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 153, + "relabelled": 0, + "coverage": 0.827027027027027, + "coverageOfAll": 0.6194331983805668, + "precision": 1, + "intentLikeCoverage": 0.6322314049586777, + "rubricPass": 1, + "qualityCoverage": 0.827027027027027, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 153, + "n": 185, + "rate": 0.827027027027027, + "low": 0.7660212684254935, + "high": 0.8747278592059029 + }, + "coverageOfAll": { + "k": 153, + "n": 247, + "rate": 0.6194331983805668, + "low": 0.5574920153260318, + "high": 0.6777163121864239 + }, + "precision": { + "k": 153, + "n": 153, + "rate": 1, + "low": 0.9755073763686055, + "high": 1 + }, + "rubricPass": { + "k": 153, + "n": 153, + "rate": 1, + "low": 0.9755073763686055, + "high": 1 + }, + "execution": { + "k": 153, + "n": 153, + "rate": 1, + "low": 0.9755073763686055, + "high": 1 + } + } + }, + "byFamily": { + "ai:advice_or_analysis": { + "conversations": 3, + "inScope": 0, + "aiIntentLike": 3, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 3, + "rate": 0, + "low": 5.551115123125783e-17, + "high": 0.5614970317550454 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:ambiguous_day": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 17, + "inScope": 0, + "aiIntentLike": 17, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 17, + "rate": 0, + "low": 0, + "high": 0.1843181350088518 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 5, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 5, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 5, + "rate": 0, + "low": 0, + "high": 0.43448246478317476 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 8, + "inScope": 0, + "aiIntentLike": 8, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 8, + "rate": 0, + "low": 0, + "high": 0.32440756488388023 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 7, + "inScope": 0, + "aiIntentLike": 7, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 7, + "rate": 0, + "low": 5.551115123125783e-17, + "high": 0.35433043506668743 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:negation_or_quote": { + "conversations": 7, + "inScope": 0, + "aiIntentLike": 7, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 7, + "rate": 0, + "low": 5.551115123125783e-17, + "high": 0.35433043506668743 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 4, + "inScope": 0, + "aiIntentLike": 4, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 4, + "rate": 0, + "low": 0, + "high": 0.4898908364545973 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 4, + "inScope": 0, + "aiIntentLike": 4, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 4, + "rate": 0, + "low": 0, + "high": 0.4898908364545973 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 4, + "inScope": 0, + "aiIntentLike": 4, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 4, + "rate": 0, + "low": 0, + "high": 0.4898908364545973 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "calc.evaluate": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.count": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 0.75, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.75, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "coverageOfAll": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "calendar.day": { + "conversations": 24, + "inScope": 24, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 18, + "wrong": 0, + "missed": 6, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 18, + "relabelled": 0, + "coverage": 0.75, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.75, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 18, + "n": 24, + "rate": 0.75, + "low": 0.55100555994846, + "high": 0.8800063377140499 + }, + "coverageOfAll": { + "k": 18, + "n": 24, + "rate": 0.75, + "low": 0.55100555994846, + "high": 0.8800063377140499 + }, + "precision": { + "k": 18, + "n": 18, + "rate": 1, + "low": 0.8241207763533425, + "high": 1 + }, + "rubricPass": { + "k": 18, + "n": 18, + "rate": 1, + "low": 0.8241207763533425, + "high": 1 + }, + "execution": { + "k": 18, + "n": 18, + "rate": 1, + "low": 0.8241207763533425, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 7, + "inScope": 7, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 0.7142857142857143, + "coverageOfAll": 0.7142857142857143, + "precision": 1, + "intentLikeCoverage": 0.7142857142857143, + "rubricPass": 1, + "qualityCoverage": 0.7142857142857143, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 7, + "rate": 0.7142857142857143, + "low": 0.35893445183261935, + "high": 0.9177810759959433 + }, + "coverageOfAll": { + "k": 5, + "n": 7, + "rate": 0.7142857142857143, + "low": 0.35893445183261935, + "high": 0.9177810759959433 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "calendar.search": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 0.75, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.75, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "coverageOfAll": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "calendar.upcoming": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 5, + "inScope": 5, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "coverageOfAll": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "contacts.find": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.create": { + "conversations": 45, + "inScope": 45, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 41, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 41, + "relabelled": 0, + "coverage": 0.9111111111111111, + "coverageOfAll": 0.9111111111111111, + "precision": 1, + "intentLikeCoverage": 0.9111111111111111, + "rubricPass": 1, + "qualityCoverage": 0.9111111111111111, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 41, + "n": 45, + "rate": 0.9111111111111111, + "low": 0.7926642466571064, + "high": 0.9648888832736299 + }, + "coverageOfAll": { + "k": 41, + "n": 45, + "rate": 0.9111111111111111, + "low": 0.7926642466571064, + "high": 0.9648888832736299 + }, + "precision": { + "k": 41, + "n": 41, + "rate": 1, + "low": 0.9143324298155681, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 41, + "n": 41, + "rate": 1, + "low": 0.9143324298155681, + "high": 0.9999999999999999 + }, + "execution": { + "k": 41, + "n": 41, + "rate": 1, + "low": 0.9143324298155681, + "high": 0.9999999999999999 + } + } + }, + "event.create_invite": { + "conversations": 17, + "inScope": 17, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 10, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 10, + "relabelled": 0, + "coverage": 0.5882352941176471, + "coverageOfAll": 0.5882352941176471, + "precision": 1, + "intentLikeCoverage": 0.5882352941176471, + "rubricPass": 1, + "qualityCoverage": 0.5882352941176471, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 10, + "n": 17, + "rate": 0.5882352941176471, + "low": 0.3600544264299978, + "high": 0.7838894320978518 + }, + "coverageOfAll": { + "k": 10, + "n": 17, + "rate": 0.5882352941176471, + "low": 0.3600544264299978, + "high": 0.7838894320978518 + }, + "precision": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "execution": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + } + } + }, + "event.create_range": { + "conversations": 6, + "inScope": 6, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 6, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 6, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "coverageOfAll": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "precision": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "rubricPass": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "execution": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 15, + "inScope": 15, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 15, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 15, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "coverageOfAll": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "precision": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "rubricPass": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "execution": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + } + } + }, + "event.hide": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.rename": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 18, + "inScope": 18, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 14, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 14, + "relabelled": 0, + "coverage": 0.7777777777777778, + "coverageOfAll": 0.7777777777777778, + "precision": 1, + "intentLikeCoverage": 0.7777777777777778, + "rubricPass": 1, + "qualityCoverage": 0.7777777777777778, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 14, + "n": 18, + "rate": 0.7777777777777778, + "low": 0.5478541568378739, + "high": 0.9099907189139831 + }, + "coverageOfAll": { + "k": 14, + "n": 18, + "rate": 0.7777777777777778, + "low": 0.5478541568378739, + "high": 0.9099907189139831 + }, + "precision": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "rubricPass": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "execution": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + } + } + }, + "event.set_detail": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.show": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.snooze": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "google.status": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "holidays.upcoming": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "invite.send": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "reminder.after": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 0.75, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.75, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "coverageOfAll": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "reminder.clear": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "reminder.list": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "reminder.set": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "slots.check_time": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 0.6666666666666666, + "coverageOfAll": 0.6666666666666666, + "precision": 1, + "intentLikeCoverage": 0.6666666666666666, + "rubricPass": 1, + "qualityCoverage": 0.6666666666666666, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "coverageOfAll": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "slots.day": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.3333333333333333, + "coverageOfAll": 0.3333333333333333, + "precision": 1, + "intentLikeCoverage": 0.3333333333333333, + "rubricPass": 1, + "qualityCoverage": 0.3333333333333333, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 3, + "rate": 0.3333333333333333, + "low": 0.06149194472039621, + "high": 0.7923403991979522 + }, + "coverageOfAll": { + "k": 1, + "n": 3, + "rate": 0.3333333333333333, + "low": 0.06149194472039621, + "high": 0.7923403991979522 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 118, + "inScope": 67, + "aiIntentLike": 48, + "aiOther": 3, + "correct": 59, + "wrong": 0, + "missed": 8, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 59, + "relabelled": 0, + "coverage": 0.8805970149253731, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5130434782608696, + "rubricPass": 1, + "qualityCoverage": 0.8805970149253731, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 59, + "n": 67, + "rate": 0.8805970149253731, + "low": 0.7816797416192777, + "high": 0.9382376746649551 + }, + "coverageOfAll": { + "k": 59, + "n": 118, + "rate": 0.5, + "low": 0.41121889415183216, + "high": 0.5887811058481678 + }, + "precision": { + "k": 59, + "n": 59, + "rate": 1, + "low": 0.9388706294732116, + "high": 1 + }, + "rubricPass": { + "k": 59, + "n": 59, + "rate": 1, + "low": 0.9388706294732116, + "high": 1 + }, + "execution": { + "k": 59, + "n": 59, + "rate": 1, + "low": 0.9388706294732116, + "high": 1 + } + } + }, + "verbless": { + "conversations": 29, + "inScope": 29, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 25, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 25, + "relabelled": 0, + "coverage": 0.8620689655172413, + "coverageOfAll": 0.8620689655172413, + "precision": 1, + "intentLikeCoverage": 0.8620689655172413, + "rubricPass": 1, + "qualityCoverage": 0.8620689655172413, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 25, + "n": 29, + "rate": 0.8620689655172413, + "low": 0.6944100627964798, + "high": 0.9450256003588804 + }, + "coverageOfAll": { + "k": 25, + "n": 29, + "rate": 0.8620689655172413, + "low": 0.6944100627964798, + "high": 0.9450256003588804 + }, + "precision": { + "k": 25, + "n": 25, + "rate": 1, + "low": 0.8668077490609515, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 25, + "n": 25, + "rate": 1, + "low": 0.8668077490609515, + "high": 0.9999999999999999 + }, + "execution": { + "k": 25, + "n": 25, + "rate": 1, + "low": 0.8668077490609515, + "high": 0.9999999999999999 + } + } + }, + "terse": { + "conversations": 58, + "inScope": 52, + "aiIntentLike": 5, + "aiOther": 1, + "correct": 46, + "wrong": 0, + "missed": 6, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 46, + "relabelled": 0, + "coverage": 0.8846153846153846, + "coverageOfAll": 0.7931034482758621, + "precision": 1, + "intentLikeCoverage": 0.8070175438596491, + "rubricPass": 1, + "qualityCoverage": 0.8846153846153846, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 46, + "n": 52, + "rate": 0.8846153846153846, + "low": 0.7702833565509788, + "high": 0.946030307969127 + }, + "coverageOfAll": { + "k": 46, + "n": 58, + "rate": 0.7931034482758621, + "low": 0.6723078207851659, + "high": 0.8774851603050945 + }, + "precision": { + "k": 46, + "n": 46, + "rate": 1, + "low": 0.9229264369144198, + "high": 1 + }, + "rubricPass": { + "k": 46, + "n": 46, + "rate": 1, + "low": 0.9229264369144198, + "high": 1 + }, + "execution": { + "k": 46, + "n": 46, + "rate": 1, + "low": 0.9229264369144198, + "high": 1 + } + } + }, + "typo": { + "conversations": 7, + "inScope": 6, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 0.8333333333333334, + "coverageOfAll": 0.7142857142857143, + "precision": 1, + "intentLikeCoverage": 0.7142857142857143, + "rubricPass": 1, + "qualityCoverage": 0.8333333333333334, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 6, + "rate": 0.8333333333333334, + "low": 0.43649717781352987, + "high": 0.9699466302516935 + }, + "coverageOfAll": { + "k": 5, + "n": 7, + "rate": 0.7142857142857143, + "low": 0.35893445183261935, + "high": 0.9177810759959433 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "slang": { + "conversations": 27, + "inScope": 25, + "aiIntentLike": 1, + "aiOther": 1, + "correct": 15, + "wrong": 0, + "missed": 10, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 15, + "relabelled": 0, + "coverage": 0.6, + "coverageOfAll": 0.5555555555555556, + "precision": 1, + "intentLikeCoverage": 0.5769230769230769, + "rubricPass": 1, + "qualityCoverage": 0.6, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 15, + "n": 25, + "rate": 0.6, + "low": 0.40739457358967, + "high": 0.7659669762225203 + }, + "coverageOfAll": { + "k": 15, + "n": 27, + "rate": 0.5555555555555556, + "low": 0.3731302451322448, + "high": 0.72414141822977 + }, + "precision": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "rubricPass": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "execution": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + } + } + }, + "no_punct": { + "conversations": 14, + "inScope": 13, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 11, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 11, + "relabelled": 0, + "coverage": 0.8461538461538461, + "coverageOfAll": 0.7857142857142857, + "precision": 1, + "intentLikeCoverage": 0.7857142857142857, + "rubricPass": 1, + "qualityCoverage": 0.8461538461538461, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 11, + "n": 13, + "rate": 0.8461538461538461, + "low": 0.5776536898051745, + "high": 0.9567418216419186 + }, + "coverageOfAll": { + "k": 11, + "n": 14, + "rate": 0.7857142857142857, + "low": 0.524107694133997, + "high": 0.9242861328730683 + }, + "precision": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "rubricPass": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "execution": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + } + } + }, + "caps": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "polite": { + "conversations": 8, + "inScope": 8, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 0.625, + "coverageOfAll": 0.625, + "precision": 1, + "intentLikeCoverage": 0.625, + "rubricPass": 1, + "qualityCoverage": 0.625, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 8, + "rate": 0.625, + "low": 0.30574239460262737, + "high": 0.8631557141764026 + }, + "coverageOfAll": { + "k": 5, + "n": 8, + "rate": 0.625, + "low": 0.30574239460262737, + "high": 0.8631557141764026 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "trailing_punct": { + "conversations": 14, + "inScope": 14, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 10, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 10, + "relabelled": 0, + "coverage": 0.7142857142857143, + "coverageOfAll": 0.7142857142857143, + "precision": 1, + "intentLikeCoverage": 0.7142857142857143, + "rubricPass": 1, + "qualityCoverage": 0.7142857142857143, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 10, + "n": 14, + "rate": 0.7142857142857143, + "low": 0.4535091567010681, + "high": 0.8827862135542308 + }, + "coverageOfAll": { + "k": 10, + "n": 14, + "rate": 0.7142857142857143, + "low": 0.4535091567010681, + "high": 0.8827862135542308 + }, + "precision": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "execution": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + } + } + }, + "long": { + "conversations": 7, + "inScope": 6, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.42857142857142855, + "precision": 1, + "intentLikeCoverage": 0.42857142857142855, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 6, + "rate": 0.5, + "low": 0.1876163064826506, + "high": 0.8123836935173494 + }, + "coverageOfAll": { + "k": 3, + "n": 7, + "rate": 0.42857142857142855, + "low": 0.15821985525146975, + "high": 0.7495416354723428 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "mixed_lang": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + } + }, + "byRule": { + "basis.event.create": 48, + "basis.calendar.day": 18, + "basis.event.delete": 15, + "basis.event.reschedule": 14, + "basis.event.create_invite": 10, + "basis.calendar.when": 5, + "basis.calendar.period": 5, + "basis.reminder.set": 4, + "basis.invite.send": 3, + "basis.reminder.after": 3, + "basis.calendar.count": 3, + "basis.event.snooze": 3, + "basis.calendar.upcoming": 3, + "basis.calendar.search": 3, + "basis.invite.status": 2, + "basis.contacts.find": 1, + "basis.contacts.list": 1, + "basis.calc.evaluate": 1, + "basis.reminder.list": 1, + "basis.calendar.image": 1, + "basis.event.rename": 1, + "basis.slots.day": 1, + "basis.reminder.clear": 1, + "basis.event.show": 1, + "basis.slots.check_time": 1, + "basis.event.set_detail": 1, + "basis.holidays.upcoming": 1, + "basis.event.hide": 1, + "basis.calendar.next": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 32 + }, + "rubricFindings": {} + }, + "all": { + "overall": { + "conversations": 755, + "inScope": 543, + "aiIntentLike": 185, + "aiOther": 27, + "correct": 491, + "wrong": 0, + "missed": 52, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 489, + "relabelled": 1, + "coverage": 0.9042357274401474, + "coverageOfAll": 0.6503311258278146, + "precision": 1, + "intentLikeCoverage": 0.6744505494505495, + "rubricPass": 0.9959266802443992, + "qualityCoverage": 0.9005524861878453, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 491, + "n": 543, + "rate": 0.9042357274401474, + "low": 0.8765693128318065, + "high": 0.926222781510743 + }, + "coverageOfAll": { + "k": 491, + "n": 755, + "rate": 0.6503311258278146, + "low": 0.6156327894728296, + "high": 0.6835074292927316 + }, + "precision": { + "k": 491, + "n": 491, + "rate": 1, + "low": 0.9922369907528583, + "high": 1 + }, + "rubricPass": { + "k": 489, + "n": 491, + "rate": 0.9959266802443992, + "low": 0.9852713565290308, + "high": 0.9988822371504846 + }, + "execution": { + "k": 491, + "n": 491, + "rate": 1, + "low": 0.9922369907528583, + "high": 1 + } + } + }, + "byFamily": { + "ai:advice_or_analysis": { + "conversations": 15, + "inScope": 0, + "aiIntentLike": 15, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 15, + "rate": 0, + "low": 0, + "high": 0.20388330103584862 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:ambiguous_day": { + "conversations": 7, + "inScope": 0, + "aiIntentLike": 7, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 7, + "rate": 0, + "low": 5.551115123125783e-17, + "high": 0.35433043506668743 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 40, + "inScope": 0, + "aiIntentLike": 40, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 40, + "rate": 0, + "low": 0, + "high": 0.08762160119728664 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 13, + "inScope": 0, + "aiIntentLike": 13, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 13, + "rate": 0, + "low": 0, + "high": 0.22809537235419838 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 27, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 27, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 27, + "rate": 0, + "low": 0, + "high": 0.12455502974186705 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 29, + "inScope": 0, + "aiIntentLike": 29, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 29, + "rate": 0, + "low": 0, + "high": 0.11696979849974075 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 20, + "inScope": 0, + "aiIntentLike": 20, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 20, + "rate": 0, + "low": 0, + "high": 0.16112515805281938 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:negation_or_quote": { + "conversations": 20, + "inScope": 0, + "aiIntentLike": 20, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 20, + "rate": 0, + "low": 0, + "high": 0.16112515805281938 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 12, + "inScope": 0, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 12, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.24249400665524085 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 11, + "inScope": 0, + "aiIntentLike": 11, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 11, + "rate": 0, + "low": 0, + "high": 0.2588329669680317 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 17, + "inScope": 0, + "aiIntentLike": 17, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 17, + "rate": 0, + "low": 0, + "high": 0.1843181350088518 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "calc.evaluate": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "calendar.count": { + "conversations": 9, + "inScope": 9, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 8, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 8, + "relabelled": 0, + "coverage": 0.8888888888888888, + "coverageOfAll": 0.8888888888888888, + "precision": 1, + "intentLikeCoverage": 0.8888888888888888, + "rubricPass": 1, + "qualityCoverage": 0.8888888888888888, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 8, + "n": 9, + "rate": 0.8888888888888888, + "low": 0.5650002944233439, + "high": 0.9801091123614551 + }, + "coverageOfAll": { + "k": 8, + "n": 9, + "rate": 0.8888888888888888, + "low": 0.5650002944233439, + "high": 0.9801091123614551 + }, + "precision": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "rubricPass": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "execution": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + } + } + }, + "calendar.day": { + "conversations": 71, + "inScope": 71, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 64, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 64, + "relabelled": 0, + "coverage": 0.9014084507042254, + "coverageOfAll": 0.9014084507042254, + "precision": 1, + "intentLikeCoverage": 0.9014084507042254, + "rubricPass": 1, + "qualityCoverage": 0.9014084507042254, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 64, + "n": 71, + "rate": 0.9014084507042254, + "low": 0.8101927775940715, + "high": 0.9514171762760307 + }, + "coverageOfAll": { + "k": 64, + "n": 71, + "rate": 0.9014084507042254, + "low": 0.8101927775940715, + "high": 0.9514171762760307 + }, + "precision": { + "k": 64, + "n": 64, + "rate": 1, + "low": 0.9433759402071946, + "high": 1 + }, + "rubricPass": { + "k": 64, + "n": 64, + "rate": 1, + "low": 0.9433759402071946, + "high": 1 + }, + "execution": { + "k": 64, + "n": 64, + "rate": 1, + "low": 0.9433759402071946, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 6, + "inScope": 6, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 0.8333333333333334, + "coverageOfAll": 0.8333333333333334, + "precision": 1, + "intentLikeCoverage": 0.8333333333333334, + "rubricPass": 0.6, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 6, + "rate": 0.8333333333333334, + "low": 0.43649717781352987, + "high": 0.9699466302516935 + }, + "coverageOfAll": { + "k": 5, + "n": 6, + "rate": 0.8333333333333334, + "low": 0.43649717781352987, + "high": 0.9699466302516935 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 5, + "rate": 0.6, + "low": 0.2307242812760128, + "high": 0.882379225767352 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 21, + "inScope": 21, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 19, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 19, + "relabelled": 0, + "coverage": 0.9047619047619048, + "coverageOfAll": 0.9047619047619048, + "precision": 1, + "intentLikeCoverage": 0.9047619047619048, + "rubricPass": 1, + "qualityCoverage": 0.9047619047619048, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 19, + "n": 21, + "rate": 0.9047619047619048, + "low": 0.7108586092608176, + "high": 0.9734812325619255 + }, + "coverageOfAll": { + "k": 19, + "n": 21, + "rate": 0.9047619047619048, + "low": 0.7108586092608176, + "high": 0.9734812325619255 + }, + "precision": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "rubricPass": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "execution": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + } + } + }, + "calendar.search": { + "conversations": 10, + "inScope": 10, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 9, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 9, + "relabelled": 0, + "coverage": 0.9, + "coverageOfAll": 0.9, + "precision": 1, + "intentLikeCoverage": 0.9, + "rubricPass": 1, + "qualityCoverage": 0.9, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 9, + "n": 10, + "rate": 0.9, + "low": 0.5958499732047615, + "high": 0.9821237869049271 + }, + "coverageOfAll": { + "k": 9, + "n": 10, + "rate": 0.9, + "low": 0.5958499732047615, + "high": 0.9821237869049271 + }, + "precision": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "rubricPass": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "execution": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + } + } + }, + "calendar.upcoming": { + "conversations": 12, + "inScope": 12, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 12, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 12, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 12, + "n": 12, + "rate": 1, + "low": 0.7575059933447592, + "high": 1 + }, + "coverageOfAll": { + "k": 12, + "n": 12, + "rate": 1, + "low": 0.7575059933447592, + "high": 1 + }, + "precision": { + "k": 12, + "n": 12, + "rate": 1, + "low": 0.7575059933447592, + "high": 1 + }, + "rubricPass": { + "k": 12, + "n": 12, + "rate": 1, + "low": 0.7575059933447592, + "high": 1 + }, + "execution": { + "k": 12, + "n": 12, + "rate": 1, + "low": 0.7575059933447592, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 15, + "inScope": 15, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 15, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 15, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "coverageOfAll": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "precision": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "rubricPass": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "execution": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + } + } + }, + "contacts.find": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 0.75, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.75, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "coverageOfAll": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "event.create": { + "conversations": 125, + "inScope": 125, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 114, + "wrong": 0, + "missed": 11, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 114, + "relabelled": 0, + "coverage": 0.912, + "coverageOfAll": 0.912, + "precision": 1, + "intentLikeCoverage": 0.912, + "rubricPass": 1, + "qualityCoverage": 0.912, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 114, + "n": 125, + "rate": 0.912, + "low": 0.8492803613696875, + "high": 0.9501517542146516 + }, + "coverageOfAll": { + "k": 114, + "n": 125, + "rate": 0.912, + "low": 0.8492803613696875, + "high": 0.9501517542146516 + }, + "precision": { + "k": 114, + "n": 114, + "rate": 1, + "low": 0.9674014658411585, + "high": 1 + }, + "rubricPass": { + "k": 114, + "n": 114, + "rate": 1, + "low": 0.9674014658411585, + "high": 1 + }, + "execution": { + "k": 114, + "n": 114, + "rate": 1, + "low": 0.9674014658411585, + "high": 1 + } + } + }, + "event.create_invite": { + "conversations": 43, + "inScope": 43, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 29, + "wrong": 0, + "missed": 14, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 29, + "relabelled": 0, + "coverage": 0.6744186046511628, + "coverageOfAll": 0.6744186046511628, + "precision": 1, + "intentLikeCoverage": 0.6744186046511628, + "rubricPass": 1, + "qualityCoverage": 0.6744186046511628, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 29, + "n": 43, + "rate": 0.6744186046511628, + "low": 0.5251621096752948, + "high": 0.7950670287343129 + }, + "coverageOfAll": { + "k": 29, + "n": 43, + "rate": 0.6744186046511628, + "low": 0.5251621096752948, + "high": 0.7950670287343129 + }, + "precision": { + "k": 29, + "n": 29, + "rate": 1, + "low": 0.8830302015002593, + "high": 1 + }, + "rubricPass": { + "k": 29, + "n": 29, + "rate": 1, + "low": 0.8830302015002593, + "high": 1 + }, + "execution": { + "k": 29, + "n": 29, + "rate": 1, + "low": 0.8830302015002593, + "high": 1 + } + } + }, + "event.create_range": { + "conversations": 16, + "inScope": 16, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 14, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 14, + "relabelled": 0, + "coverage": 0.875, + "coverageOfAll": 0.875, + "precision": 1, + "intentLikeCoverage": 0.875, + "rubricPass": 1, + "qualityCoverage": 0.875, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 14, + "n": 16, + "rate": 0.875, + "low": 0.639771727342413, + "high": 0.9650225122567595 + }, + "coverageOfAll": { + "k": 14, + "n": 16, + "rate": 0.875, + "low": 0.639771727342413, + "high": 0.9650225122567595 + }, + "precision": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "rubricPass": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "execution": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 47, + "inScope": 47, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 47, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 47, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 47, + "n": 47, + "rate": 1, + "low": 0.9244423958360037, + "high": 1 + }, + "coverageOfAll": { + "k": 47, + "n": 47, + "rate": 1, + "low": 0.9244423958360037, + "high": 1 + }, + "precision": { + "k": 47, + "n": 47, + "rate": 1, + "low": 0.9244423958360037, + "high": 1 + }, + "rubricPass": { + "k": 47, + "n": 47, + "rate": 1, + "low": 0.9244423958360037, + "high": 1 + }, + "execution": { + "k": 47, + "n": 47, + "rate": 1, + "low": 0.9244423958360037, + "high": 1 + } + } + }, + "event.hide": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "event.rename": { + "conversations": 6, + "inScope": 6, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 6, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 6, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "coverageOfAll": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "precision": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "rubricPass": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "execution": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 43, + "inScope": 42, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 38, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 38, + "relabelled": 1, + "coverage": 0.9047619047619048, + "coverageOfAll": 0.8837209302325582, + "precision": 1, + "intentLikeCoverage": 0.8837209302325582, + "rubricPass": 1, + "qualityCoverage": 0.9047619047619048, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 38, + "n": 42, + "rate": 0.9047619047619048, + "low": 0.7793489419142897, + "high": 0.9623377511944261 + }, + "coverageOfAll": { + "k": 38, + "n": 43, + "rate": 0.8837209302325582, + "low": 0.7552083818991787, + "high": 0.9492957226019584 + }, + "precision": { + "k": 38, + "n": 38, + "rate": 1, + "low": 0.908190131774416, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 38, + "n": 38, + "rate": 1, + "low": 0.908190131774416, + "high": 0.9999999999999999 + }, + "execution": { + "k": 38, + "n": 38, + "rate": 1, + "low": 0.908190131774416, + "high": 0.9999999999999999 + } + } + }, + "event.set_detail": { + "conversations": 6, + "inScope": 6, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 0.8333333333333334, + "coverageOfAll": 0.8333333333333334, + "precision": 1, + "intentLikeCoverage": 0.8333333333333334, + "rubricPass": 1, + "qualityCoverage": 0.8333333333333334, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 6, + "rate": 0.8333333333333334, + "low": 0.43649717781352987, + "high": 0.9699466302516935 + }, + "coverageOfAll": { + "k": 5, + "n": 6, + "rate": 0.8333333333333334, + "low": 0.43649717781352987, + "high": 0.9699466302516935 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "event.show": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "event.snooze": { + "conversations": 9, + "inScope": 9, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 8, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 8, + "relabelled": 0, + "coverage": 0.8888888888888888, + "coverageOfAll": 0.8888888888888888, + "precision": 1, + "intentLikeCoverage": 0.8888888888888888, + "rubricPass": 1, + "qualityCoverage": 0.8888888888888888, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 8, + "n": 9, + "rate": 0.8888888888888888, + "low": 0.5650002944233439, + "high": 0.9801091123614551 + }, + "coverageOfAll": { + "k": 8, + "n": 9, + "rate": 0.8888888888888888, + "low": 0.5650002944233439, + "high": 0.9801091123614551 + }, + "precision": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "rubricPass": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "execution": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + } + } + }, + "google.status": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 0.6666666666666666, + "coverageOfAll": 0.6666666666666666, + "precision": 1, + "intentLikeCoverage": 0.6666666666666666, + "rubricPass": 1, + "qualityCoverage": 0.6666666666666666, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "coverageOfAll": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "holidays.upcoming": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "invite.send": { + "conversations": 11, + "inScope": 11, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 11, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 11, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "coverageOfAll": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "precision": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "rubricPass": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "execution": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 6, + "inScope": 6, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 6, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 6, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "coverageOfAll": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "precision": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "rubricPass": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "execution": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + } + } + }, + "reminder.after": { + "conversations": 14, + "inScope": 14, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 13, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 13, + "relabelled": 0, + "coverage": 0.9285714285714286, + "coverageOfAll": 0.9285714285714286, + "precision": 1, + "intentLikeCoverage": 0.9285714285714286, + "rubricPass": 1, + "qualityCoverage": 0.9285714285714286, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 13, + "n": 14, + "rate": 0.9285714285714286, + "low": 0.6853129557584889, + "high": 0.9872777847521093 + }, + "coverageOfAll": { + "k": 13, + "n": 14, + "rate": 0.9285714285714286, + "low": 0.6853129557584889, + "high": 0.9872777847521093 + }, + "precision": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + }, + "execution": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + } + } + }, + "reminder.clear": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "reminder.list": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "reminder.set": { + "conversations": 11, + "inScope": 11, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 11, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 11, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "coverageOfAll": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "precision": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "rubricPass": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "execution": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + } + } + }, + "settings.view": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "slots.check_time": { + "conversations": 12, + "inScope": 12, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 10, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 10, + "relabelled": 0, + "coverage": 0.8333333333333334, + "coverageOfAll": 0.8333333333333334, + "precision": 1, + "intentLikeCoverage": 0.8333333333333334, + "rubricPass": 1, + "qualityCoverage": 0.8333333333333334, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 10, + "n": 12, + "rate": 0.8333333333333334, + "low": 0.5519691377470266, + "high": 0.9530348578161463 + }, + "coverageOfAll": { + "k": 10, + "n": 12, + "rate": 0.8333333333333334, + "low": 0.5519691377470266, + "high": 0.9530348578161463 + }, + "precision": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "execution": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + } + } + }, + "slots.day": { + "conversations": 11, + "inScope": 11, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 9, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 9, + "relabelled": 0, + "coverage": 0.8181818181818182, + "coverageOfAll": 0.8181818181818182, + "precision": 1, + "intentLikeCoverage": 0.8181818181818182, + "rubricPass": 1, + "qualityCoverage": 0.8181818181818182, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 9, + "n": 11, + "rate": 0.8181818181818182, + "low": 0.523019438039156, + "high": 0.9486323102539149 + }, + "coverageOfAll": { + "k": 9, + "n": 11, + "rate": 0.8181818181818182, + "low": 0.523019438039156, + "high": 0.9486323102539149 + }, + "precision": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "rubricPass": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "execution": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + } + } + }, + "time.now": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + } + }, + "byStyle": { + "typo": { + "conversations": 36, + "inScope": 34, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 30, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 30, + "relabelled": 0, + "coverage": 0.8823529411764706, + "coverageOfAll": 0.8333333333333334, + "precision": 1, + "intentLikeCoverage": 0.8333333333333334, + "rubricPass": 1, + "qualityCoverage": 0.8823529411764706, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 30, + "n": 34, + "rate": 0.8823529411764706, + "low": 0.7337915722962817, + "high": 0.9532855333511766 + }, + "coverageOfAll": { + "k": 30, + "n": 36, + "rate": 0.8333333333333334, + "low": 0.6810920568872054, + "high": 0.9212955240498188 + }, + "precision": { + "k": 30, + "n": 30, + "rate": 1, + "low": 0.8864866068260312, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 30, + "n": 30, + "rate": 1, + "low": 0.8864866068260312, + "high": 0.9999999999999999 + }, + "execution": { + "k": 30, + "n": 30, + "rate": 1, + "low": 0.8864866068260312, + "high": 0.9999999999999999 + } + } + }, + "plain": { + "conversations": 249, + "inScope": 136, + "aiIntentLike": 97, + "aiOther": 16, + "correct": 121, + "wrong": 0, + "missed": 15, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 121, + "relabelled": 0, + "coverage": 0.8897058823529411, + "coverageOfAll": 0.4859437751004016, + "precision": 1, + "intentLikeCoverage": 0.51931330472103, + "rubricPass": 1, + "qualityCoverage": 0.8897058823529411, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 121, + "n": 136, + "rate": 0.8897058823529411, + "low": 0.825989068198359, + "high": 0.9320121775975252 + }, + "coverageOfAll": { + "k": 121, + "n": 249, + "rate": 0.4859437751004016, + "low": 0.42455106847387564, + "high": 0.5477635984614858 + }, + "precision": { + "k": 121, + "n": 121, + "rate": 1, + "low": 0.9692293020525218, + "high": 1 + }, + "rubricPass": { + "k": 121, + "n": 121, + "rate": 1, + "low": 0.9692293020525218, + "high": 1 + }, + "execution": { + "k": 121, + "n": 121, + "rate": 1, + "low": 0.9692293020525218, + "high": 1 + } + } + }, + "no_punct": { + "conversations": 169, + "inScope": 137, + "aiIntentLike": 27, + "aiOther": 5, + "correct": 132, + "wrong": 0, + "missed": 5, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 130, + "relabelled": 0, + "coverage": 0.9635036496350365, + "coverageOfAll": 0.7810650887573964, + "precision": 1, + "intentLikeCoverage": 0.8048780487804879, + "rubricPass": 0.9848484848484849, + "qualityCoverage": 0.948905109489051, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 132, + "n": 137, + "rate": 0.9635036496350365, + "low": 0.9174111125832916, + "high": 0.9843120097374185 + }, + "coverageOfAll": { + "k": 132, + "n": 169, + "rate": 0.7810650887573964, + "low": 0.7128537569078812, + "high": 0.8367828907355126 + }, + "precision": { + "k": 132, + "n": 132, + "rate": 1, + "low": 0.9717210132013917, + "high": 1 + }, + "rubricPass": { + "k": 130, + "n": 132, + "rate": 0.9848484848484849, + "low": 0.9464398960595944, + "high": 0.9958350258326641 + }, + "execution": { + "k": 132, + "n": 132, + "rate": 1, + "low": 0.9717210132013917, + "high": 1 + } + } + }, + "caps": { + "conversations": 5, + "inScope": 5, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "coverageOfAll": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "trailing_punct": { + "conversations": 71, + "inScope": 57, + "aiIntentLike": 10, + "aiOther": 4, + "correct": 52, + "wrong": 0, + "missed": 5, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 52, + "relabelled": 1, + "coverage": 0.9122807017543859, + "coverageOfAll": 0.7323943661971831, + "precision": 1, + "intentLikeCoverage": 0.7761194029850746, + "rubricPass": 1, + "qualityCoverage": 0.9122807017543859, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 52, + "n": 57, + "rate": 0.9122807017543859, + "low": 0.8105512290044132, + "high": 0.9619483282220778 + }, + "coverageOfAll": { + "k": 52, + "n": 71, + "rate": 0.7323943661971831, + "low": 0.6194601134214525, + "high": 0.8214719651349224 + }, + "precision": { + "k": 52, + "n": 52, + "rate": 1, + "low": 0.9312077638761376, + "high": 1 + }, + "rubricPass": { + "k": 52, + "n": 52, + "rate": 1, + "low": 0.9312077638761376, + "high": 1 + }, + "execution": { + "k": 52, + "n": 52, + "rate": 1, + "low": 0.9312077638761376, + "high": 1 + } + } + }, + "terse": { + "conversations": 206, + "inScope": 156, + "aiIntentLike": 46, + "aiOther": 4, + "correct": 149, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 148, + "relabelled": 0, + "coverage": 0.9551282051282052, + "coverageOfAll": 0.7233009708737864, + "precision": 1, + "intentLikeCoverage": 0.7376237623762376, + "rubricPass": 0.9932885906040269, + "qualityCoverage": 0.9487179487179487, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 149, + "n": 156, + "rate": 0.9551282051282052, + "low": 0.9102836178517977, + "high": 0.9780966625064529 + }, + "coverageOfAll": { + "k": 149, + "n": 206, + "rate": 0.7233009708737864, + "low": 0.6585459173659068, + "high": 0.7798803141187512 + }, + "precision": { + "k": 149, + "n": 149, + "rate": 1, + "low": 0.9748663821299904, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 148, + "n": 149, + "rate": 0.9932885906040269, + "low": 0.9629666399496506, + "high": 0.9988142873866486 + }, + "execution": { + "k": 149, + "n": 149, + "rate": 1, + "low": 0.9748663821299904, + "high": 0.9999999999999999 + } + } + }, + "verbless": { + "conversations": 82, + "inScope": 79, + "aiIntentLike": 3, + "aiOther": 0, + "correct": 73, + "wrong": 0, + "missed": 6, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 72, + "relabelled": 0, + "coverage": 0.9240506329113924, + "coverageOfAll": 0.8902439024390244, + "precision": 1, + "intentLikeCoverage": 0.8902439024390244, + "rubricPass": 0.9863013698630136, + "qualityCoverage": 0.9113924050632911, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 73, + "n": 79, + "rate": 0.9240506329113924, + "low": 0.8440458309717235, + "high": 0.9647279515496333 + }, + "coverageOfAll": { + "k": 73, + "n": 82, + "rate": 0.8902439024390244, + "low": 0.804387633588628, + "high": 0.9411728552386109 + }, + "precision": { + "k": 73, + "n": 73, + "rate": 1, + "low": 0.9500079920442688, + "high": 0.9999999999999998 + }, + "rubricPass": { + "k": 72, + "n": 73, + "rate": 0.9863013698630136, + "low": 0.9264026272903616, + "high": 0.997577748533516 + }, + "execution": { + "k": 73, + "n": 73, + "rate": 1, + "low": 0.9500079920442688, + "high": 0.9999999999999998 + } + } + }, + "slang": { + "conversations": 53, + "inScope": 50, + "aiIntentLike": 2, + "aiOther": 1, + "correct": 38, + "wrong": 0, + "missed": 12, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 37, + "relabelled": 0, + "coverage": 0.76, + "coverageOfAll": 0.7169811320754716, + "precision": 1, + "intentLikeCoverage": 0.7307692307692307, + "rubricPass": 0.9736842105263158, + "qualityCoverage": 0.74, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 38, + "n": 50, + "rate": 0.76, + "low": 0.6258731624205708, + "high": 0.8570260860300828 + }, + "coverageOfAll": { + "k": 38, + "n": 53, + "rate": 0.7169811320754716, + "low": 0.5842971358358297, + "high": 0.8203370955366142 + }, + "precision": { + "k": 38, + "n": 38, + "rate": 1, + "low": 0.908190131774416, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 37, + "n": 38, + "rate": 0.9736842105263158, + "low": 0.8650512388404787, + "high": 0.9953394123142312 + }, + "execution": { + "k": 38, + "n": 38, + "rate": 1, + "low": 0.908190131774416, + "high": 0.9999999999999999 + } + } + }, + "polite": { + "conversations": 32, + "inScope": 31, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 24, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 24, + "relabelled": 0, + "coverage": 0.7741935483870968, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.7741935483870968, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 24, + "n": 31, + "rate": 0.7741935483870968, + "low": 0.6018758161665794, + "high": 0.886048644387375 + }, + "coverageOfAll": { + "k": 24, + "n": 32, + "rate": 0.75, + "low": 0.5789344100575552, + "high": 0.8674759908149096 + }, + "precision": { + "k": 24, + "n": 24, + "rate": 1, + "low": 0.8620237953250198, + "high": 1 + }, + "rubricPass": { + "k": 24, + "n": 24, + "rate": 1, + "low": 0.8620237953250198, + "high": 1 + }, + "execution": { + "k": 24, + "n": 24, + "rate": 1, + "low": 0.8620237953250198, + "high": 1 + } + } + }, + "long": { + "conversations": 8, + "inScope": 7, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 0.42857142857142855, + "coverageOfAll": 0.375, + "precision": 1, + "intentLikeCoverage": 0.375, + "rubricPass": 1, + "qualityCoverage": 0.42857142857142855, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 7, + "rate": 0.42857142857142855, + "low": 0.15821985525146975, + "high": 0.7495416354723428 + }, + "coverageOfAll": { + "k": 3, + "n": 8, + "rate": 0.375, + "low": 0.13684428582359742, + "high": 0.6942576053973727 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "mixed_lang": { + "conversations": 14, + "inScope": 13, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 11, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 11, + "relabelled": 0, + "coverage": 0.8461538461538461, + "coverageOfAll": 0.7857142857142857, + "precision": 1, + "intentLikeCoverage": 0.7857142857142857, + "rubricPass": 1, + "qualityCoverage": 0.8461538461538461, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 11, + "n": 13, + "rate": 0.8461538461538461, + "low": 0.5776536898051745, + "high": 0.9567418216419186 + }, + "coverageOfAll": { + "k": 11, + "n": 14, + "rate": 0.7857142857142857, + "low": 0.524107694133997, + "high": 0.9242861328730683 + }, + "precision": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "rubricPass": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "execution": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + } + } + } + }, + "byRule": { + "basis.event.create": 129, + "basis.calendar.day": 64, + "basis.event.delete": 47, + "basis.event.reschedule": 38, + "basis.event.create_invite": 29, + "basis.calendar.period": 19, + "basis.calendar.when": 15, + "basis.reminder.after": 13, + "basis.invite.send": 11, + "basis.reminder.set": 11, + "basis.calendar.upcoming": 10, + "basis.calendar.search": 9, + "basis.slots.check_time": 9, + "basis.slots.day": 9, + "basis.event.snooze": 8, + "basis.calendar.count": 8, + "basis.event.rename": 6, + "basis.invite.status": 6, + "basis.event.set_detail": 5, + "basis.calendar.image": 5, + "basis.time.now": 4, + "basis.event.show": 4, + "basis.event.hide": 4, + "basis.holidays.upcoming": 4, + "basis.calc.evaluate": 4, + "basis.contacts.find": 3, + "basis.reminder.list": 3, + "basis.reminder.clear": 3, + "basis.contacts.list": 3, + "basis.settings.view": 2, + "basis.bot.info": 2, + "basis.google.status": 2, + "basis.calendar.next": 2 + }, + "failureClasses": { + "missed:fell_through_to_ai": 52 + }, + "rubricFindings": { + "RAW_ISO_DATE": 2 + } + }, + "matrix": { + "overall": { + "conversations": 3156, + "inScope": 2130, + "aiIntentLike": 900, + "aiOther": 126, + "correct": 1920, + "wrong": 0, + "missed": 210, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1878, + "relabelled": 54, + "coverage": 0.9014084507042254, + "coverageOfAll": 0.6083650190114068, + "precision": 1, + "intentLikeCoverage": 0.6336633663366337, + "rubricPass": 0.978125, + "qualityCoverage": 0.8816901408450705, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 1920, + "n": 2130, + "rate": 0.9014084507042254, + "low": 0.8880164354639921, + "high": 0.9133551907790662 + }, + "coverageOfAll": { + "k": 1920, + "n": 3156, + "rate": 0.6083650190114068, + "low": 0.5912136014275136, + "high": 0.6252529551747534 + }, + "precision": { + "k": 1920, + "n": 1920, + "rate": 1, + "low": 0.9980032352442133, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 1878, + "n": 1920, + "rate": 0.978125, + "low": 0.9705646020352869, + "high": 0.9837759916669921 + }, + "execution": { + "k": 1920, + "n": 1920, + "rate": 1, + "low": 0.9980032352442133, + "high": 0.9999999999999999 + } + } + }, + "byFamily": { + "ai:advice_or_analysis": { + "conversations": 84, + "inScope": 0, + "aiIntentLike": 84, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 84, + "rate": 0, + "low": 0, + "high": 0.04373172841465989 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:ambiguous_day": { + "conversations": 6, + "inScope": 0, + "aiIntentLike": 6, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 6, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.3903342879021653 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 126, + "inScope": 0, + "aiIntentLike": 126, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 126, + "rate": 0, + "low": 1.734723475976807e-18, + "high": 0.02958576448219845 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 126, + "inScope": 0, + "aiIntentLike": 126, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 126, + "rate": 0, + "low": 1.734723475976807e-18, + "high": 0.02958576448219845 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 126, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 126, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 126, + "rate": 0, + "low": 1.734723475976807e-18, + "high": 0.02958576448219845 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 210, + "inScope": 0, + "aiIntentLike": 210, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 210, + "rate": 0, + "low": 0, + "high": 0.017964050759283236 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 42, + "inScope": 0, + "aiIntentLike": 42, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:negation_or_quote": { + "conversations": 126, + "inScope": 0, + "aiIntentLike": 126, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 126, + "rate": 0, + "low": 1.734723475976807e-18, + "high": 0.02958576448219845 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 42, + "inScope": 0, + "aiIntentLike": 42, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 84, + "inScope": 0, + "aiIntentLike": 84, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 84, + "rate": 0, + "low": 0, + "high": 0.04373172841465989 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "calc.evaluate": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "calendar.day": { + "conversations": 420, + "inScope": 402, + "aiIntentLike": 18, + "aiOther": 0, + "correct": 360, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 360, + "relabelled": 18, + "coverage": 0.8955223880597015, + "coverageOfAll": 0.8571428571428571, + "precision": 1, + "intentLikeCoverage": 0.8571428571428571, + "rubricPass": 1, + "qualityCoverage": 0.8955223880597015, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 360, + "n": 402, + "rate": 0.8955223880597015, + "low": 0.861784942681361, + "high": 0.9217722644212688 + }, + "coverageOfAll": { + "k": 360, + "n": 420, + "rate": 0.8571428571428571, + "low": 0.8204352436514356, + "high": 0.8873765896570062 + }, + "precision": { + "k": 360, + "n": 360, + "rate": 1, + "low": 0.9894419431113063, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 360, + "n": 360, + "rate": 1, + "low": 0.9894419431113063, + "high": 0.9999999999999999 + }, + "execution": { + "k": 360, + "n": 360, + "rate": 1, + "low": 0.9894419431113063, + "high": 0.9999999999999999 + } + } + }, + "calendar.image": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 0, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "calendar.period": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "contacts.find": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "contacts.list": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "event.create": { + "conversations": 504, + "inScope": 486, + "aiIntentLike": 18, + "aiOther": 0, + "correct": 402, + "wrong": 0, + "missed": 84, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 402, + "relabelled": 18, + "coverage": 0.8271604938271605, + "coverageOfAll": 0.7976190476190477, + "precision": 1, + "intentLikeCoverage": 0.7976190476190477, + "rubricPass": 1, + "qualityCoverage": 0.8271604938271605, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 402, + "n": 486, + "rate": 0.8271604938271605, + "low": 0.7910127146172659, + "high": 0.8581769247521602 + }, + "coverageOfAll": { + "k": 402, + "n": 504, + "rate": 0.7976190476190477, + "low": 0.7603518109989148, + "high": 0.8303837321303483 + }, + "precision": { + "k": 402, + "n": 402, + "rate": 1, + "low": 0.9905345825637017, + "high": 1 + }, + "rubricPass": { + "k": 402, + "n": 402, + "rate": 1, + "low": 0.9905345825637017, + "high": 1 + }, + "execution": { + "k": 402, + "n": 402, + "rate": 1, + "low": 0.9905345825637017, + "high": 1 + } + } + }, + "event.create_invite": { + "conversations": 210, + "inScope": 204, + "aiIntentLike": 6, + "aiOther": 0, + "correct": 126, + "wrong": 0, + "missed": 78, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 126, + "relabelled": 6, + "coverage": 0.6176470588235294, + "coverageOfAll": 0.6, + "precision": 1, + "intentLikeCoverage": 0.6, + "rubricPass": 1, + "qualityCoverage": 0.6176470588235294, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 126, + "n": 204, + "rate": 0.6176470588235294, + "low": 0.5493698847799827, + "high": 0.6815753764415037 + }, + "coverageOfAll": { + "k": 126, + "n": 210, + "rate": 0.6, + "low": 0.5325179857434847, + "high": 0.6638892041046585 + }, + "precision": { + "k": 126, + "n": 126, + "rate": 1, + "low": 0.9704142355178014, + "high": 0.9999999999999998 + }, + "rubricPass": { + "k": 126, + "n": 126, + "rate": 1, + "low": 0.9704142355178014, + "high": 0.9999999999999998 + }, + "execution": { + "k": 126, + "n": 126, + "rate": 1, + "low": 0.9704142355178014, + "high": 0.9999999999999998 + } + } + }, + "event.delete": { + "conversations": 336, + "inScope": 336, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 336, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 336, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + }, + "precision": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + }, + "execution": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + } + } + }, + "event.rename": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "event.reschedule": { + "conversations": 294, + "inScope": 282, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 276, + "wrong": 0, + "missed": 6, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 276, + "relabelled": 12, + "coverage": 0.9787234042553191, + "coverageOfAll": 0.9387755102040817, + "precision": 1, + "intentLikeCoverage": 0.9387755102040817, + "rubricPass": 1, + "qualityCoverage": 0.9787234042553191, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 276, + "n": 282, + "rate": 0.9787234042553191, + "low": 0.9543664300048462, + "high": 0.9902131320850907 + }, + "coverageOfAll": { + "k": 276, + "n": 294, + "rate": 0.9387755102040817, + "low": 0.9053074648870244, + "high": 0.9609251980254973 + }, + "precision": { + "k": 276, + "n": 276, + "rate": 1, + "low": 0.9862727315785058, + "high": 0.9999999999999998 + }, + "rubricPass": { + "k": 276, + "n": 276, + "rate": 1, + "low": 0.9862727315785058, + "high": 0.9999999999999998 + }, + "execution": { + "k": 276, + "n": 276, + "rate": 1, + "low": 0.9862727315785058, + "high": 0.9999999999999998 + } + } + }, + "event.snooze": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "invite.status": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "reminder.set": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "settings.view": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + } + }, + "byStyle": { + "typo": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "trailing_punct": { + "conversations": 378, + "inScope": 282, + "aiIntentLike": 96, + "aiOther": 0, + "correct": 282, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 282, + "relabelled": 12, + "coverage": 1, + "coverageOfAll": 0.746031746031746, + "precision": 1, + "intentLikeCoverage": 0.746031746031746, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 282, + "n": 282, + "rate": 1, + "low": 0.9865608759606008, + "high": 1 + }, + "coverageOfAll": { + "k": 282, + "n": 378, + "rate": 0.746031746031746, + "low": 0.6998273601630959, + "high": 0.7872858008185729 + }, + "precision": { + "k": 282, + "n": 282, + "rate": 1, + "low": 0.9865608759606008, + "high": 1 + }, + "rubricPass": { + "k": 282, + "n": 282, + "rate": 1, + "low": 0.9865608759606008, + "high": 1 + }, + "execution": { + "k": 282, + "n": 282, + "rate": 1, + "low": 0.9865608759606008, + "high": 1 + } + } + }, + "plain": { + "conversations": 1056, + "inScope": 498, + "aiIntentLike": 474, + "aiOther": 84, + "correct": 336, + "wrong": 0, + "missed": 162, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 336, + "relabelled": 6, + "coverage": 0.6746987951807228, + "coverageOfAll": 0.3181818181818182, + "precision": 1, + "intentLikeCoverage": 0.345679012345679, + "rubricPass": 1, + "qualityCoverage": 0.6746987951807228, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 336, + "n": 498, + "rate": 0.6746987951807228, + "low": 0.6323511583604485, + "high": 0.7143718892109746 + }, + "coverageOfAll": { + "k": 336, + "n": 1056, + "rate": 0.3181818181818182, + "low": 0.2907916917908576, + "high": 0.34688996634385616 + }, + "precision": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + }, + "execution": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + } + } + }, + "terse": { + "conversations": 966, + "inScope": 738, + "aiIntentLike": 228, + "aiOther": 0, + "correct": 738, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 696, + "relabelled": 18, + "coverage": 1, + "coverageOfAll": 0.7639751552795031, + "precision": 1, + "intentLikeCoverage": 0.7639751552795031, + "rubricPass": 0.943089430894309, + "qualityCoverage": 0.943089430894309, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 738, + "n": 738, + "rate": 1, + "low": 0.9948217253497791, + "high": 1 + }, + "coverageOfAll": { + "k": 738, + "n": 966, + "rate": 0.7639751552795031, + "low": 0.7361842198929713, + "high": 0.7896749247760437 + }, + "precision": { + "k": 738, + "n": 738, + "rate": 1, + "low": 0.9948217253497791, + "high": 1 + }, + "rubricPass": { + "k": 696, + "n": 738, + "rate": 0.943089430894309, + "low": 0.923966696194483, + "high": 0.9576232880585733 + }, + "execution": { + "k": 738, + "n": 738, + "rate": 1, + "low": 0.9948217253497791, + "high": 1 + } + } + }, + "slang": { + "conversations": 168, + "inScope": 156, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 114, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 114, + "relabelled": 12, + "coverage": 0.7307692307692307, + "coverageOfAll": 0.6785714285714286, + "precision": 1, + "intentLikeCoverage": 0.6785714285714286, + "rubricPass": 1, + "qualityCoverage": 0.7307692307692307, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 114, + "n": 156, + "rate": 0.7307692307692307, + "low": 0.6562367115839256, + "high": 0.794209627752652 + }, + "coverageOfAll": { + "k": 114, + "n": 168, + "rate": 0.6785714285714286, + "low": 0.6046383896622172, + "high": 0.7445206573556873 + }, + "precision": { + "k": 114, + "n": 114, + "rate": 1, + "low": 0.9674014658411585, + "high": 1 + }, + "rubricPass": { + "k": 114, + "n": 114, + "rate": 1, + "low": 0.9674014658411585, + "high": 1 + }, + "execution": { + "k": 114, + "n": 114, + "rate": 1, + "low": 0.9674014658411585, + "high": 1 + } + } + }, + "mixed_lang": { + "conversations": 126, + "inScope": 120, + "aiIntentLike": 6, + "aiOther": 0, + "correct": 78, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 78, + "relabelled": 6, + "coverage": 0.65, + "coverageOfAll": 0.6190476190476191, + "precision": 1, + "intentLikeCoverage": 0.6190476190476191, + "rubricPass": 1, + "qualityCoverage": 0.65, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 78, + "n": 120, + "rate": 0.65, + "low": 0.561213245977243, + "high": 0.729481004097146 + }, + "coverageOfAll": { + "k": 78, + "n": 126, + "rate": 0.6190476190476191, + "low": 0.5319219933564636, + "high": 0.6991290151001559 + }, + "precision": { + "k": 78, + "n": 78, + "rate": 1, + "low": 0.9530621902878059, + "high": 1 + }, + "rubricPass": { + "k": 78, + "n": 78, + "rate": 1, + "low": 0.9530621902878059, + "high": 1 + }, + "execution": { + "k": 78, + "n": 78, + "rate": 1, + "low": 0.9530621902878059, + "high": 1 + } + } + }, + "polite": { + "conversations": 126, + "inScope": 120, + "aiIntentLike": 6, + "aiOther": 0, + "correct": 120, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 120, + "relabelled": 6, + "coverage": 1, + "coverageOfAll": 0.9523809523809523, + "precision": 1, + "intentLikeCoverage": 0.9523809523809523, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 120, + "n": 120, + "rate": 1, + "low": 0.9689808335812966, + "high": 1 + }, + "coverageOfAll": { + "k": 120, + "n": 126, + "rate": 0.9523809523809523, + "low": 0.8999983205231048, + "high": 0.9779955116120489 + }, + "precision": { + "k": 120, + "n": 120, + "rate": 1, + "low": 0.9689808335812966, + "high": 1 + }, + "rubricPass": { + "k": 120, + "n": 120, + "rate": 1, + "low": 0.9689808335812966, + "high": 1 + }, + "execution": { + "k": 120, + "n": 120, + "rate": 1, + "low": 0.9689808335812966, + "high": 1 + } + } + }, + "no_punct": { + "conversations": 714, + "inScope": 582, + "aiIntentLike": 90, + "aiOther": 42, + "correct": 576, + "wrong": 0, + "missed": 6, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 534, + "relabelled": 6, + "coverage": 0.9896907216494846, + "coverageOfAll": 0.8067226890756303, + "precision": 1, + "intentLikeCoverage": 0.8571428571428571, + "rubricPass": 0.9270833333333334, + "qualityCoverage": 0.9175257731958762, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 576, + "n": 582, + "rate": 0.9896907216494846, + "low": 0.9776926382546434, + "high": 0.9952668398052005 + }, + "coverageOfAll": { + "k": 576, + "n": 714, + "rate": 0.8067226890756303, + "low": 0.7761487623998028, + "high": 0.83401382241324 + }, + "precision": { + "k": 576, + "n": 576, + "rate": 1, + "low": 0.9933749842094647, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 534, + "n": 576, + "rate": 0.9270833333333334, + "low": 0.9029031240390288, + "high": 0.9456046749732223 + }, + "execution": { + "k": 576, + "n": 576, + "rate": 1, + "low": 0.9933749842094647, + "high": 0.9999999999999999 + } + } + }, + "verbless": { + "conversations": 420, + "inScope": 402, + "aiIntentLike": 18, + "aiOther": 0, + "correct": 402, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 360, + "relabelled": 18, + "coverage": 1, + "coverageOfAll": 0.9571428571428572, + "precision": 1, + "intentLikeCoverage": 0.9571428571428572, + "rubricPass": 0.8955223880597015, + "qualityCoverage": 0.8955223880597015, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 402, + "n": 402, + "rate": 1, + "low": 0.9905345825637017, + "high": 1 + }, + "coverageOfAll": { + "k": 402, + "n": 420, + "rate": 0.9571428571428572, + "low": 0.9332776936660707, + "high": 0.9727214529687349 + }, + "precision": { + "k": 402, + "n": 402, + "rate": 1, + "low": 0.9905345825637017, + "high": 1 + }, + "rubricPass": { + "k": 360, + "n": 402, + "rate": 0.8955223880597015, + "low": 0.861784942681361, + "high": 0.9217722644212688 + }, + "execution": { + "k": 402, + "n": 402, + "rate": 1, + "low": 0.9905345825637017, + "high": 1 + } + } + } + }, + "byRule": { + "basis.event.create": 402, + "basis.calendar.day": 360, + "basis.event.delete": 336, + "basis.event.reschedule": 276, + "basis.event.create_invite": 126, + "basis.contacts.find": 42, + "basis.event.snooze": 42, + "basis.settings.view": 42, + "basis.event.rename": 42, + "basis.calendar.period": 42, + "basis.calendar.image": 42, + "basis.invite.status": 42, + "basis.calc.evaluate": 42, + "basis.contacts.list": 42, + "basis.reminder.set": 42 + }, + "failureClasses": { + "missed:fell_through_to_ai": 210 + }, + "rubricFindings": { + "RAW_ISO_DATE": 42 + }, + "cells": 3156, + "anchors": 11 + }, + "real": null, + "latency": { + "matcherMicrosP50": 28.270000009797513, + "matcherMicrosP95": 104.30999999516644, + "matcherMicrosMax": 399.2209999996703, + "layerMsP50": 14.665628999995533, + "layerMsP95": 24.39432300000044, + "samples": 2265 + }, + "historyKind": "synthetic", + "generatedAt": "2026-11-02T05:36:34.475Z", + "source": { + "sha": "5169da3c5ac8a1b5d132ac45c05ea1d07625724c", + "dirty": true, + "engineDigest": "736154156269de819a0a775de5e6162f364a300052aca709ea53273c941d4d2f" + }, + "checksum": "ea4b5676135a499470488f6a9a6125d5677fdc371774e46ff833b600a63a4d54" +} diff --git a/docs/intents/kpi-gate.json b/docs/intents/kpi-gate.json index 6cfc27cab..39e196f83 100644 --- a/docs/intents/kpi-gate.json +++ b/docs/intents/kpi-gate.json @@ -62,7 +62,7 @@ "version": "1", "checksum": "98447f88cd71fa57404321b1da6fad4df63a2784d8ace86de0aa3eb84d1f64f8" }, - "benchmarkResult": "docs/intents/benchmarks/20261029T120052Z-synthetic-current.json" + "benchmarkResult": "docs/intents/benchmarks/20261102T053634Z-synthetic-current.json" }, "informational": { "intervals95": { @@ -100,7 +100,7 @@ "labelsChecksum": "19ee899c60362252cd89a08b71fc5621f417f2ece970dbf58222792ffff05f7e", "splitChecksum": "310707bb8c752d46692ceb9b50cc86bda217b681ac54867263ca3e8dda8354f0" }, - "engineDigest": "de45d59076375ddabf818004177c1d8834e9a5b3da1bffed8a38f9e451b147b2", - "benchmarkResult": "docs/intents/benchmarks/20261029T115902Z-real-current.json", + "engineDigest": "736154156269de819a0a775de5e6162f364a300052aca709ea53273c941d4d2f", + "benchmarkResult": "docs/intents/benchmarks/20261102T053443Z-real-current.json", "autoActivate": true } diff --git a/docs/reference/deploy-runbook.md b/docs/reference/deploy-runbook.md index d0962ba13..0c9bf80a7 100644 --- a/docs/reference/deploy-runbook.md +++ b/docs/reference/deploy-runbook.md @@ -116,11 +116,13 @@ GramIO hands every Telegram update to its handler without waiting for the previo - The place in a chat's line is reserved when the message is saved, in the same synchronous step (`chat-logging` middleware → `CalendarBotAgent.reserveChatTurn` → `ChatRunGate.reserve`), not when its handler reaches the agent: a handler that waits on a scene or retry lookup first cannot be overtaken by a message saved after it. The update's handling runs inside that reservation (`withChatTurn`, an async context, `src/services/ai/chat-turn-context.ts`), so the agent run it starts (text pipeline, `/cal`, a button) takes the place over with no id passed along; a run started elsewhere (scheduled, retry) finds none and joins the end without a message of its own, whatever row id it borrows for the action log; the reservation is released when the update's handling ends (an intent answered it, a command, an error) or after 60 s unused (`Not waiting any longer` / `A reserved place in the chat was never used` in the log). - A queued run starts after the earlier one has written its turn. It re-reads the user and the ±2 week schedule and reads its own message after the earlier run's rows (`ChatHistoryRepository.placeUserTurnAfterLatest` sets `chat_history.turn_order`; the row keeps its id, so the action log, the per-user `chatHistoryIds` map and built contexts stay valid). Only a run that took over a reserved place has an own message to place (and only after a bot row of the same conversation, private or group); scheduled/synthetic runs borrow the user's latest row id for the action log and never move it. It is not shown turns queued behind it, in its history or through `get_history` (`ctx.deferredTurnRowIds`), so its model reads «request, what was done, this message». The request budget (90 s) counts from the start of the run, not from arrival. +- A turn that waited is moved by reading position only, never in time: its `chat_history.created_at` stays the time the user sent it, `turn_order` records the move, and the history readers order a row by when it reads (the time of the newest row it was placed behind), then `turn_order`, then id. The model is shown such a turn with `[queued: …]` after its timestamp (`QUEUED_TURN_MARKER`, explained in the system prompt) and the day-reference guard ignores every row saved after the message when it looks for the `ask_user` question the message answers, so a message sent while an earlier run was still working is never read as the answer to a question that run asked afterwards (a queued «Да» could otherwise confirm a duplicate, a forced create or a delete nobody offered it). A button answer, which explicitly belongs to the question it was tapped under, still binds to that question. A voice message takes its place in the line when its transcript row is saved (`handleVoiceMessage`), like a text message. +- Known limits of the queued-turn rule (#929, #953, #957): (1) a button tap that waited behind a run started by another user message, or a reply that reached the chat before the asking run wrote its turn, is also read as queued and the guard loses the question's day; the model is then told to ask what it refers to, it is never executed on a guess. The guard infers the question from row order, it is not carried on the button. (2) The refusal to treat a queued «Да» as consent for `allow_duplicate`, `force` or a delete confirmation is the system prompt plus the guard: no tool handler checks it (a delete still needs the bot's own button tap). (3) A voice transcript can be bound to a question saved while it was being transcribed (#953). - A run that never finishes holds the chat for at most its budget plus 30 s (`CHAT_RUN_GRACE_MS`); then the next run goes on and the log says `Not waiting any longer for the earlier agent run in this chat`. A queued run is in `CalendarBotAgent.inFlight`, so the shutdown drain waits for it; at shutdown the wait ends at once and the run takes its normal fail-fast path (notice and stored retry), so no queued message is dropped. - Log lines to look for: `Agent run started after waiting for the previous run in this chat` (`waitedMs`, `queuedBehind`) and `Routing to AI agent` with `updateId`/`messageId`: the same ids twice mean a Telegram redelivery, new ids a second message. -- While a message waits the chat shows the typing indicator (refreshed every 4 s). If the run ahead outlives its lease (`CHAT_RUN_GRACE_MS` after the 90 s budget, so about 120 s of waiting for the run directly ahead; the lease bounds the wait for that one run only, so the Nth message in a long queue of slow runs waits for every run ahead of it in turn, up to about (N-1) leases), the waiting message goes on beside it and the user is told once (`agent_queue_lease_expired`, ru and en; not for a scheduled run, which has no message of theirs, nor for a supplement); every run that starts beside a still-unfinished earlier run gets the notice, so it is never a silent parallel run. `Agent run started` and `Agent run complete` carry `updateId`/`messageId` too. +- While a message waits the chat shows the typing indicator (refreshed every 4 s). If the run ahead outlives its lease (`CHAT_RUN_GRACE_MS` after the 90 s budget, so about 120 s of waiting for the run directly ahead; the lease bounds the wait for that one run only, so the Nth message in a long queue of slow runs waits for every run ahead of it in turn, up to about (N-1) leases), the waiting message goes on beside it and the user is told once per hung run (`agent_queue_lease_expired`, ru and en): the first later run that has a message of its own to show it announces that the earlier run was given up on, and a run without one (scheduled, or a supplement) never uses the notice up for the next. `Agent run started` and `Agent run complete` carry `updateId`/`messageId` too. - Every kind of run uses the gate, keyed by its chat id: supplement runs (silent, optional text: no typing, no notice, so a supplement can sit behind a main run for up to the same ~120 s and then speaks to nobody) and scheduled/system runs (their chat is the user's private chat, so a scheduled reminder run waits behind an active conversation and vice versa). The worst case for any run is that wait plus its own 90 s budget. -- `create_event` (`EventRepository.createUnlessDuplicate`, one immediate SQLite transaction, so it holds across connections and restarts) compares with the live, non-recurring events of the same calendar (the owner's own events with no Google calendar, the `primary` alias or the id registered as the user's primary calendar, a secretary's `owner_id`, or the same group; events imported from a secondary Google calendar never count) that have the same title (trimmed, whitespace collapsed, case folded) and all-day flag: +- `create_event` (`EventRepository.createUnlessDuplicate`, one immediate SQLite transaction, so it holds across connections and restarts) compares with the live events of the same calendar (a one-off only with one-offs, a series only with series of the same recurrence rule; the owner's own events with no Google calendar, the `primary` alias or the id registered as the user's primary calendar, a secretary's `owner_id`, or the same group; events imported from a secondary Google calendar never count) that have the same title (trimmed, whitespace collapsed, case folded) and all-day flag. A series is matched only as an exact repeat within the window (`duplicate`), never by `same_day`: - `duplicate`: same start and end (all-day: the same floating dates, as `allDayDates` reads them), the same description, location and reminders the caller stated, created within 10 minutes (`RECENT_DUPLICATE_WINDOW_MS`). It comes back as `Event already exists …`: success, `alreadyExists` on the tool result, no second row, reminders, notifications or calendar push. - `same_day`: any other candidate on the same local day, whatever its age (the «move by creating» case of #520): `SAME_DAY_EVENT` error, `mutationState: not_applied`, pointing at `update_event` with the existing id. - The model makes a deliberate second copy with `allow_duplicate: true`. diff --git a/docs/reference/migrations/069_chat_history_turn_order.md b/docs/reference/migrations/069_chat_history_turn_order.md new file mode 100644 index 000000000..7bcd4672f --- /dev/null +++ b/docs/reference/migrations/069_chat_history_turn_order.md @@ -0,0 +1,66 @@ +--- +migration: 069_chat_history_turn_order +rollback-compatible: yes +data-deletion: no +--- + +# Migration 069: where a queued user turn is read in the conversation + +`src/database/migrations.ts`, migration `069_chat_history_turn_order`. Refs #929 (per-chat run +serialization) and #520. + +## What it does + +```sql +ALTER TABLE chat_history ADD COLUMN turn_order REAL; +``` + +One nullable column. Two messages of one chat used to run two agents on overlapping history; a run now +waits for the run ahead of it in its chat (`src/services/ai/chat-run-gate.ts`). The message that waited was +saved before everything the earlier run wrote, so by row order it sits in the middle of that run's output. +When its run starts, `ChatHistoryRepository.placeUserTurnAfterLatest` sets `turn_order` to the newest row's id +plus 0.5 and the history readers order a row by the time of the row it was placed behind and then by +`COALESCE(turn_order, id)`. The row keeps its id (the action log and the per-user row map hold ids) and its +`created_at` (the time the user sent it). A non-NULL `turn_order` also tells the reader that the turn was +sent before the rows it now follows, so the model is shown it as queued and never as the answer to a question +asked after it was sent. + +NULL, which every row written before this migration has, means the row reads at its own id and time exactly +as before. No existing column or row is read, changed or deleted. + +## Order in the file + +Appended after `067_ai_usage_role_preset_cost`, the last entry. Nothing is moved or renumbered. The number +069 leaves 068 to the other open changes that claim it (the AI speed metrics and the image deliveries). + +## Why it is safe to deploy automatically + +- Additive DDL only: `ALTER TABLE ... ADD COLUMN` of one nullable column; existing rows read back NULL. +- The image before this migration names the columns it reads, or selects `*` and ignores the extra one, and + inserts with an explicit column list, so it keeps working on the migrated database. It orders by + `created_at, id`: for a queued turn that is the order the messages were sent in, which is only a different + reading position, never a lost row. + +## Rollback + +An image rollback needs no schema step: the previous image ignores `turn_order` and applied migrations it does +not know. Keep the column and the `migrations` row. + +Dropping the column is never required. If it ever is, it deletes only the reading positions of queued turns: + +```sql +BEGIN IMMEDIATE; +ALTER TABLE chat_history DROP COLUMN turn_order; +DELETE FROM migrations WHERE name = '069_chat_history_turn_order'; +COMMIT; +``` + +## Verifying after deploy + +```sql +SELECT name FROM migrations WHERE name = '069_chat_history_turn_order'; +SELECT count(*) FROM chat_history WHERE turn_order IS NOT NULL; +``` + +Expect one migration row. The count stays 0 until two messages of one chat arrive while an earlier agent run +of that chat is still working. diff --git a/src/bot/handlers/message/voice-message.ts b/src/bot/handlers/message/voice-message.ts index f66a19746..907253d07 100644 --- a/src/bot/handlers/message/voice-message.ts +++ b/src/bot/handlers/message/voice-message.ts @@ -5,6 +5,7 @@ import { mkdir } from 'node:fs/promises'; import { InlineKeyboard } from 'gramio'; import { t } from '../../../config/constants.ts'; import type { User } from '../../../database/types.ts'; +import { withChatTurn } from '../../../services/ai/chat-turn-context.ts'; import type { AgentContext } from '../../../services/ai/types.ts'; import { fixDateOrdinals, @@ -150,16 +151,23 @@ export async function handleVoiceMessage( const logChatId = Number(chatId) !== user.telegram_id ? Number(chatId) : undefined; const chatHistoryId = deps.conversationLogger.logUserMessage(user.telegram_id, transcription, logChatId); - - const agentContext: AgentContext = { - ...buildAgentContextFactory(deps)(user, Number(chatId), transcription), - inputMode: 'voice_message', - incomingMessageId: ctx.id, - chatHistoryId, - voiceFileId: voice.file_id, - }; - - const { responseText } = await deps.agent.run(agentContext); + // The transcript is saved here, not by the logging middleware (which saw no text): its place in the chat's + // line is reserved in the same step, so a voice message sent behind a running turn is answered after it. + const reservation = deps.agent.reserveChatTurn(Number(chatId), chatHistoryId); + + let responseText: string | undefined; + try { + const agentContext: AgentContext = { + ...buildAgentContextFactory(deps)(user, Number(chatId), transcription), + inputMode: 'voice_message', + incomingMessageId: ctx.id, + chatHistoryId, + voiceFileId: voice.file_id, + }; + ({ responseText } = await withChatTurn(reservation, () => deps.agent.run(agentContext))); + } finally { + reservation.release(); + } // Send voice reply if TTS is available and user has opted in if (responseText && user.voice_response_enabled === 1 && deps.sendVoice) { diff --git a/src/database/repositories/chat-history.repository.ts b/src/database/repositories/chat-history.repository.ts index 108b973bd..7ea84d809 100644 --- a/src/database/repositories/chat-history.repository.ts +++ b/src/database/repositories/chat-history.repository.ts @@ -2,15 +2,22 @@ import type { Database } from 'bun:sqlite'; import type { ChatHistoryMessage } from '../types.ts'; -/** `AND id NOT IN (?, …)` for the given rows; nothing when there are none. */ +/** `id NOT IN (?, …)` for the given rows. */ function excludedRowsSql(excludeIds: ReadonlySet): string { - return excludeIds.size > 0 ? `AND id NOT IN (${[...excludeIds].map(() => '?').join(', ')})` : ''; + return `id NOT IN (${[...excludeIds].map(() => '?').join(', ')})`; } -const COLUMNS = 'id, user_id, role, content, chat_id, created_at'; -/** Reading order of a conversation: by time, then by where a turn was placed (its own id unless it was moved after later rows). */ -const ORDER_ASC = 'created_at ASC, COALESCE(turn_order, id) ASC, id ASC'; -const ORDER_DESC = 'created_at DESC, COALESCE(turn_order, id) DESC, id DESC'; +const COLUMNS = 'id, user_id, role, content, chat_id, created_at, turn_order'; +/** + * When a row reads in the conversation: its own time, or for a turn that was moved after later rows + * (`turn_order` = the newest row's id at the move + 0.5) the time of that newest row. `created_at` itself is + * never rewritten: it stays the time the user sent the message. + */ +const POSITION_AT = + 'COALESCE((SELECT p.created_at FROM chat_history p WHERE p.id = CAST(chat_history.turn_order AS INTEGER)), chat_history.created_at)'; +/** Reading order: by when the row reads, then by its place (own id unless moved), then by id. */ +const ORDER_ASC = 'position_at ASC, position ASC, id ASC'; +const ORDER_DESC = 'position_at DESC, position DESC, id DESC'; export class ChatHistoryRepository { constructor(private db: Database) {} @@ -23,11 +30,13 @@ export class ChatHistoryRepository { } /** - * Puts a user turn after everything saved so far, keeping its id. A message that waited behind an earlier - * run was saved before what that run wrote, so by row order it sits in the middle of that run's output; the - * model must read «earlier request, what was done, this message». The row keeps its id (the action log, - * the per-user map and every context built from it stay valid, and ids stay the order messages arrived - * in); only where it reads moves. Returns false when the row is not a user turn of `userId`. + * Puts a user turn after everything saved so far, keeping its id and the time it was sent. A message that + * waited behind an earlier run was saved before what that run wrote, so by row order it sits in the middle + * of that run's output; the model must read «earlier request, what was done, this message». The row keeps + * its id (the action log, the per-user map and every context built from it stay valid, and ids stay the + * order messages arrived in) and its `created_at`; only where it reads moves, and `turn_order` records + * that it was moved, so a reader can tell it was sent before the rows it now follows. Returns false when + * the row is not a user turn of `userId` or nothing was saved after it. */ placeUserTurnAfterLatest(rowId: number, userId: number): boolean { // "After" means after a bot row of the same conversation: the group's rows (whoever triggered them), or @@ -35,7 +44,7 @@ export class ChatHistoryRepository { const result = this.db .prepare( `UPDATE chat_history - SET created_at = datetime('now'), turn_order = (SELECT MAX(id) FROM chat_history) + 0.5 + SET turn_order = (SELECT MAX(id) FROM chat_history) + 0.5 WHERE id = ? AND user_id = ? AND role = 'user' AND EXISTS ( SELECT 1 FROM chat_history later @@ -48,18 +57,28 @@ export class ChatHistoryRepository { return result.changes > 0; } + /** The newest `limit` rows matching `conditions`, oldest first, in reading order. */ + private newestInReadingOrder( + conditions: readonly string[], + args: readonly (number | string)[], + limit: number, + ): ChatHistoryMessage[] { + return this.db + .prepare( + `SELECT ${COLUMNS} FROM ( + SELECT ${COLUMNS}, ${POSITION_AT} AS position_at, COALESCE(turn_order, id) AS position + FROM chat_history WHERE ${conditions.join(' AND ')} + ORDER BY ${ORDER_DESC} LIMIT ? + ) sub ORDER BY ${ORDER_ASC}`, + ) + .all(...args, limit) as ChatHistoryMessage[]; + } + /** `excludeIds` rows (turns queued behind the reader) are left out BEFORE the limit, so they cost no context. */ getRecentByChat(chatId: number, limit = 10, excludeIds: ReadonlySet = new Set()): ChatHistoryMessage[] { - return this.db - .prepare(` - SELECT ${COLUMNS} FROM ( - SELECT ${COLUMNS}, turn_order FROM chat_history - WHERE chat_id = ? ${excludedRowsSql(excludeIds)} - ORDER BY ${ORDER_DESC} - LIMIT ? - ) sub ORDER BY ${ORDER_ASC} - `) - .all(chatId, ...excludeIds, limit) as ChatHistoryMessage[]; + const conditions = ['chat_id = ?']; + if (excludeIds.size > 0) conditions.push(excludedRowsSql(excludeIds)); + return this.newestInReadingOrder(conditions, [chatId, ...excludeIds], limit); } searchByChat( @@ -70,7 +89,7 @@ export class ChatHistoryRepository { const conditions: string[] = ['chat_id = ?']; const args: (number | string)[] = [chatId]; if (excludeIds.size > 0) { - conditions.push(`id NOT IN (${[...excludeIds].map(() => '?').join(', ')})`); + conditions.push(excludedRowsSql(excludeIds)); args.push(...excludeIds); } @@ -79,28 +98,13 @@ export class ChatHistoryRepository { args.push(`%${search}%`); } - const where = conditions.join(' AND '); - return this.db - .prepare( - `SELECT ${COLUMNS} FROM ( - SELECT ${COLUMNS}, turn_order FROM chat_history WHERE ${where} - ORDER BY ${ORDER_DESC} LIMIT ? - ) sub ORDER BY ${ORDER_ASC}`, - ) - .all(...args, limit) as ChatHistoryMessage[]; + return this.newestInReadingOrder(conditions, args, limit); } getRecent(userId: number, limit = 10, excludeIds: ReadonlySet = new Set()): ChatHistoryMessage[] { - return this.db - .prepare(` - SELECT ${COLUMNS} FROM ( - SELECT ${COLUMNS}, turn_order FROM chat_history - WHERE user_id = ? AND chat_id IS NULL ${excludedRowsSql(excludeIds)} - ORDER BY ${ORDER_DESC} - LIMIT ? - ) sub ORDER BY ${ORDER_ASC} - `) - .all(userId, ...excludeIds, limit) as ChatHistoryMessage[]; + const conditions = ['user_id = ?', 'chat_id IS NULL']; + if (excludeIds.size > 0) conditions.push(excludedRowsSql(excludeIds)); + return this.newestInReadingOrder(conditions, [userId, ...excludeIds], limit); } search( @@ -111,7 +115,7 @@ export class ChatHistoryRepository { const conditions: string[] = ['user_id = ?']; const args: (number | string)[] = [userId]; if (excludeIds.size > 0) { - conditions.push(`id NOT IN (${[...excludeIds].map(() => '?').join(', ')})`); + conditions.push(excludedRowsSql(excludeIds)); args.push(...excludeIds); } @@ -128,15 +132,7 @@ export class ChatHistoryRepository { args.push(after); } - const where = conditions.join(' AND '); - return this.db - .prepare( - `SELECT ${COLUMNS} FROM ( - SELECT ${COLUMNS}, turn_order FROM chat_history WHERE ${where} - ORDER BY ${ORDER_DESC} LIMIT ? - ) sub ORDER BY ${ORDER_ASC}`, - ) - .all(...args, limit) as ChatHistoryMessage[]; + return this.newestInReadingOrder(conditions, args, limit); } deleteOlderThan(days: number): number { diff --git a/src/database/repositories/event.repository.ts b/src/database/repositories/event.repository.ts index 7d7256206..e2d946b3c 100644 --- a/src/database/repositories/event.repository.ts +++ b/src/database/repositories/event.repository.ts @@ -279,20 +279,21 @@ export class EventRepository { } /** - * Creates the event unless the calendar already holds it. Candidates are the live, non-recurring events - * of the same calendar (the user's own primary-calendar events, or the same group) with the same title - * (trimmed, whitespace collapsed, case folded) and the same all-day flag: + * Creates the event unless the calendar already holds it. Candidates are the live events of the same + * calendar (the user's own primary-calendar events, or the same group) with the same title (trimmed, + * whitespace collapsed, case folded) and the same all-day flag; a one-off is compared with one-offs, a series + * (with a recurrence rule) with series of the same rule, never across: * - `duplicate`: same start and end (all-day: the same dates) and the same details the caller stated, * created within `windowMs`. That is what a repeated message, a retry after a lost reply or a second * overlapping run produces; the existing row comes back and nothing is written. - * - `same_day`: any other candidate on the same local day (all-day: the same first date), whatever its - * age: a second copy, or the old half of a move done by creating (#520). The caller decides whether the - * user really wants it; nothing is written. + * - `same_day` (one-offs only): any other candidate on the same local day (all-day: the same first date), + * whatever its age: a second copy, or the old half of a move done by creating (#520). The caller decides + * whether the user really wants it; nothing is written. A series is matched only as an exact, recent + * repeat: two series on one day may be a deliberate pair. * - `created`: otherwise. `onCreated` runs in the same transaction as the insert, so an event never * persists without the effects that must accompany it. * The check and the insert are one immediate transaction, so two writers (connections, or a process - * restarted in between) cannot both miss each other's row. A recurring event is always created: a series - * never stands in for another. + * restarted in between) cannot both miss each other's row. */ createUnlessDuplicate( data: CreateEventData, @@ -301,7 +302,7 @@ export class EventRepository { ): { event: CalendarEvent; outcome: CreateOutcome } { return this.db .transaction((): { event: CalendarEvent; outcome: CreateOutcome } => { - const match = data.recurrence_rule ? null : this.findSameEvent(data, windowMs); + const match = this.findSameEvent(data, windowMs); if (match) return { event: this.findCreated(match.id, data), outcome: match.outcome }; const event = this.create(data); onCreated?.(event); @@ -316,13 +317,14 @@ export class EventRepository { ): { id: number; outcome: Exclude } | null { assertValidEventTimestamps(data); const isGroup = data.owner_type === 'group' && data.group_id != null; + const series = data.recurrence_rule != null && data.recurrence_rule !== ''; const candidates = this.db .prepare( `SELECT id, title, description, location, start_at, end_at, timezone, reminder_overrides, created_at >= datetime('now', ?) AS recent FROM events WHERE is_deleted = 0 AND is_cancelled = 0 - AND recurrence_rule IS NULL AND parent_event_id IS NULL + AND ${series ? 'recurrence_rule = ?' : 'recurrence_rule IS NULL'} AND parent_event_id IS NULL AND ${ isGroup ? "owner_type = 'group' AND group_id = ?" @@ -336,6 +338,7 @@ export class EventRepository { ) .all( `-${Math.ceil(windowMs / 1000)} seconds`, + ...(series ? [data.recurrence_rule!] : []), ...(isGroup ? [data.group_id!] : [data.user_id, data.user_id]), data.all_day ? 1 : 0, data.start_at, @@ -361,6 +364,7 @@ export class EventRepository { : (c: { start_at: string }) => localDay(c.start_at, data.timezone) === localDay(data.start_at, data.timezone); const duplicate = sameTitle.find((c) => c.recent === 1 && sameSpan(c) && sameDetails(data, c)); if (duplicate) return { id: duplicate.id, outcome: 'duplicate' }; + if (series) return null; const sameDayEvent = sameTitle.find(sameDay); return sameDayEvent ? { id: sameDayEvent.id, outcome: 'same_day' } : null; } diff --git a/src/database/types.ts b/src/database/types.ts index e9beba88a..844e650e3 100644 --- a/src/database/types.ts +++ b/src/database/types.ts @@ -150,6 +150,8 @@ export interface ChatHistoryMessage { content: string; // plain text for user, JSON content blocks for assistant/tool chat_id: number | null; created_at: string; + /** Set when this user turn waited behind an earlier run and reads after rows saved after it (it was sent before them). */ + turn_order?: number | null; } // ── Google Sync row types ── diff --git a/src/services/ai/agent/history-rows.ts b/src/services/ai/agent/history-rows.ts index 8fd75b603..4f67c420e 100644 --- a/src/services/ai/agent/history-rows.ts +++ b/src/services/ai/agent/history-rows.ts @@ -12,6 +12,13 @@ function withTimestamp(text: string, createdAt: string, timezone: string): strin return `[${local}] ${text}`; } +/** + * Shown after the time of a user turn that waited behind an earlier run of its chat and reads after what that + * run wrote: the user sent it before those rows existed, so it replies to none of the questions in them. + */ +export const QUEUED_TURN_MARKER = + '[queued: sent while the assistant was still handling the previous message; it answers nothing the assistant said after it was sent]'; + /** Plain-text fallback for group-chat sender attribution. */ /** * Exported so the system prompt's description of this prefix can be pinned to @@ -148,19 +155,24 @@ export function parseHistoryRow(msg: ChatHistoryMessage, timezone: string): Mess } // role === 'user' + const queued = msg.turn_order == null ? '' : `${QUEUED_TURN_MARKER} `; const activity = ActivityEventCodec.safeParse(msg.content); if (activity.success) { return [ { role: 'user', - content: withTimestamp(formatActivityEvent(activity.data as ActivityEvent), msg.created_at, timezone), + content: withTimestamp( + `${queued}${formatActivityEvent(activity.data as ActivityEvent)}`, + msg.created_at, + timezone, + ), }, ]; } return [ { role: 'user', - content: withTimestamp(msg.content, msg.created_at, timezone), + content: withTimestamp(`${queued}${msg.content}`, msg.created_at, timezone), }, ]; } diff --git a/src/services/ai/agent/queued-turn.ts b/src/services/ai/agent/queued-turn.ts index 8ed919d12..820a9b692 100644 --- a/src/services/ai/agent/queued-turn.ts +++ b/src/services/ai/agent/queued-turn.ts @@ -72,27 +72,29 @@ export function scheduleWindowAround( /** * Sets up a run that has just been given its turn. Always: the turns queued behind it (reserved at the moment * they were saved, so a message whose handler is still busy elsewhere counts too) are kept out of its history, - * and the message it answers is read after what earlier runs wrote (same row, same id). After a wait: the user - * and the schedule window are read again (the earlier run may have changed either). A step that fails leaves - * the run on what it had; this is a refinement, never a reason to fail the run. + * and the message it answers is read after what earlier runs wrote (same row, same id). After a wait, or when + * that placement found rows saved after the message (an earlier run finished just before this one reached the + * gate, so it never waited): the user and the schedule window are read again (the earlier run may have changed + * either). A step that fails leaves the run on what it had; this is a refinement, never a reason to fail the run. */ export function prepareQueuedTurn(ctx: AgentContext, slot: ChatRunSlot): void { ctx.deferredTurnRowIds = () => slot.queuedTurnRowIds(); const userId = ctx.user.telegram_id; - if (slot.waited) { + let placed = false; + // Only a run's own message is placed: a scheduled or synthetic run borrows the user's latest row for the + // action log, and a supplement answers a message the main run already placed. + if (slot.ownRowId !== undefined && !ctx.supplementMode) { try { - ctx.user = ctx.userRepo.findByTelegramId(userId) ?? ctx.user; - if (ctx.recentEventsWindow) ctx.recentEventsWindow = scheduleWindowAround(ctx.eventService, userId); + placed = ctx.chatHistory.placeUserTurnAfterLatest(slot.ownRowId, userId); } catch (err) { - queuedTurnLogger.warn({ err, userId }, 'Could not refresh the context of a run that waited for its turn'); + queuedTurnLogger.warn({ err, userId }, 'Could not place the message of a run behind the earlier turn'); } } - // Only a run's own message is placed: a scheduled or synthetic run borrows the user's latest row for the - // action log, and a supplement answers a message the main run already placed. - if (slot.ownRowId === undefined || ctx.supplementMode) return; + if (!slot.waited && !placed) return; try { - ctx.chatHistory.placeUserTurnAfterLatest(slot.ownRowId, userId); + ctx.user = ctx.userRepo.findByTelegramId(userId) ?? ctx.user; + if (ctx.recentEventsWindow) ctx.recentEventsWindow = scheduleWindowAround(ctx.eventService, userId); } catch (err) { - queuedTurnLogger.warn({ err, userId }, 'Could not place the message of a run behind the earlier turn'); + queuedTurnLogger.warn({ err, userId }, 'Could not refresh the context of a run that waited for its turn'); } } diff --git a/src/services/ai/day-reference-guard.ts b/src/services/ai/day-reference-guard.ts index 31a1eb52c..08bd417a9 100644 --- a/src/services/ai/day-reference-guard.ts +++ b/src/services/ai/day-reference-guard.ts @@ -93,6 +93,19 @@ function pendingFrom(origin: ChatHistoryMessage, question: AskedQuestion | null) return { origin, question: question.text, options: question.options, askedAt: question.askedAt }; } +/** + * The rows the sender of the message at `index` could have seen, newest first: those before it in reading + * order that were saved before it. A turn that waited behind an earlier run is read after what that run + * wrote, but the sender never saw those rows, so they are never what the message answers. + */ +function rowsSeenBefore(history: ChatHistoryMessage[], index: number): ChatHistoryMessage[] { + const sentAtRow = history[index]!.id; + return history + .slice(0, index) + .filter((row) => row.id <= sentAtRow) + .reverse(); +} + /** * The ask_user question the current message answers, with the user message that led to * it and when the question was asked — or null when the turn before this message did not @@ -100,11 +113,10 @@ function pendingFrom(origin: ChatHistoryMessage, question: AskedQuestion | null) * transcript is never saved). */ function pendingQuestion(messageText: string, history: ChatHistoryMessage[]): PendingQuestion | null { - let index = currentMessageIndex(messageText, history); + const index = currentMessageIndex(messageText, history); if (index < 0) return null; let question: AskedQuestion | null = null; - for (index--; index >= 0; index--) { - const row = history[index]!; + for (const row of rowsSeenBefore(history, index)) { if (row.role === 'tool') continue; if (row.role === 'user') return pendingFrom(row, question); if (ActivityCodec.safeParse(row.content).success) continue; @@ -160,13 +172,13 @@ function assistantText(content: string): string { /** * Whether the bot's last reply before this message asked something in plain text ("Во * сколько?"): the message then answers it, and the day may have been named before. Any - * question counts, a closing "Что-то ещё?" too: the time-only rule then imposes nothing. + * question counts, a closing "Что-то ещё?" too: the time-only rule then imposes nothing. Rows saved after + * the message was are skipped, as in pendingQuestion. */ function answersPlainQuestion(messageText: string, history: ChatHistoryMessage[]): boolean { - let index = currentMessageIndex(messageText, history); + const index = currentMessageIndex(messageText, history); if (index < 0) return false; - for (index--; index >= 0; index--) { - const row = history[index]!; + for (const row of rowsSeenBefore(history, index)) { if (row.role === 'tool') continue; if (row.role === 'user') return false; const text = assistantText(row.content); diff --git a/src/services/ai/system-prompt.ts b/src/services/ai/system-prompt.ts index 68818adf5..e16f23929 100644 --- a/src/services/ai/system-prompt.ts +++ b/src/services/ai/system-prompt.ts @@ -209,7 +209,8 @@ function buildContextSection(): string { - "Current local time" in User Info is the authoritative clock. Each message includes a LOCAL timestamp in brackets, e.g. [2026-03-18 10:30] — already in the user's timezone, no conversion needed. - CALCULATE RULE: For ANY arithmetic — time, dates, durations, numbers — ALWAYS call the \`calculate\` tool. Never compute in your head. If calculate returns an error, report it to the user — do not compute manually. - Messages from group chats are prefixed with [From: name (id:N)] after the timestamp. In groups, be brief and relevant — you were triggered by a calendar keyword or direct mention. -- Messages from private chats have no group prefix.`; +- Messages from private chats have no group prefix. +- A user message marked [queued: …] was sent while you were still handling the previous message, so the user had not seen anything you said after sending it. It answers none of the questions, confirmations or warnings you asked after that time (an ask_user question or SAME_DAY_EVENT notice of your previous run included): treat it as its own request, never as consent to a write, a duplicate or a delete that such a question offered; when it is a bare "yes" with nothing earlier to confirm, ask what it refers to.`; } function buildLanguageRule(ctx: AgentContext): string { diff --git a/src/services/ai/tools.ts b/src/services/ai/tools.ts index 124bde0de..a31efcad6 100644 --- a/src/services/ai/tools.ts +++ b/src/services/ai/tools.ts @@ -86,8 +86,7 @@ const toolDefinitions: ToolDefinition[] = [ name: 'create_event', description: 'Create a new calendar event. Returns the created event. ' + - 'A repeat of an event saved minutes ago returns it as "already exists": do not call again. ' + - 'Same title on the same day as an existing event is rejected with SAME_DAY_EVENT: to move or change that event use update_event. ' + + 'A just-saved repeat returns "already exists": do not repeat. SAME_DAY_EVENT: to change it use update_event. ' + 'If the event time is in the past, the tool will reject with PAST_EVENT error — ' + 'you must ask the user to confirm via ask_user, then retry with force: true.', input_schema: { @@ -119,7 +118,7 @@ const toolDefinitions: ToolDefinition[] = [ }, allow_duplicate: { type: 'boolean', - description: 'True only if the user explicitly confirmed a second event with the same title on the same day.', + description: 'Only after the user confirmed a second same-day event.', }, scope: scopeProperty, owner_id: ownerIdProperty, diff --git a/test/bot/handlers/message-handler-fixtures.ts b/test/bot/handlers/message-handler-fixtures.ts index ffabfa96b..ef7e0af41 100644 --- a/test/bot/handlers/message-handler-fixtures.ts +++ b/test/bot/handlers/message-handler-fixtures.ts @@ -7,9 +7,12 @@ import type { BotCommandContext } from '../../../src/bot/types.ts'; import { stub } from '../partial-stub.ts'; import { makeUserRow, sentMessage, telegramChat, telegramUser } from '../telegram-fixtures.ts'; +/** What `agent.reserveChatTurn` returns for a voice turn: a place nobody queues behind. */ +export const reserveIdleChatTurn = () => mock(() => ({ chatId: 100, rowId: 1, release: () => {} })); + export function makeDeps(overrides: { [key: string]: unknown } = {}) { return { - agent: { run: mock(() => Promise.resolve()) }, + agent: { run: mock(() => Promise.resolve()), reserveChatTurn: reserveIdleChatTurn() }, eventService: { getEventsInRange: mock(() => []), getLatestCreated: mock(() => null) }, holidayService: {}, chatHistory: {}, @@ -60,6 +63,8 @@ export function makeMessageCtx(overrides: Partial = {}): MessageCtx threadId: undefined, venue: undefined, location: undefined, + chatHistoryId: undefined, + updateId: undefined, voice: undefined, replyMessage: undefined, text: 'привет', @@ -75,7 +80,10 @@ export function makeMessageCtx(overrides: Partial = {}): MessageCtx /** Message handler dependencies: the required ones stubbed for a plain text turn; `overrides` replace or add. */ export function makeHandlerDeps(overrides: Partial = {}): MessageHandlerDeps { return { - agent: stub({ run: mock(() => Promise.resolve({ responseText: '', toolCalls: [], toolResults: [] })) }), + agent: stub({ + run: mock(() => Promise.resolve({ responseText: '', toolCalls: [], toolResults: [] })), + reserveChatTurn: reserveIdleChatTurn(), + }), eventService: stub({ getEventsInRange: mock(() => []), getLatestCreated: mock(() => null) }), holidayService: stub({}), chatHistory: stub({}), diff --git a/test/bot/handlers/message-handler-units.test.ts b/test/bot/handlers/message-handler-units.test.ts index 678bdb89e..8c87b19b5 100644 --- a/test/bot/handlers/message-handler-units.test.ts +++ b/test/bot/handlers/message-handler-units.test.ts @@ -9,12 +9,15 @@ import { } from '../../../src/bot/handlers/message.handler.ts'; import type { UserMemoryRepository } from '../../../src/database/repositories/user-memory.repository.ts'; import type { Intent } from '../../../src/database/types.ts'; +import type { CalendarBotAgent } from '../../../src/services/ai/agent.ts'; +import type { ChatTurnReservation } from '../../../src/services/ai/chat-run-gate.ts'; +import { currentChatTurn } from '../../../src/services/ai/chat-turn-context.ts'; import type { KokoroTtsService } from '../../../src/services/voice/kokoro-tts-service.ts'; import type { SileroTtsService } from '../../../src/services/voice/silero-tts-service.ts'; import type { StressDictionary } from '../../../src/services/voice/stress-dictionary.ts'; import { stub } from '../partial-stub.ts'; import { makeUserRow } from '../telegram-fixtures.ts'; -import { makeHandlerDeps, makeMessageCtx } from './message-handler-fixtures.ts'; +import { makeHandlerDeps, makeMessageCtx, reserveIdleChatTurn } from './message-handler-fixtures.ts'; describe('toEventSummary', () => { const BASE_EVENT = { @@ -112,6 +115,7 @@ describe('voice reply TTS fallback', () => { return makeHandlerDeps({ agent: stub({ run: mock(() => Promise.resolve({ responseText: 'Ответ бота', toolCalls: [], toolResults: [] })), + reserveChatTurn: reserveIdleChatTurn(), }), sendVoice: mock(() => Promise.resolve()), botToken: 'test-token', @@ -129,6 +133,26 @@ describe('voice reply TTS fallback', () => { }); } + test('a transcript takes its place in the chat line when it is saved, runs inside it and gives it up after', async () => { + const reservation: ChatTurnReservation = { chatId: 1, rowId: 77, release: mock(() => {}) }; + const reserveChatTurn = mock(() => reservation); + const seenByRun: (ChatTurnReservation | undefined)[] = []; + const run = mock(() => { + seenByRun.push(currentChatTurn()); + return Promise.resolve({ responseText: 'Ответ бота', toolCalls: [], toolResults: [] }); + }); + const deps = makeVoiceDeps({ + agent: stub({ run, reserveChatTurn }), + conversationLogger: stub({ logUserMessage: mock(() => 77), logBotResponse: mock(() => {}) }), + }); + + await createMessageHandler(deps)(makeVoiceCtx()); + + expect(reserveChatTurn).toHaveBeenCalledWith(100, 77); + expect(seenByRun).toEqual([reservation]); + expect(reservation.release).toHaveBeenCalledTimes(1); + }); + test('uses primary sileroTts when available', async () => { const synthesize = mock(() => Promise.resolve(audioBuffer)); const fallbackTts = { synthesize: mock(() => Promise.resolve(audioBuffer)) }; diff --git a/test/bot/handlers/message.handler.test.ts b/test/bot/handlers/message.handler.test.ts index 3c8d7718a..4fe865345 100644 --- a/test/bot/handlers/message.handler.test.ts +++ b/test/bot/handlers/message.handler.test.ts @@ -1,6 +1,14 @@ import { describe, expect, mock, test } from 'bun:test'; import { createMessageHandler } from '../../../src/bot/handlers/message.handler.ts'; -import { type MockCtxOverrides, makeCtx, makeDeps } from './message-handler-fixtures.ts'; +import type { CalendarBotAgent } from '../../../src/services/ai/agent.ts'; +import { stub } from '../partial-stub.ts'; +import { + type MockCtxOverrides, + makeCtx, + makeDeps, + makeHandlerDeps, + makeMessageCtx, +} from './message-handler-fixtures.ts'; describe('createMessageHandler', () => { test('routes text message to AI agent in private chat', async () => { @@ -12,11 +20,12 @@ describe('createMessageHandler', () => { test('links the run to the row saved for this very message, not to the user’s latest row', async () => { // A newer message of the same user already replaced the per-user entry (row 99) while this one was handled. - const deps = makeDeps({ chatHistoryIds: new Map([[100, 99]]) }); - const handler = createMessageHandler(deps as never); - await handler({ ...makeCtx(), chatHistoryId: 41 } as never); - const call = (deps.agent.run as ReturnType).mock.calls[0]![0] as { chatHistoryId: number }; - expect(call.chatHistoryId).toBe(41); + const run = mock(() => + Promise.resolve({ responseText: '', toolCalls: [], toolResults: [] }), + ); + const deps = makeHandlerDeps({ agent: stub({ run }), chatHistoryIds: new Map([[100, 99]]) }); + await createMessageHandler(deps)(makeMessageCtx({ chatHistoryId: 41 })); + expect(run.mock.calls[0]?.[0].chatHistoryId).toBe(41); }); test('ignores messages without text', async () => { diff --git a/test/bot/handlers/voice-prompt.test.ts b/test/bot/handlers/voice-prompt.test.ts index 695b47edc..1b1b3523a 100644 --- a/test/bot/handlers/voice-prompt.test.ts +++ b/test/bot/handlers/voice-prompt.test.ts @@ -17,7 +17,10 @@ function makeFetch(filePath = 'voice/file.ogg') { function makeVoiceDeps(overrides: Partial & { [key: string]: unknown } = {}) { return { - agent: { run: mock(() => Promise.resolve({ responseText: 'Ответ от AI', toolCalls: [], toolResults: [] })) }, + agent: { + run: mock(() => Promise.resolve({ responseText: 'Ответ от AI', toolCalls: [], toolResults: [] })), + reserveChatTurn: mock(() => ({ chatId: 100, rowId: 1, release: () => {} })), + }, eventService: { getEventsInRange: mock(() => []) }, holidayService: {}, chatHistory: {}, diff --git a/test/bot/pipeline/ai-agent-layer-overlap.test.ts b/test/bot/pipeline/ai-agent-layer-overlap.test.ts index 23b200e86..e49255ff7 100644 --- a/test/bot/pipeline/ai-agent-layer-overlap.test.ts +++ b/test/bot/pipeline/ai-agent-layer-overlap.test.ts @@ -8,6 +8,7 @@ import { createAiAgentLayer } from '../../../src/bot/pipeline/ai-agent-layer.ts' import type { PipelineLayer } from '../../../src/bot/pipeline/types.ts'; import type { BotCommandContext } from '../../../src/bot/types.ts'; import { ActionLogRepository } from '../../../src/database/repositories/action-log.repository.ts'; +import { QUEUED_TURN_MARKER } from '../../../src/services/ai/agent/history-rows.ts'; import { CalendarBotAgent } from '../../../src/services/ai/agent.ts'; import type { ChatTurnReservation } from '../../../src/services/ai/chat-run-gate.ts'; import { withChatTurn } from '../../../src/services/ai/chat-turn-context.ts'; @@ -15,12 +16,15 @@ import { executeTool } from '../../../src/services/ai/tool-executor.ts'; import type { AgentContext } from '../../../src/services/ai/types.ts'; import { cmdLogger } from '../../../src/utils/logger.ts'; import { + ASKED, + AskingHost, type ModelRequest, REVIEW, recordDelivery, SlowHost, STANDUP, settle, + textOf, titlesInCalendar, userTurn, } from '../../helpers/overlap-fixtures.ts'; @@ -262,4 +266,46 @@ describe('overlapping confirmations in one private chat (2026-11-01)', () => { info.mockRestore(); } }); + + describe('a message queued behind a run that ends in ask_user', () => { + const SENT_BEFORE_THE_QUESTION = '2020-01-01 00:00:05'; + + test('is an independent turn: it is read after the question but answers none of it, and creates nothing', async () => { + const asking = new AskingHost(); + fixture.ctx.sender = { ...fixture.sender, sendButtons: async () => ({ message_id: 43 }) }; + const layer = layerFor(new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: asking.model })); + + const first = arrive(layer, 'Move the standup'); + await asking.reached; + const secondRow = receive('Yes'); + // Telegram stamps the message when the user sent it, long before the first run's question existed. + fixture.db + .prepare('UPDATE chat_history SET created_at = ? WHERE id = ?') + .run(SENT_BEFORE_THE_QUESTION, secondRow); + const second = deliver(layer, 'Yes', secondRow); + await settle(); + asking.release(); + await Promise.all([first, second]); + + // The model read the queued «Yes» as the answer to a question it was never shown: nothing is created. + expect(titlesInCalendar(fixture)).toEqual([]); + + const secondRun = asking.requests.find((request) => userTurn(request.messages.at(-1)).includes('Yes')); + expect(secondRun).toBeDefined(); + const shown = secondRun?.messages ?? []; + const questionAt = shown.findIndex( + (m) => m.role === 'assistant' && JSON.stringify(m.tool_calls ?? []).includes(ASKED.question), + ); + // Order: it is the newest turn, after the question the first run asked... + expect(questionAt).toBeGreaterThan(-1); + expect(shown.at(-1)?.role).toBe('user'); + // ...but it is shown with the time it was sent and as queued, not as a reply to that question. + expect(textOf(shown.at(-1))).toContain(QUEUED_TURN_MARKER); + expect(JSON.stringify(shown.at(-1))).toContain(SENT_BEFORE_THE_QUESTION); + + // The stored row keeps the time the user sent it. + const stored = fixture.db.prepare('SELECT created_at FROM chat_history WHERE id = ?').get(secondRow); + expect(stored).toEqual({ created_at: SENT_BEFORE_THE_QUESTION }); + }); + }); }); diff --git a/test/database/migrations-067.test.ts b/test/database/migrations-067.test.ts index eb9e0a50b..ef53089ac 100644 --- a/test/database/migrations-067.test.ts +++ b/test/database/migrations-067.test.ts @@ -7,14 +7,19 @@ import { runMigrations } from '../../src/database/schema.ts'; const MIGRATION = '067_ai_usage_role_preset_cost'; -test('is the last migration, appended after 066', () => { - expect(migrations.at(-1)?.name).toBe(MIGRATION); - expect(migrations.at(-2)?.name).toBe('066_ai_usage_events'); +const BEFORE_067 = migrations.slice( + 0, + migrations.findIndex((migration) => migration.name === MIGRATION), +); + +test('is appended directly after 066', () => { + const names = migrations.map((migration) => migration.name); + expect(names.indexOf(MIGRATION)).toBe(names.indexOf('066_ai_usage_events') + 1); }); test('adds role, preset and cost_micro_usd, all nullable, once; rows written before it read back NULL', () => { const db = new Database(':memory:'); - runMigrations(db, migrations.slice(0, -1)); + runMigrations(db, BEFORE_067); db.exec( "INSERT INTO ai_usage_events (at, provider, model, chain, purpose, outcome) VALUES (1, 'zai', 'm', 'smart', 'agent', 'ok')", ); diff --git a/test/database/repositories/chat-history.repository.test.ts b/test/database/repositories/chat-history.repository.test.ts index 2054494be..950d1d738 100644 --- a/test/database/repositories/chat-history.repository.test.ts +++ b/test/database/repositories/chat-history.repository.test.ts @@ -286,6 +286,25 @@ describe('ChatHistoryRepository queued-turn handling', () => { expect(repo.getRecent(USER_ID, 20).at(-1)?.id).toBe(confirm); }); + test('the turn keeps the time it was sent and is marked as moved; reading order does not depend on that time', () => { + const confirm = repo.save(USER_ID, 'user', 'Yes'); + db.prepare('UPDATE chat_history SET created_at = ? WHERE id = ?').run('2020-01-01 00:00:05', confirm); + const output = repo.save(USER_ID, 'assistant', 'earlier output'); + const other = repo.save(USER_ID, 'user', 'Thanks'); + + expect(repo.placeUserTurnAfterLatest(confirm, USER_ID)).toBe(true); + const answer = repo.save(USER_ID, 'assistant', 'answer to the moved turn'); + + const rows = repo.getRecent(USER_ID, 20); + expect(rows.map((row) => row.content)).toEqual(['earlier output', 'Thanks', 'Yes', 'answer to the moved turn']); + const moved = rows.find((row) => row.id === confirm); + expect(moved?.created_at).toBe('2020-01-01 00:00:05'); + expect(moved?.turn_order).toBe(other + 0.5); + expect(rows.find((row) => row.id === output)?.turn_order).toBeNull(); + // The newest two rows by reading order, not by the time the moved turn was sent. + expect(repo.getRecent(USER_ID, 2).map((row) => row.id)).toEqual([confirm, answer]); + }); + test('rows saved afterwards, in the same second, still read after the moved turn', () => { const confirm = repo.save(USER_ID, 'user', 'Yes'); repo.save(USER_ID, 'assistant', 'earlier output'); diff --git a/test/helpers/overlap-fixtures.ts b/test/helpers/overlap-fixtures.ts index 519305636..728a8fb0b 100644 --- a/test/helpers/overlap-fixtures.ts +++ b/test/helpers/overlap-fixtures.ts @@ -3,6 +3,7 @@ // model is scripted; agents, tools, the 5 s repeated-call throttle and the database are real. import { spyOn } from 'bun:test'; import type OpenAI from 'openai'; +import { QUEUED_TURN_MARKER } from '../../src/services/ai/agent/history-rows.ts'; import type { StreamCallbacks, StreamRoundOptions, StreamRoundResult } from '../../src/services/ai/streaming.ts'; import type { TelegramSender } from '../../src/services/ai/types.ts'; import { type AgentRunFixture, fakeMetrics, isValidatorCall, USER_ID } from '../services/ai/agent-run-helpers.ts'; @@ -21,9 +22,11 @@ export function textOf(message: OpenAI.ChatCompletionMessageParam | undefined): return typeof content === 'string' ? content : JSON.stringify(content ?? ''); } -/** A user turn as the model is shown it, without the timestamp the history builder prefixes. */ +/** A user turn as the model is shown it, without the timestamp (and queued marker) the history builder prefixes. */ export function userTurn(message: OpenAI.ChatCompletionMessageParam | undefined): string { - return textOf(message).replace(/^\[[^\]]*\] /, ''); + return textOf(message) + .replace(/^\[[^\]]*\] /, '') + .replace(`${QUEUED_TURN_MARKER} `, ''); } function textRound(text: string): StreamRoundResult { @@ -150,3 +153,64 @@ export function recordDelivery(sender: TelegramSender): string[] { }; return shown; } + +export const ASKED = { question: 'Which day for the Synthetic Standup?', options: ['Monday', 'Tuesday'] }; + +function singleToolRound(id: string, name: string, args: object): StreamRoundResult { + const call = { id, name, arguments: JSON.stringify(args) }; + return { + text: '', + toolCalls: [call], + finishReason: 'tool_calls', + assistantMessage: { + role: 'assistant', + content: null, + tool_calls: [{ id, type: 'function' as const, function: { name, arguments: call.arguments } }], + }, + providerUsed: 'mock', + }; +} + +/** + * The first run ends in ask_user. Every later request is answered by a model that reads what it is shown: a + * user turn that follows that question and is not marked as queued is the user's answer to it (the Standup is + * created, a deliberate copy); a queued turn was written before the question existed and answers nothing. + */ +export class AskingHost { + readonly requests: ModelRequest[] = []; + private readonly released = Promise.withResolvers(); + private readonly reachedFirst = Promise.withResolvers(); + private firstRequestSeen = false; + + /** Settles once the first model request is waiting. */ + get reached(): Promise { + return this.reachedFirst.promise; + } + + /** The parked request returns its ask_user question. */ + release(): void { + this.released.resolve(); + } + + model = async (opts: StreamRoundOptions, callbacks: StreamCallbacks = {}): Promise => { + if (isValidatorCall(opts)) return { ...textRound('APPROVE'), metrics: fakeMetrics('hf', 10, 2) }; + this.requests.push({ messages: opts.messages }); + if (!this.firstRequestSeen) { + this.firstRequestSeen = true; + this.reachedFirst.resolve(); + await this.released.promise; + return singleToolRound('ask-1', 'ask_user', ASKED); + } + const messages = opts.messages; + const last = messages.at(-1); + const questionAbove = messages + .slice(0, -1) + .some((m) => m.role === 'assistant' && JSON.stringify(m.tool_calls ?? []).includes('ask_user')); + const readAsAnswer = last?.role === 'user' && questionAbove && !/queued/i.test(textOf(last)); + const round = readAsAnswer + ? singleToolRound('create-answer', 'create_event', { ...STANDUP, allow_duplicate: true }) + : textRound('Noted: that message answers nothing I asked.'); + callbacks.onTextDelta?.(round.text); + return round; + }; +} diff --git a/test/scripts/report-ai-usage.test.ts b/test/scripts/report-ai-usage.test.ts index ee3c4eff1..9ab56cbe1 100644 --- a/test/scripts/report-ai-usage.test.ts +++ b/test/scripts/report-ai-usage.test.ts @@ -152,7 +152,13 @@ describe('scripts/report-ai-usage.ts', () => { dirs.push(dir); const path = join(dir, 'calendar.db'); const db = new Database(path); - runMigrations(db, migrations.slice(0, -1)); + runMigrations( + db, + migrations.slice( + 0, + migrations.findIndex((migration) => migration.name === '067_ai_usage_role_preset_cost'), + ), + ); db.close(); const result = run('--database', path); diff --git a/test/services/ai/agent-chat-queue.test.ts b/test/services/ai/agent-chat-queue.test.ts index 212eed753..170f38436 100644 --- a/test/services/ai/agent-chat-queue.test.ts +++ b/test/services/ai/agent-chat-queue.test.ts @@ -100,7 +100,8 @@ describe('CalendarBotAgent.run() in one chat', () => { expect(titlesInCalendar(fixture)).toEqual([STANDUP.title, REVIEW.title]); }); - test('a run that waited reads the user and schedule as the earlier run left them', async () => { + /** A model that changes the user's timezone during its first request (parked until released) and records each system prompt. */ + function timezoneChangingModel() { const prompts: string[] = []; const parked = Promise.withResolvers(); const reached = Promise.withResolvers(); @@ -126,19 +127,40 @@ describe('CalendarBotAgent.run() in one chat', () => { } return answer; }; - const agent = new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl }); + return { prompts, streamImpl, reached: reached.promise, release: () => parked.resolve() }; + } + + test('a run that waited reads the user and schedule as the earlier run left them', async () => { + const model = timezoneChangingModel(); + const agent = new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: model.streamImpl }); const first = go(agent, turn(agent, 'first message')); - await reached.promise; + await model.reached; const second = go(agent, turn(agent, 'second message')); // its context still holds the old user await settle(); - expect(prompts).toHaveLength(1); - parked.resolve(); + expect(model.prompts).toHaveLength(1); + model.release(); await Promise.all([first, second]); - expect(prompts[0]).toContain('UTC'); - expect(prompts[0]).not.toContain('Pacific/Auckland'); - expect(prompts[1]).toContain('Pacific/Auckland'); + expect(model.prompts[0]).toContain('UTC'); + expect(model.prompts[0]).not.toContain('Pacific/Auckland'); + expect(model.prompts[1]).toContain('Pacific/Auckland'); + }); + + test('a run whose earlier run finished just before it reached the gate reads the same fresh context', async () => { + const model = timezoneChangingModel(); + const agent = new CalendarBotAgent(fixture.config, fixture.sender, { streamImpl: model.streamImpl }); + + const first = go(agent, turn(agent, 'first message')); + await model.reached; + const secondContext = turn(agent, 'second message'); // saved while the first run works; its handler is slow + model.release(); + await first; + const second = go(agent, secondContext); // nothing is ahead any more: it never waits + await second; + + expect(model.prompts[0]).not.toContain('Pacific/Auckland'); + expect(model.prompts[1]).toContain('Pacific/Auckland'); }); test('a queued message gets one answer of its own and its turn is saved once, after the earlier run', async () => { diff --git a/test/services/ai/day-reference-guard.test.ts b/test/services/ai/day-reference-guard.test.ts index 6b5d00b8c..6f7efa7d6 100644 --- a/test/services/ai/day-reference-guard.test.ts +++ b/test/services/ai/day-reference-guard.test.ts @@ -289,6 +289,54 @@ describe('deleting on a day the user did not name', () => { expect(liveRows().map((row) => row.id)).toEqual([past]); }); + test('a "Да" that waited behind the run asking the question is not its answer: it was sent before the question', () => { + const origin = 'Во вторник отмени английский'; + say(origin); + say('Да'); // sent while the run for the line above was still working + toolTurn( + 'c1', + 'ask_user', + { question: 'Удалить «Английский» во вторник?', options: ['Да', 'Нет'] }, + 'Вопрос отправлен.', + ); + const queued = history.getRecent(USER, 30).findLast((row) => row.content === 'Да')!; + expect(history.placeUserTurnAfterLatest(queued.id, USER)).toBe(true); + + const rows = history.getRecent(USER, 30); + expect(rows.at(-1)?.content).toBe('Да'); // read last, after the question + expect(resolveTurnDayReferences('Да', rows, SUNDAY_NIGHT, TZ)).toBeNull(); + }); + + test('a time-only message that waited behind a run which then asked in plain text is not the answer to it', () => { + say('в 23:30'); + const alone = resolveTurnDayReferences('в 23:30', history.getRecent(USER, 30), SUNDAY_NIGHT, TZ); + expect(alone).not.toBeNull(); // a fresh time-only message means today while that time is ahead + save('assistant', 'Во сколько?'); // the earlier run's question, saved after the message was sent + const queued = history.getRecent(USER, 30).findLast((row) => row.content === 'в 23:30')!; + expect(history.placeUserTurnAfterLatest(queued.id, USER)).toBe(true); + + const rows = history.getRecent(USER, 30); + expect(rows.at(-1)?.content).toBe('в 23:30'); + expect(resolveTurnDayReferences('в 23:30', rows, SUNDAY_NIGHT, TZ)).toEqual(alone); + }); + + test('a button answer that waited behind another run still answers the question asked before it was sent', () => { + say('Во вторник отмени английский'); + toolTurn( + 'c1', + 'ask_user', + { question: 'Удалить «Английский» во вторник?', options: ['Да', 'Нет'] }, + 'Вопрос отправлен.', + ); + say('Да'); // the tap + toolTurn('c2', 'search_events', { query: 'x' }, 'nothing'); // a run that was busy meanwhile wrote this + const tap = history.getRecent(USER, 30).findLast((row) => row.content === 'Да')!; + expect(history.placeUserTurnAfterLatest(tap.id, USER)).toBe(true); + + const references = resolveTurnDayReferences('Да', history.getRecent(USER, 30), SUNDAY_NIGHT, TZ); + expect(references?.allowedDates.has('2026-09-29')).toBe(true); + }); + test('a late "Да" confirms the Tuesday meant when the question was asked', async () => { // Asked on Sunday the 27th (Tuesday = the 29th), answered on Thursday 1 October, when // the coming Tuesday is already the 6th. diff --git a/test/services/ai/tool-handlers/events-duplicate-create.test.ts b/test/services/ai/tool-handlers/events-duplicate-create.test.ts index 731236b17..fdc17e1dc 100644 --- a/test/services/ai/tool-handlers/events-duplicate-create.test.ts +++ b/test/services/ai/tool-handlers/events-duplicate-create.test.ts @@ -210,14 +210,34 @@ describe('create_event repeated within minutes', () => { expect(rows()).toHaveLength(1); }); - test('recurring events are never matched, whichever side is recurring', async () => { + test('a one-off and a series never stand in for each other', async () => { await handleCreateEvent(ctx, { ...STANDUP, recurrence_rule: 'FREQ=WEEKLY' }); - await handleCreateEvent(laterRun(), { ...STANDUP, recurrence_rule: 'FREQ=WEEKLY' }); - await handleCreateEvent(laterRun(), { ...STANDUP }); - expect(rows()).toHaveLength(3); + const oneOff = await handleCreateEvent(laterRun(), { ...STANDUP }); + expect(oneOff.output).not.toContain('already exists'); // the series is not an earlier copy of the one-off + expect(rows()).toHaveLength(2); const once = await handleCreateEvent(laterRun(), { ...STANDUP }); - expect(once.output).toContain('already exists'); // the one-off from the line above, not the recurring ones - expect(rows()).toHaveLength(3); + expect(once.output).toContain('already exists'); // the one-off from the line above, not the series + expect(rows()).toHaveLength(2); + }); + + test('a repeated create of a series creates it once; another rule or start is another series', async () => { + const weekly = { ...STANDUP, recurrence_rule: 'FREQ=WEEKLY' }; + await handleCreateEvent(ctx, weekly); + const again = await handleCreateEvent(laterRun(), weekly); + expect(again.output).toContain('already exists'); + expect(rows()).toHaveLength(1); + + await handleCreateEvent(laterRun(), { ...STANDUP, recurrence_rule: 'FREQ=DAILY' }); + await handleCreateEvent(laterRun(), { + ...weekly, + start_at: '2030-03-05T11:00:00Z', + end_at: '2030-03-05T11:30:00Z', + }); + expect(rows()).toHaveLength(3); // a same-day series is not a conflict: two series on one day may be deliberate + + ageEvents(11); + await handleCreateEvent(laterRun(), weekly); + expect(rows()).toHaveLength(4); // not a repeat any more: ten minutes passed }); test('another owner’s identical event is not matched', async () => { From 1abbe94352e7623ae25128127f7c1de4c0ef5704 Mon Sep 17 00:00:00 2001 From: Alex Ultra <111590326+alex-mextner@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:53:57 +0100 Subject: [PATCH 3/4] fix(intent): re-measure the KPI gate after formatting intent-executor.ts and the engine digest it changes (Refs #929) --- .../20261102T094538Z-real-current.json | 12740 ++++++++++++++++ .../20261102T094731Z-synthetic-current.json | 12195 +++++++++++++++ docs/intents/kpi-gate.json | 6 +- src/services/intent/intent-executor.ts | 7 +- test/services/ai/chat-run-gate.test.ts | 7 +- 5 files changed, 24948 insertions(+), 7 deletions(-) create mode 100644 docs/intents/benchmarks/20261102T094538Z-real-current.json create mode 100644 docs/intents/benchmarks/20261102T094731Z-synthetic-current.json diff --git a/docs/intents/benchmarks/20261102T094538Z-real-current.json b/docs/intents/benchmarks/20261102T094538Z-real-current.json new file mode 100644 index 000000000..e23303983 --- /dev/null +++ b/docs/intents/benchmarks/20261102T094538Z-real-current.json @@ -0,0 +1,12740 @@ +{ + "schemaVersion": 2, + "label": "current", + "ruleSet": { + "fingerprint": "77224e790a7503b776e6d5650325890439081b0ba34c5aca99b9bd93afe6d007", + "rules": 54 + }, + "modelHistory": { + "version": 1, + "checksum": "923fbb5548f405252e9036ce60b86e5de33341ffe04317bb266c42b2e05ec47d" + }, + "dev": { + "overall": { + "conversations": 356, + "inScope": 83, + "aiIntentLike": 220, + "aiOther": 53, + "correct": 64, + "wrong": 0, + "missed": 19, + "falseMatch": 4, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 64, + "relabelled": 0, + "coverage": 0.7710843373493976, + "coverageOfAll": 0.1797752808988764, + "precision": 0.9411764705882353, + "intentLikeCoverage": 0.21122112211221122, + "rubricPass": 1, + "qualityCoverage": 0.7710843373493976, + "aiFallbackCorrect": 0.9853479853479854, + "execution": 1, + "intervals": { + "coverage": { + "k": 64, + "n": 83, + "rate": 0.7710843373493976, + "low": 0.6699193229025606, + "high": 0.848266352592755 + }, + "coverageOfAll": { + "k": 64, + "n": 356, + "rate": 0.1797752808988764, + "low": 0.14337117596598248, + "high": 0.22301645277690113 + }, + "precision": { + "k": 64, + "n": 68, + "rate": 0.9411764705882353, + "low": 0.8582831156689853, + "high": 0.9768892344603642 + }, + "rubricPass": { + "k": 64, + "n": 64, + "rate": 1, + "low": 0.9433759402071946, + "high": 1 + }, + "execution": { + "k": 64, + "n": 64, + "rate": 1, + "low": 0.9433759402071946, + "high": 1 + } + } + }, + "byFamily": { + "ai:bare_hour": { + "conversations": 15, + "inScope": 0, + "aiIntentLike": 15, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 15, + "rate": 0, + "low": 0, + "high": 0.20388330103584862 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 12, + "inScope": 0, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 12, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.24249400665524085 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 44, + "inScope": 0, + "aiIntentLike": 6, + "aiOther": 38, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 44, + "rate": 0, + "low": 0, + "high": 0.08029560375848066 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 118, + "inScope": 0, + "aiIntentLike": 111, + "aiOther": 7, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 118, + "rate": 0, + "low": 0, + "high": 0.03152833902249433 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 47, + "inScope": 0, + "aiIntentLike": 47, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 47, + "rate": 0, + "low": 0, + "high": 0.07555760416399632 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 5, + "inScope": 0, + "aiIntentLike": 5, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 5, + "rate": 0, + "low": 0, + "high": 0.43448246478317476 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 31, + "inScope": 0, + "aiIntentLike": 23, + "aiOther": 8, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 4, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": 0, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 0.8709677419354839, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 31, + "rate": 0, + "low": 0, + "high": 0.11025539546043596 + }, + "precision": { + "k": 0, + "n": 4, + "rate": 0, + "low": 0, + "high": 0.4898908364545973 + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "calc.evaluate": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "calendar.day": { + "conversations": 14, + "inScope": 14, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 13, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 13, + "relabelled": 0, + "coverage": 0.9285714285714286, + "coverageOfAll": 0.9285714285714286, + "precision": 1, + "intentLikeCoverage": 0.9285714285714286, + "rubricPass": 1, + "qualityCoverage": 0.9285714285714286, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 13, + "n": 14, + "rate": 0.9285714285714286, + "low": 0.6853129557584889, + "high": 0.9872777847521093 + }, + "coverageOfAll": { + "k": 13, + "n": 14, + "rate": 0.9285714285714286, + "low": 0.6853129557584889, + "high": 0.9872777847521093 + }, + "precision": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + }, + "execution": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + } + } + }, + "calendar.image": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 7, + "inScope": 7, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 7, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 7, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "coverageOfAll": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "precision": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "rubricPass": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "execution": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "event.create": { + "conversations": 40, + "inScope": 40, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 35, + "wrong": 0, + "missed": 5, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 35, + "relabelled": 0, + "coverage": 0.875, + "coverageOfAll": 0.875, + "precision": 1, + "intentLikeCoverage": 0.875, + "rubricPass": 1, + "qualityCoverage": 0.875, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 35, + "n": 40, + "rate": 0.875, + "low": 0.738878801611489, + "high": 0.945404997490546 + }, + "coverageOfAll": { + "k": 35, + "n": 40, + "rate": 0.875, + "low": 0.738878801611489, + "high": 0.945404997490546 + }, + "precision": { + "k": 35, + "n": 35, + "rate": 1, + "low": 0.9010990076755959, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 35, + "n": 35, + "rate": 1, + "low": 0.9010990076755959, + "high": 0.9999999999999999 + }, + "execution": { + "k": 35, + "n": 35, + "rate": 1, + "low": 0.9010990076755959, + "high": 0.9999999999999999 + } + } + }, + "event.create_invite": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.create_range": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.25, + "coverageOfAll": 0.25, + "precision": 1, + "intentLikeCoverage": 0.25, + "rubricPass": 1, + "qualityCoverage": 0.25, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 4, + "rate": 0.25, + "low": 0.04558726080970055, + "high": 0.6993581574175981 + }, + "coverageOfAll": { + "k": 1, + "n": 4, + "rate": 0.25, + "low": 0.04558726080970055, + "high": 0.6993581574175981 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.snooze": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "invite.send": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "invite.status": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "reminder.after": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "settings.view": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "time.now": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 356, + "inScope": 83, + "aiIntentLike": 220, + "aiOther": 53, + "correct": 64, + "wrong": 0, + "missed": 19, + "falseMatch": 4, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 64, + "relabelled": 0, + "coverage": 0.7710843373493976, + "coverageOfAll": 0.1797752808988764, + "precision": 0.9411764705882353, + "intentLikeCoverage": 0.21122112211221122, + "rubricPass": 1, + "qualityCoverage": 0.7710843373493976, + "aiFallbackCorrect": 0.9853479853479854, + "execution": 1, + "intervals": { + "coverage": { + "k": 64, + "n": 83, + "rate": 0.7710843373493976, + "low": 0.6699193229025606, + "high": 0.848266352592755 + }, + "coverageOfAll": { + "k": 64, + "n": 356, + "rate": 0.1797752808988764, + "low": 0.14337117596598248, + "high": 0.22301645277690113 + }, + "precision": { + "k": 64, + "n": 68, + "rate": 0.9411764705882353, + "low": 0.8582831156689853, + "high": 0.9768892344603642 + }, + "rubricPass": { + "k": 64, + "n": 64, + "rate": 1, + "low": 0.9433759402071946, + "high": 1 + }, + "execution": { + "k": 64, + "n": 64, + "rate": 1, + "low": 0.9433759402071946, + "high": 1 + } + } + } + }, + "byRule": { + "basis.event.create": 36, + "basis.calendar.day": 13, + "basis.calendar.period": 7, + "basis.contacts.list": 2, + "basis.event.delete": 1, + "basis.event.snooze": 1, + "basis.event.reschedule": 1, + "basis.calendar.when": 1, + "basis.calendar.image": 1, + "basis.invite.status": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 19, + "false_match:a rule answered an AI-correct message (unsupported)": 4 + }, + "rubricFindings": {} + }, + "heldout": { + "overall": { + "conversations": 183, + "inScope": 42, + "aiIntentLike": 115, + "aiOther": 26, + "correct": 36, + "wrong": 0, + "missed": 6, + "falseMatch": 1, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 36, + "relabelled": 0, + "coverage": 0.8571428571428571, + "coverageOfAll": 0.19672131147540983, + "precision": 0.972972972972973, + "intentLikeCoverage": 0.22929936305732485, + "rubricPass": 1, + "qualityCoverage": 0.8571428571428571, + "aiFallbackCorrect": 0.9929078014184397, + "execution": 1, + "intervals": { + "coverage": { + "k": 36, + "n": 42, + "rate": 0.8571428571428571, + "low": 0.7215891052555522, + "high": 0.9328403298403735 + }, + "coverageOfAll": { + "k": 36, + "n": 183, + "rate": 0.19672131147540983, + "low": 0.14561727285621828, + "high": 0.26029616466129113 + }, + "precision": { + "k": 36, + "n": 37, + "rate": 0.972972972972973, + "low": 0.861759301362695, + "high": 0.9952130255480596 + }, + "rubricPass": { + "k": 36, + "n": 36, + "rate": 1, + "low": 0.9035813714055363, + "high": 1 + }, + "execution": { + "k": 36, + "n": 36, + "rate": 1, + "low": 0.9035813714055363, + "high": 1 + } + } + }, + "byFamily": { + "ai:ambiguous_day": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 8, + "inScope": 0, + "aiIntentLike": 8, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 8, + "rate": 0, + "low": 0, + "high": 0.32440756488388023 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 5, + "inScope": 0, + "aiIntentLike": 5, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 5, + "rate": 0, + "low": 0, + "high": 0.43448246478317476 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 22, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 20, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 22, + "rate": 0, + "low": 0, + "high": 0.14865487460862087 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 67, + "inScope": 0, + "aiIntentLike": 65, + "aiOther": 2, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 67, + "rate": 0, + "low": 0, + "high": 0.05422613939130178 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 19, + "inScope": 0, + "aiIntentLike": 19, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 19, + "rate": 0, + "low": 0, + "high": 0.16817922405261626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 4, + "inScope": 0, + "aiIntentLike": 4, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 4, + "rate": 0, + "low": 0, + "high": 0.4898908364545973 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 14, + "inScope": 0, + "aiIntentLike": 10, + "aiOther": 4, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 1, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": 0, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 0.9285714285714286, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 14, + "rate": 0, + "low": 1.3877787807814457e-17, + "high": 0.2153108027376358 + }, + "precision": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.day": { + "conversations": 8, + "inScope": 8, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 6, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 6, + "relabelled": 0, + "coverage": 0.75, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.75, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 6, + "n": 8, + "rate": 0.75, + "low": 0.40927543031016883, + "high": 0.9285207872478909 + }, + "coverageOfAll": { + "k": 6, + "n": 8, + "rate": 0.75, + "low": 0.40927543031016883, + "high": 0.9285207872478909 + }, + "precision": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "rubricPass": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "execution": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "event.create": { + "conversations": 21, + "inScope": 21, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 19, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 19, + "relabelled": 0, + "coverage": 0.9047619047619048, + "coverageOfAll": 0.9047619047619048, + "precision": 1, + "intentLikeCoverage": 0.9047619047619048, + "rubricPass": 1, + "qualityCoverage": 0.9047619047619048, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 19, + "n": 21, + "rate": 0.9047619047619048, + "low": 0.7108586092608176, + "high": 0.9734812325619255 + }, + "coverageOfAll": { + "k": 19, + "n": 21, + "rate": 0.9047619047619048, + "low": 0.7108586092608176, + "high": 0.9734812325619255 + }, + "precision": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "rubricPass": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "execution": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + } + } + }, + "event.create_invite": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.create_range": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.delete": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 183, + "inScope": 42, + "aiIntentLike": 115, + "aiOther": 26, + "correct": 36, + "wrong": 0, + "missed": 6, + "falseMatch": 1, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 36, + "relabelled": 0, + "coverage": 0.8571428571428571, + "coverageOfAll": 0.19672131147540983, + "precision": 0.972972972972973, + "intentLikeCoverage": 0.22929936305732485, + "rubricPass": 1, + "qualityCoverage": 0.8571428571428571, + "aiFallbackCorrect": 0.9929078014184397, + "execution": 1, + "intervals": { + "coverage": { + "k": 36, + "n": 42, + "rate": 0.8571428571428571, + "low": 0.7215891052555522, + "high": 0.9328403298403735 + }, + "coverageOfAll": { + "k": 36, + "n": 183, + "rate": 0.19672131147540983, + "low": 0.14561727285621828, + "high": 0.26029616466129113 + }, + "precision": { + "k": 36, + "n": 37, + "rate": 0.972972972972973, + "low": 0.861759301362695, + "high": 0.9952130255480596 + }, + "rubricPass": { + "k": 36, + "n": 36, + "rate": 1, + "low": 0.9035813714055363, + "high": 1 + }, + "execution": { + "k": 36, + "n": 36, + "rate": 1, + "low": 0.9035813714055363, + "high": 1 + } + } + } + }, + "byRule": { + "basis.event.create": 19, + "basis.calendar.day": 6, + "basis.event.delete": 3, + "basis.contacts.list": 2, + "basis.calendar.period": 2, + "basis.bot.info": 1, + "basis.event.reschedule": 1, + "basis.calendar.image": 1, + "basis.invite.status": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 6, + "false_match:a rule answered an AI-correct message (unsupported)": 1 + }, + "rubricFindings": {} + }, + "all": { + "overall": { + "conversations": 539, + "inScope": 125, + "aiIntentLike": 335, + "aiOther": 79, + "correct": 100, + "wrong": 0, + "missed": 25, + "falseMatch": 5, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 100, + "relabelled": 0, + "coverage": 0.8, + "coverageOfAll": 0.18552875695732837, + "precision": 0.9523809523809523, + "intentLikeCoverage": 0.21739130434782608, + "rubricPass": 1, + "qualityCoverage": 0.8, + "aiFallbackCorrect": 0.9879227053140096, + "execution": 1, + "intervals": { + "coverage": { + "k": 100, + "n": 125, + "rate": 0.8, + "low": 0.7214100720202135, + "high": 0.8607006917547908 + }, + "coverageOfAll": { + "k": 100, + "n": 539, + "rate": 0.18552875695732837, + "low": 0.15497795005616277, + "high": 0.22053032313935406 + }, + "precision": { + "k": 100, + "n": 105, + "rate": 0.9523809523809523, + "low": 0.893338334167072, + "high": 0.9794908343256379 + }, + "rubricPass": { + "k": 100, + "n": 100, + "rate": 1, + "low": 0.9630065017930143, + "high": 1 + }, + "execution": { + "k": 100, + "n": 100, + "rate": 1, + "low": 0.9630065017930143, + "high": 1 + } + } + }, + "byFamily": { + "ai:ambiguous_day": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 23, + "inScope": 0, + "aiIntentLike": 23, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 23, + "rate": 0, + "low": 0, + "high": 0.14311661845042686 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 17, + "inScope": 0, + "aiIntentLike": 17, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 17, + "rate": 0, + "low": 0, + "high": 0.1843181350088518 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 66, + "inScope": 0, + "aiIntentLike": 8, + "aiOther": 58, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 66, + "rate": 0, + "low": 0, + "high": 0.05500255701354129 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 185, + "inScope": 0, + "aiIntentLike": 176, + "aiOther": 9, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 185, + "rate": 0, + "low": 0, + "high": 0.020342242877617298 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 66, + "inScope": 0, + "aiIntentLike": 66, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 66, + "rate": 0, + "low": 0, + "high": 0.05500255701354129 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 9, + "inScope": 0, + "aiIntentLike": 9, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 9, + "rate": 0, + "low": 0, + "high": 0.2991450484195441 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 45, + "inScope": 0, + "aiIntentLike": 33, + "aiOther": 12, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 5, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": 0, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 0.8888888888888888, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 45, + "rate": 0, + "low": 0, + "high": 0.07865159873288836 + }, + "precision": { + "k": 0, + "n": 5, + "rate": 0, + "low": 0, + "high": 0.43448246478317476 + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calc.evaluate": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "calendar.day": { + "conversations": 22, + "inScope": 22, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 19, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 19, + "relabelled": 0, + "coverage": 0.8636363636363636, + "coverageOfAll": 0.8636363636363636, + "precision": 1, + "intentLikeCoverage": 0.8636363636363636, + "rubricPass": 1, + "qualityCoverage": 0.8636363636363636, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 19, + "n": 22, + "rate": 0.8636363636363636, + "low": 0.6666501283342828, + "high": 0.9525099628594474 + }, + "coverageOfAll": { + "k": 19, + "n": 22, + "rate": 0.8636363636363636, + "low": 0.6666501283342828, + "high": 0.9525099628594474 + }, + "precision": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "rubricPass": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "execution": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 9, + "inScope": 9, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 9, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 9, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "coverageOfAll": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "precision": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "rubricPass": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "execution": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "event.create": { + "conversations": 61, + "inScope": 61, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 54, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 54, + "relabelled": 0, + "coverage": 0.8852459016393442, + "coverageOfAll": 0.8852459016393442, + "precision": 1, + "intentLikeCoverage": 0.8852459016393442, + "rubricPass": 1, + "qualityCoverage": 0.8852459016393442, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 54, + "n": 61, + "rate": 0.8852459016393442, + "low": 0.7815569152759893, + "high": 0.9432879733926779 + }, + "coverageOfAll": { + "k": 54, + "n": 61, + "rate": 0.8852459016393442, + "low": 0.7815569152759893, + "high": 0.9432879733926779 + }, + "precision": { + "k": 54, + "n": 54, + "rate": 1, + "low": 0.9335864119091037, + "high": 1 + }, + "rubricPass": { + "k": 54, + "n": 54, + "rate": 1, + "low": 0.9335864119091037, + "high": 1 + }, + "execution": { + "k": 54, + "n": 54, + "rate": 1, + "low": 0.9335864119091037, + "high": 1 + } + } + }, + "event.create_invite": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.create_range": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.3333333333333333, + "coverageOfAll": 0.3333333333333333, + "precision": 1, + "intentLikeCoverage": 0.3333333333333333, + "rubricPass": 1, + "qualityCoverage": 0.3333333333333333, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 3, + "rate": 0.3333333333333333, + "low": 0.06149194472039621, + "high": 0.7923403991979522 + }, + "coverageOfAll": { + "k": 1, + "n": 3, + "rate": 0.3333333333333333, + "low": 0.06149194472039621, + "high": 0.7923403991979522 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 7, + "inScope": 7, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 0.5714285714285714, + "coverageOfAll": 0.5714285714285714, + "precision": 1, + "intentLikeCoverage": 0.5714285714285714, + "rubricPass": 1, + "qualityCoverage": 0.5714285714285714, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 7, + "rate": 0.5714285714285714, + "low": 0.2504583645276572, + "high": 0.8417801447485302 + }, + "coverageOfAll": { + "k": 4, + "n": 7, + "rate": 0.5714285714285714, + "low": 0.2504583645276572, + "high": 0.8417801447485302 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 0.6666666666666666, + "coverageOfAll": 0.6666666666666666, + "precision": 1, + "intentLikeCoverage": 0.6666666666666666, + "rubricPass": 1, + "qualityCoverage": 0.6666666666666666, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "coverageOfAll": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "event.snooze": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "invite.send": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "invite.status": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "reminder.after": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "settings.view": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "time.now": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 539, + "inScope": 125, + "aiIntentLike": 335, + "aiOther": 79, + "correct": 100, + "wrong": 0, + "missed": 25, + "falseMatch": 5, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 100, + "relabelled": 0, + "coverage": 0.8, + "coverageOfAll": 0.18552875695732837, + "precision": 0.9523809523809523, + "intentLikeCoverage": 0.21739130434782608, + "rubricPass": 1, + "qualityCoverage": 0.8, + "aiFallbackCorrect": 0.9879227053140096, + "execution": 1, + "intervals": { + "coverage": { + "k": 100, + "n": 125, + "rate": 0.8, + "low": 0.7214100720202135, + "high": 0.8607006917547908 + }, + "coverageOfAll": { + "k": 100, + "n": 539, + "rate": 0.18552875695732837, + "low": 0.15497795005616277, + "high": 0.22053032313935406 + }, + "precision": { + "k": 100, + "n": 105, + "rate": 0.9523809523809523, + "low": 0.893338334167072, + "high": 0.9794908343256379 + }, + "rubricPass": { + "k": 100, + "n": 100, + "rate": 1, + "low": 0.9630065017930143, + "high": 1 + }, + "execution": { + "k": 100, + "n": 100, + "rate": 1, + "low": 0.9630065017930143, + "high": 1 + } + } + } + }, + "byRule": { + "basis.event.create": 55, + "basis.calendar.day": 19, + "basis.calendar.period": 9, + "basis.contacts.list": 4, + "basis.event.delete": 4, + "basis.event.reschedule": 2, + "basis.calendar.image": 2, + "basis.invite.status": 2, + "basis.event.snooze": 1, + "basis.calendar.when": 1, + "basis.bot.info": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 25, + "false_match:a rule answered an AI-correct message (unsupported)": 5 + }, + "rubricFindings": {} + }, + "matrix": { + "overall": { + "conversations": 18318, + "inScope": 3540, + "aiIntentLike": 11880, + "aiOther": 2898, + "correct": 3012, + "wrong": 0, + "missed": 528, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3012, + "relabelled": 156, + "coverage": 0.8508474576271187, + "coverageOfAll": 0.16442843105142482, + "precision": 1, + "intentLikeCoverage": 0.1953307392996109, + "rubricPass": 1, + "qualityCoverage": 0.8508474576271187, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 3012, + "n": 3540, + "rate": 0.8508474576271187, + "low": 0.8387322202759286, + "high": 0.8622020706239288 + }, + "coverageOfAll": { + "k": 3012, + "n": 18318, + "rate": 0.16442843105142482, + "low": 0.15913117670418145, + "high": 0.16986640098846403 + }, + "precision": { + "k": 3012, + "n": 3012, + "rate": 1, + "low": 0.9987262397996887, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 3012, + "n": 3012, + "rate": 1, + "low": 0.9987262397996887, + "high": 0.9999999999999999 + }, + "execution": { + "k": 3012, + "n": 3012, + "rate": 1, + "low": 0.9987262397996887, + "high": 0.9999999999999999 + } + } + }, + "byFamily": { + "ai:ambiguous_day": { + "conversations": 6, + "inScope": 0, + "aiIntentLike": 6, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 6, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.3903342879021653 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 840, + "inScope": 0, + "aiIntentLike": 840, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 840, + "rate": 0, + "low": 0, + "high": 0.004552346629262249 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 672, + "inScope": 0, + "aiIntentLike": 672, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 672, + "rate": 0, + "low": 0, + "high": 0.005683964442485163 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 2478, + "inScope": 0, + "aiIntentLike": 252, + "aiOther": 2226, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2478, + "rate": 0, + "low": 0, + "high": 0.0015478260333838913 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 6594, + "inScope": 0, + "aiIntentLike": 6216, + "aiOther": 378, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 6594, + "rate": 0, + "low": 0, + "high": 0.0005822296344448313 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 2436, + "inScope": 0, + "aiIntentLike": 2436, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2436, + "rate": 0, + "low": 0, + "high": 0.0015744706717750866 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 252, + "inScope": 0, + "aiIntentLike": 252, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 252, + "rate": 0, + "low": 8.673617379884035e-19, + "high": 0.015014997328429099 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 1344, + "inScope": 0, + "aiIntentLike": 1050, + "aiOther": 294, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1344, + "rate": 0, + "low": 0, + "high": 0.0028500821039109782 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 84, + "inScope": 84, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 84, + "rate": 0.5, + "low": 0.3954393376854136, + "high": 0.6045606623145864 + }, + "coverageOfAll": { + "k": 42, + "n": 84, + "rate": 0.5, + "low": 0.3954393376854136, + "high": 0.6045606623145864 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "calendar.day": { + "conversations": 924, + "inScope": 906, + "aiIntentLike": 18, + "aiOther": 0, + "correct": 786, + "wrong": 0, + "missed": 120, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 786, + "relabelled": 18, + "coverage": 0.8675496688741722, + "coverageOfAll": 0.8506493506493507, + "precision": 1, + "intentLikeCoverage": 0.8506493506493507, + "rubricPass": 1, + "qualityCoverage": 0.8675496688741722, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 786, + "n": 906, + "rate": 0.8675496688741722, + "low": 0.8439170642648448, + "high": 0.8880785966750806 + }, + "coverageOfAll": { + "k": 786, + "n": 924, + "rate": 0.8506493506493507, + "low": 0.82621714526898, + "high": 0.8721780318808118 + }, + "precision": { + "k": 786, + "n": 786, + "rate": 1, + "low": 0.9951364178496913, + "high": 1 + }, + "rubricPass": { + "k": 786, + "n": 786, + "rate": 1, + "low": 0.9951364178496913, + "high": 1 + }, + "execution": { + "k": 786, + "n": 786, + "rate": 1, + "low": 0.9951364178496913, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 84, + "inScope": 84, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 84, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 84, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "precision": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "execution": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + } + } + }, + "calendar.period": { + "conversations": 378, + "inScope": 378, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 378, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 378, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 378, + "n": 378, + "rate": 1, + "low": 0.9899396497369397, + "high": 1 + }, + "coverageOfAll": { + "k": 378, + "n": 378, + "rate": 1, + "low": 0.9899396497369397, + "high": 1 + }, + "precision": { + "k": 378, + "n": 378, + "rate": 1, + "low": 0.9899396497369397, + "high": 1 + }, + "rubricPass": { + "k": 378, + "n": 378, + "rate": 1, + "low": 0.9899396497369397, + "high": 1 + }, + "execution": { + "k": 378, + "n": 378, + "rate": 1, + "low": 0.9899396497369397, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 84, + "inScope": 84, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 84, + "rate": 0.5, + "low": 0.3954393376854136, + "high": 0.6045606623145864 + }, + "coverageOfAll": { + "k": 42, + "n": 84, + "rate": 0.5, + "low": 0.3954393376854136, + "high": 0.6045606623145864 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "contacts.list": { + "conversations": 168, + "inScope": 168, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 168, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 168, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "coverageOfAll": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "precision": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "rubricPass": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "execution": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + } + } + }, + "event.create": { + "conversations": 1344, + "inScope": 1236, + "aiIntentLike": 108, + "aiOther": 0, + "correct": 1152, + "wrong": 0, + "missed": 84, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1152, + "relabelled": 108, + "coverage": 0.9320388349514563, + "coverageOfAll": 0.8571428571428571, + "precision": 1, + "intentLikeCoverage": 0.8571428571428571, + "rubricPass": 1, + "qualityCoverage": 0.9320388349514563, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 1152, + "n": 1236, + "rate": 0.9320388349514563, + "low": 0.9166272501368763, + "high": 0.9447732074795795 + }, + "coverageOfAll": { + "k": 1152, + "n": 1344, + "rate": 0.8571428571428571, + "low": 0.8374159958727159, + "high": 0.8748339454816334 + }, + "precision": { + "k": 1152, + "n": 1152, + "rate": 1, + "low": 0.9966764829281919, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 1152, + "n": 1152, + "rate": 1, + "low": 0.9966764829281919, + "high": 0.9999999999999999 + }, + "execution": { + "k": 1152, + "n": 1152, + "rate": 1, + "low": 0.9966764829281919, + "high": 0.9999999999999999 + } + } + }, + "event.create_invite": { + "conversations": 84, + "inScope": 72, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 72, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 12, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 72, + "rate": 0, + "low": 0, + "high": 0.05065117259645781 + }, + "coverageOfAll": { + "k": 0, + "n": 84, + "rate": 0, + "low": 0, + "high": 0.04373172841465989 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.create_range": { + "conversations": 84, + "inScope": 72, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 36, + "wrong": 0, + "missed": 36, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 36, + "relabelled": 12, + "coverage": 0.5, + "coverageOfAll": 0.42857142857142855, + "precision": 1, + "intentLikeCoverage": 0.42857142857142855, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 36, + "n": 72, + "rate": 0.5, + "low": 0.3874709230948976, + "high": 0.6125290769051024 + }, + "coverageOfAll": { + "k": 36, + "n": 84, + "rate": 0.42857142857142855, + "low": 0.3281597743735877, + "high": 0.5352304725427922 + }, + "precision": { + "k": 36, + "n": 36, + "rate": 1, + "low": 0.9035813714055363, + "high": 1 + }, + "rubricPass": { + "k": 36, + "n": 36, + "rate": 1, + "low": 0.9035813714055363, + "high": 1 + }, + "execution": { + "k": 36, + "n": 36, + "rate": 1, + "low": 0.9035813714055363, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 168, + "inScope": 168, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 168, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 168, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "coverageOfAll": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "precision": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "rubricPass": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + }, + "execution": { + "k": 168, + "n": 168, + "rate": 1, + "low": 0.9776453316501321, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 84, + "inScope": 78, + "aiIntentLike": 6, + "aiOther": 0, + "correct": 72, + "wrong": 0, + "missed": 6, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 72, + "relabelled": 6, + "coverage": 0.9230769230769231, + "coverageOfAll": 0.8571428571428571, + "precision": 1, + "intentLikeCoverage": 0.8571428571428571, + "rubricPass": 1, + "qualityCoverage": 0.9230769230769231, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 72, + "n": 78, + "rate": 0.9230769230769231, + "low": 0.8421677019381915, + "high": 0.9642695359976443 + }, + "coverageOfAll": { + "k": 72, + "n": 84, + "rate": 0.8571428571428571, + "low": 0.7666989708237691, + "high": 0.9163497945943307 + }, + "precision": { + "k": 72, + "n": 72, + "rate": 1, + "low": 0.9493488274035422, + "high": 1 + }, + "rubricPass": { + "k": 72, + "n": 72, + "rate": 1, + "low": 0.9493488274035422, + "high": 1 + }, + "execution": { + "k": 72, + "n": 72, + "rate": 1, + "low": 0.9493488274035422, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 84, + "inScope": 84, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 84, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 84, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "precision": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + }, + "execution": { + "k": 84, + "n": 84, + "rate": 1, + "low": 0.95626827158534, + "high": 0.9999999999999999 + } + } + }, + "reminder.after": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "coverageOfAll": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "settings.view": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "coverageOfAll": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "time.now": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "coverageOfAll": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 18318, + "inScope": 3540, + "aiIntentLike": 11880, + "aiOther": 2898, + "correct": 3012, + "wrong": 0, + "missed": 528, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3012, + "relabelled": 156, + "coverage": 0.8508474576271187, + "coverageOfAll": 0.16442843105142482, + "precision": 1, + "intentLikeCoverage": 0.1953307392996109, + "rubricPass": 1, + "qualityCoverage": 0.8508474576271187, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 3012, + "n": 3540, + "rate": 0.8508474576271187, + "low": 0.8387322202759286, + "high": 0.8622020706239288 + }, + "coverageOfAll": { + "k": 3012, + "n": 18318, + "rate": 0.16442843105142482, + "low": 0.15913117670418145, + "high": 0.16986640098846403 + }, + "precision": { + "k": 3012, + "n": 3012, + "rate": 1, + "low": 0.9987262397996887, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 3012, + "n": 3012, + "rate": 1, + "low": 0.9987262397996887, + "high": 0.9999999999999999 + }, + "execution": { + "k": 3012, + "n": 3012, + "rate": 1, + "low": 0.9987262397996887, + "high": 0.9999999999999999 + } + } + } + }, + "byRule": { + "basis.event.create": 1188, + "basis.calendar.day": 786, + "basis.calendar.period": 378, + "basis.contacts.list": 168, + "basis.event.delete": 168, + "basis.calendar.image": 84, + "basis.invite.status": 84, + "basis.event.reschedule": 72, + "basis.calendar.when": 42, + "basis.bot.info": 42 + }, + "failureClasses": { + "missed:fell_through_to_ai": 528 + }, + "rubricFindings": {}, + "cells": 18318, + "anchors": 11 + }, + "real": { + "confirmed": { + "dev": { + "overall": { + "conversations": 326, + "inScope": 74, + "aiIntentLike": 203, + "aiOther": 49, + "correct": 62, + "wrong": 0, + "missed": 12, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 62, + "relabelled": 0, + "coverage": 0.8378378378378378, + "coverageOfAll": 0.1901840490797546, + "precision": 1, + "intentLikeCoverage": 0.22382671480144403, + "rubricPass": 1, + "qualityCoverage": 0.8378378378378378, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 62, + "n": 74, + "rate": 0.8378378378378378, + "low": 0.7376018659372965, + "high": 0.9047293642002805 + }, + "coverageOfAll": { + "k": 62, + "n": 326, + "rate": 0.1901840490797546, + "low": 0.15128668659976197, + "high": 0.2362978798697622 + }, + "precision": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + }, + "execution": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + } + } + }, + "byFamily": { + "ai:bare_hour": { + "conversations": 14, + "inScope": 0, + "aiIntentLike": 14, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 14, + "rate": 0, + "low": 1.3877787807814457e-17, + "high": 0.2153108027376358 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 12, + "inScope": 0, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 12, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.24249400665524085 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 44, + "inScope": 0, + "aiIntentLike": 6, + "aiOther": 38, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 44, + "rate": 0, + "low": 0, + "high": 0.08029560375848066 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 105, + "inScope": 0, + "aiIntentLike": 98, + "aiOther": 7, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 105, + "rate": 0, + "low": 0, + "high": 0.035294076929109894 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 46, + "inScope": 0, + "aiIntentLike": 46, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 46, + "rate": 0, + "low": 0, + "high": 0.07707356308558039 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 5, + "inScope": 0, + "aiIntentLike": 5, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 5, + "rate": 0, + "low": 0, + "high": 0.43448246478317476 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 25, + "inScope": 0, + "aiIntentLike": 21, + "aiOther": 4, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 25, + "rate": 0, + "low": 1.3877787807814457e-17, + "high": 0.13319225093904846 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "calendar.day": { + "conversations": 14, + "inScope": 14, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 13, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 13, + "relabelled": 0, + "coverage": 0.9285714285714286, + "coverageOfAll": 0.9285714285714286, + "precision": 1, + "intentLikeCoverage": 0.9285714285714286, + "rubricPass": 1, + "qualityCoverage": 0.9285714285714286, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 13, + "n": 14, + "rate": 0.9285714285714286, + "low": 0.6853129557584889, + "high": 0.9872777847521093 + }, + "coverageOfAll": { + "k": 13, + "n": 14, + "rate": 0.9285714285714286, + "low": 0.6853129557584889, + "high": 0.9872777847521093 + }, + "precision": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + }, + "execution": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + } + } + }, + "calendar.image": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 7, + "inScope": 7, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 7, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 7, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "coverageOfAll": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "precision": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "rubricPass": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "execution": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "event.create": { + "conversations": 36, + "inScope": 36, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 34, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 34, + "relabelled": 0, + "coverage": 0.9444444444444444, + "coverageOfAll": 0.9444444444444444, + "precision": 1, + "intentLikeCoverage": 0.9444444444444444, + "rubricPass": 1, + "qualityCoverage": 0.9444444444444444, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 34, + "n": 36, + "rate": 0.9444444444444444, + "low": 0.8185534279135275, + "high": 0.9846300133358382 + }, + "coverageOfAll": { + "k": 34, + "n": 36, + "rate": 0.9444444444444444, + "low": 0.8185534279135275, + "high": 0.9846300133358382 + }, + "precision": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + }, + "execution": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + } + } + }, + "event.create_invite": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.create_range": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "reminder.after": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "settings.view": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "time.now": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 326, + "inScope": 74, + "aiIntentLike": 203, + "aiOther": 49, + "correct": 62, + "wrong": 0, + "missed": 12, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 62, + "relabelled": 0, + "coverage": 0.8378378378378378, + "coverageOfAll": 0.1901840490797546, + "precision": 1, + "intentLikeCoverage": 0.22382671480144403, + "rubricPass": 1, + "qualityCoverage": 0.8378378378378378, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 62, + "n": 74, + "rate": 0.8378378378378378, + "low": 0.7376018659372965, + "high": 0.9047293642002805 + }, + "coverageOfAll": { + "k": 62, + "n": 326, + "rate": 0.1901840490797546, + "low": 0.15128668659976197, + "high": 0.2362978798697622 + }, + "precision": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + }, + "execution": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + } + } + } + }, + "byRule": { + "basis.event.create": 35, + "basis.calendar.day": 13, + "basis.calendar.period": 7, + "basis.contacts.list": 2, + "basis.event.delete": 1, + "basis.event.reschedule": 1, + "basis.calendar.when": 1, + "basis.calendar.image": 1, + "basis.invite.status": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 12 + }, + "rubricFindings": {} + }, + "heldout": { + "overall": { + "conversations": 170, + "inScope": 38, + "aiIntentLike": 112, + "aiOther": 20, + "correct": 34, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 34, + "relabelled": 0, + "coverage": 0.8947368421052632, + "coverageOfAll": 0.2, + "precision": 1, + "intentLikeCoverage": 0.22666666666666666, + "rubricPass": 1, + "qualityCoverage": 0.8947368421052632, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 34, + "n": 38, + "rate": 0.8947368421052632, + "low": 0.7586947827711269, + "high": 0.9582974265244647 + }, + "coverageOfAll": { + "k": 34, + "n": 170, + "rate": 0.2, + "low": 0.1467999521507426, + "high": 0.26645853813145065 + }, + "precision": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + }, + "execution": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + } + } + }, + "byFamily": { + "ai:ambiguous_day": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 7, + "inScope": 0, + "aiIntentLike": 7, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 7, + "rate": 0, + "low": 5.551115123125783e-17, + "high": 0.35433043506668743 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 5, + "inScope": 0, + "aiIntentLike": 5, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 5, + "rate": 0, + "low": 0, + "high": 0.43448246478317476 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 17, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 15, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 17, + "rate": 0, + "low": 0, + "high": 0.1843181350088518 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 65, + "inScope": 0, + "aiIntentLike": 63, + "aiOther": 2, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 65, + "rate": 0, + "low": 0, + "high": 0.05580153132285689 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 19, + "inScope": 0, + "aiIntentLike": 19, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 19, + "rate": 0, + "low": 0, + "high": 0.16817922405261626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 4, + "inScope": 0, + "aiIntentLike": 4, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 4, + "rate": 0, + "low": 0, + "high": 0.4898908364545973 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 13, + "inScope": 0, + "aiIntentLike": 10, + "aiOther": 3, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 13, + "rate": 0, + "low": 0, + "high": 0.22809537235419838 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.day": { + "conversations": 8, + "inScope": 8, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 6, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 6, + "relabelled": 0, + "coverage": 0.75, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.75, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 6, + "n": 8, + "rate": 0.75, + "low": 0.40927543031016883, + "high": 0.9285207872478909 + }, + "coverageOfAll": { + "k": 6, + "n": 8, + "rate": 0.75, + "low": 0.40927543031016883, + "high": 0.9285207872478909 + }, + "precision": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "rubricPass": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "execution": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "event.create": { + "conversations": 17, + "inScope": 17, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 17, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 17, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 17, + "n": 17, + "rate": 1, + "low": 0.8156818649911483, + "high": 1 + }, + "coverageOfAll": { + "k": 17, + "n": 17, + "rate": 1, + "low": 0.8156818649911483, + "high": 1 + }, + "precision": { + "k": 17, + "n": 17, + "rate": 1, + "low": 0.8156818649911483, + "high": 1 + }, + "rubricPass": { + "k": 17, + "n": 17, + "rate": 1, + "low": 0.8156818649911483, + "high": 1 + }, + "execution": { + "k": 17, + "n": 17, + "rate": 1, + "low": 0.8156818649911483, + "high": 1 + } + } + }, + "event.create_invite": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.create_range": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.delete": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 170, + "inScope": 38, + "aiIntentLike": 112, + "aiOther": 20, + "correct": 34, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 34, + "relabelled": 0, + "coverage": 0.8947368421052632, + "coverageOfAll": 0.2, + "precision": 1, + "intentLikeCoverage": 0.22666666666666666, + "rubricPass": 1, + "qualityCoverage": 0.8947368421052632, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 34, + "n": 38, + "rate": 0.8947368421052632, + "low": 0.7586947827711269, + "high": 0.9582974265244647 + }, + "coverageOfAll": { + "k": 34, + "n": 170, + "rate": 0.2, + "low": 0.1467999521507426, + "high": 0.26645853813145065 + }, + "precision": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + }, + "execution": { + "k": 34, + "n": 34, + "rate": 1, + "low": 0.8984854458466761, + "high": 0.9999999999999999 + } + } + } + }, + "byRule": { + "basis.event.create": 17, + "basis.calendar.day": 6, + "basis.event.delete": 3, + "basis.contacts.list": 2, + "basis.calendar.period": 2, + "basis.bot.info": 1, + "basis.event.reschedule": 1, + "basis.calendar.image": 1, + "basis.invite.status": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 4 + }, + "rubricFindings": {} + }, + "all": { + "overall": { + "conversations": 496, + "inScope": 112, + "aiIntentLike": 315, + "aiOther": 69, + "correct": 96, + "wrong": 0, + "missed": 16, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 96, + "relabelled": 0, + "coverage": 0.8571428571428571, + "coverageOfAll": 0.1935483870967742, + "precision": 1, + "intentLikeCoverage": 0.22482435597189696, + "rubricPass": 1, + "qualityCoverage": 0.8571428571428571, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 96, + "n": 112, + "rate": 0.8571428571428571, + "low": 0.7804856487003439, + "high": 0.9101133891089818 + }, + "coverageOfAll": { + "k": 96, + "n": 496, + "rate": 0.1935483870967742, + "low": 0.16118853653117643, + "high": 0.23061861624633545 + }, + "precision": { + "k": 96, + "n": 96, + "rate": 1, + "low": 0.9615244121423243, + "high": 1 + }, + "rubricPass": { + "k": 96, + "n": 96, + "rate": 1, + "low": 0.9615244121423243, + "high": 1 + }, + "execution": { + "k": 96, + "n": 96, + "rate": 1, + "low": 0.9615244121423243, + "high": 1 + } + } + }, + "byFamily": { + "ai:ambiguous_day": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 21, + "inScope": 0, + "aiIntentLike": 21, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 21, + "rate": 0, + "low": 0, + "high": 0.154639018924847 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 17, + "inScope": 0, + "aiIntentLike": 17, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 17, + "rate": 0, + "low": 0, + "high": 0.1843181350088518 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 61, + "inScope": 0, + "aiIntentLike": 8, + "aiOther": 53, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 61, + "rate": 0, + "low": 0, + "high": 0.05924386789811283 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 170, + "inScope": 0, + "aiIntentLike": 161, + "aiOther": 9, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 170, + "rate": 0, + "low": 0, + "high": 0.022097483803655452 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 65, + "inScope": 0, + "aiIntentLike": 65, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 65, + "rate": 0, + "low": 0, + "high": 0.05580153132285689 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 9, + "inScope": 0, + "aiIntentLike": 9, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 9, + "rate": 0, + "low": 0, + "high": 0.2991450484195441 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 38, + "inScope": 0, + "aiIntentLike": 31, + "aiOther": 7, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 38, + "rate": 0, + "low": 0, + "high": 0.09180986822558396 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.day": { + "conversations": 22, + "inScope": 22, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 19, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 19, + "relabelled": 0, + "coverage": 0.8636363636363636, + "coverageOfAll": 0.8636363636363636, + "precision": 1, + "intentLikeCoverage": 0.8636363636363636, + "rubricPass": 1, + "qualityCoverage": 0.8636363636363636, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 19, + "n": 22, + "rate": 0.8636363636363636, + "low": 0.6666501283342828, + "high": 0.9525099628594474 + }, + "coverageOfAll": { + "k": 19, + "n": 22, + "rate": 0.8636363636363636, + "low": 0.6666501283342828, + "high": 0.9525099628594474 + }, + "precision": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "rubricPass": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "execution": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 9, + "inScope": 9, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 9, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 9, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "coverageOfAll": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "precision": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "rubricPass": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "execution": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "event.create": { + "conversations": 53, + "inScope": 53, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 51, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 51, + "relabelled": 0, + "coverage": 0.9622641509433962, + "coverageOfAll": 0.9622641509433962, + "precision": 1, + "intentLikeCoverage": 0.9622641509433962, + "rubricPass": 1, + "qualityCoverage": 0.9622641509433962, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 51, + "n": 53, + "rate": 0.9622641509433962, + "low": 0.8724571280258164, + "high": 0.9895897127241731 + }, + "coverageOfAll": { + "k": 51, + "n": 53, + "rate": 0.9622641509433962, + "low": 0.8724571280258164, + "high": 0.9895897127241731 + }, + "precision": { + "k": 51, + "n": 51, + "rate": 1, + "low": 0.9299533801014687, + "high": 1 + }, + "rubricPass": { + "k": 51, + "n": 51, + "rate": 1, + "low": 0.9299533801014687, + "high": 1 + }, + "execution": { + "k": 51, + "n": 51, + "rate": 1, + "low": 0.9299533801014687, + "high": 1 + } + } + }, + "event.create_invite": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "event.create_range": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.3333333333333333, + "coverageOfAll": 0.3333333333333333, + "precision": 1, + "intentLikeCoverage": 0.3333333333333333, + "rubricPass": 1, + "qualityCoverage": 0.3333333333333333, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 3, + "rate": 0.3333333333333333, + "low": 0.06149194472039621, + "high": 0.7923403991979522 + }, + "coverageOfAll": { + "k": 1, + "n": 3, + "rate": 0.3333333333333333, + "low": 0.06149194472039621, + "high": 0.7923403991979522 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 5, + "inScope": 5, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 0.8, + "coverageOfAll": 0.8, + "precision": 1, + "intentLikeCoverage": 0.8, + "rubricPass": 1, + "qualityCoverage": 0.8, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 5, + "rate": 0.8, + "low": 0.37553462976252533, + "high": 0.9637758913675698 + }, + "coverageOfAll": { + "k": 4, + "n": 5, + "rate": 0.8, + "low": 0.37553462976252533, + "high": 0.9637758913675698 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 0.6666666666666666, + "coverageOfAll": 0.6666666666666666, + "precision": 1, + "intentLikeCoverage": 0.6666666666666666, + "rubricPass": 1, + "qualityCoverage": 0.6666666666666666, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "coverageOfAll": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "reminder.after": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "settings.view": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "time.now": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 496, + "inScope": 112, + "aiIntentLike": 315, + "aiOther": 69, + "correct": 96, + "wrong": 0, + "missed": 16, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 96, + "relabelled": 0, + "coverage": 0.8571428571428571, + "coverageOfAll": 0.1935483870967742, + "precision": 1, + "intentLikeCoverage": 0.22482435597189696, + "rubricPass": 1, + "qualityCoverage": 0.8571428571428571, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 96, + "n": 112, + "rate": 0.8571428571428571, + "low": 0.7804856487003439, + "high": 0.9101133891089818 + }, + "coverageOfAll": { + "k": 96, + "n": 496, + "rate": 0.1935483870967742, + "low": 0.16118853653117643, + "high": 0.23061861624633545 + }, + "precision": { + "k": 96, + "n": 96, + "rate": 1, + "low": 0.9615244121423243, + "high": 1 + }, + "rubricPass": { + "k": 96, + "n": 96, + "rate": 1, + "low": 0.9615244121423243, + "high": 1 + }, + "execution": { + "k": 96, + "n": 96, + "rate": 1, + "low": 0.9615244121423243, + "high": 1 + } + } + } + }, + "byRule": { + "basis.event.create": 52, + "basis.calendar.day": 19, + "basis.calendar.period": 9, + "basis.contacts.list": 4, + "basis.event.delete": 4, + "basis.event.reschedule": 2, + "basis.calendar.image": 2, + "basis.invite.status": 2, + "basis.calendar.when": 1, + "basis.bot.info": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 16 + }, + "rubricFindings": {} + } + }, + "unclear": { + "dev": { + "conversations": 30, + "inScope": 9, + "aiIntentLike": 17, + "aiOther": 4, + "correct": 2, + "wrong": 0, + "missed": 7, + "falseMatch": 4, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 0.2222222222222222, + "coverageOfAll": 0.06666666666666667, + "precision": 0.3333333333333333, + "intentLikeCoverage": 0.07692307692307693, + "rubricPass": 1, + "qualityCoverage": 0.2222222222222222, + "aiFallbackCorrect": 0.8095238095238095, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 9, + "rate": 0.2222222222222222, + "low": 0.06322510711784671, + "high": 0.5474110308930111 + }, + "coverageOfAll": { + "k": 2, + "n": 30, + "rate": 0.06666666666666667, + "low": 0.018477023791270378, + "high": 0.2132345836261692 + }, + "precision": { + "k": 2, + "n": 6, + "rate": 0.3333333333333333, + "low": 0.09677141110578041, + "high": 0.700006684861608 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "heldout": { + "conversations": 13, + "inScope": 4, + "aiIntentLike": 3, + "aiOther": 6, + "correct": 2, + "wrong": 0, + "missed": 2, + "falseMatch": 1, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.15384615384615385, + "precision": 0.6666666666666666, + "intentLikeCoverage": 0.2857142857142857, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": 0.8888888888888888, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 4, + "rate": 0.5, + "low": 0.15003898915214947, + "high": 0.8499610108478506 + }, + "coverageOfAll": { + "k": 2, + "n": 13, + "rate": 0.15384615384615385, + "low": 0.043258178358081184, + "high": 0.4223463101948254 + }, + "precision": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + } + }, + "byUser": { + "u01": { + "conversations": 29, + "inScope": 0, + "aiIntentLike": 27, + "aiOther": 2, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 29, + "rate": 0, + "low": 0, + "high": 0.11696979849974075 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u02": { + "conversations": 11, + "inScope": 2, + "aiIntentLike": 8, + "aiOther": 1, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 0.18181818181818182, + "precision": 1, + "intentLikeCoverage": 0.2, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 11, + "rate": 0.18181818181818182, + "low": 0.05136768974608513, + "high": 0.4769805619608442 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "u03": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 1, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u04": { + "conversations": 4, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 4, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 4, + "rate": 0, + "low": 0, + "high": 0.4898908364545973 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u05": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 1, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u06": { + "conversations": 11, + "inScope": 0, + "aiIntentLike": 11, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 11, + "rate": 0, + "low": 0, + "high": 0.2588329669680317 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u07": { + "conversations": 188, + "inScope": 26, + "aiIntentLike": 120, + "aiOther": 42, + "correct": 19, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 19, + "relabelled": 0, + "coverage": 0.7307692307692307, + "coverageOfAll": 0.10106382978723404, + "precision": 1, + "intentLikeCoverage": 0.13013698630136986, + "rubricPass": 1, + "qualityCoverage": 0.7307692307692307, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 19, + "n": 26, + "rate": 0.7307692307692307, + "low": 0.5391695962082381, + "high": 0.8629555167798162 + }, + "coverageOfAll": { + "k": 19, + "n": 188, + "rate": 0.10106382978723404, + "low": 0.06565857689789162, + "high": 0.1524457842652578 + }, + "precision": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "rubricPass": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "execution": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + } + } + }, + "u08": { + "conversations": 6, + "inScope": 0, + "aiIntentLike": 5, + "aiOther": 1, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 6, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.3903342879021653 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u09": { + "conversations": 218, + "inScope": 84, + "aiIntentLike": 120, + "aiOther": 14, + "correct": 75, + "wrong": 0, + "missed": 9, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 75, + "relabelled": 0, + "coverage": 0.8928571428571429, + "coverageOfAll": 0.3440366972477064, + "precision": 1, + "intentLikeCoverage": 0.36764705882352944, + "rubricPass": 1, + "qualityCoverage": 0.8928571428571429, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 75, + "n": 84, + "rate": 0.8928571428571429, + "low": 0.8087538548365549, + "high": 0.9425997871233551 + }, + "coverageOfAll": { + "k": 75, + "n": 218, + "rate": 0.3440366972477064, + "low": 0.28416629211048217, + "high": 0.4093084967703384 + }, + "precision": { + "k": 75, + "n": 75, + "rate": 1, + "low": 0.9512761575171942, + "high": 1 + }, + "rubricPass": { + "k": 75, + "n": 75, + "rate": 1, + "low": 0.9512761575171942, + "high": 1 + }, + "execution": { + "k": 75, + "n": 75, + "rate": 1, + "low": 0.9512761575171942, + "high": 1 + } + } + }, + "u10": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 1, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u11": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 1, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u12": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 1, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u13": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u14": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u15": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u16": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u17": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u18": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u19": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u20": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u21": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u22": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u23": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u24": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u25": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u26": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u27": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u28": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u29": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "u30": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + } + } + }, + "latency": { + "matcherMicrosP50": 40.76000000350177, + "matcherMicrosP95": 221.67100000660866, + "matcherMicrosMax": 2194.2980000021635, + "layerMsP50": 13.671277999999802, + "layerMsP95": 22.270618999999897, + "samples": 1617 + }, + "historyKind": "real", + "realHistory": { + "corpusChecksum": "2f793c2471dc077084a074d650ea29306a25a0998544b8174e9377d65a1d2de9", + "labelsChecksum": "19ee899c60362252cd89a08b71fc5621f417f2ece970dbf58222792ffff05f7e", + "splitChecksum": "310707bb8c752d46692ceb9b50cc86bda217b681ac54867263ca3e8dda8354f0", + "excludedQaProbes": 13, + "heldoutShare": 0.3395176252319109 + }, + "generatedAt": "2026-11-02T09:45:38.404Z", + "source": { + "sha": "1e81955372ac0053687c140f95c68e03a94c2c64", + "dirty": true, + "engineDigest": "6935a57b68b65dd6f414637a134039f280b5c1ff1b2344d609fa21a6ecd1112b" + }, + "checksum": "984e27ac50edcf2ee66492ca50b02e0d6b0c5cad9484044a77029b3e8cbe3cb0" +} diff --git a/docs/intents/benchmarks/20261102T094731Z-synthetic-current.json b/docs/intents/benchmarks/20261102T094731Z-synthetic-current.json new file mode 100644 index 000000000..bf5244b33 --- /dev/null +++ b/docs/intents/benchmarks/20261102T094731Z-synthetic-current.json @@ -0,0 +1,12195 @@ +{ + "schemaVersion": 2, + "label": "current", + "ruleSet": { + "fingerprint": "77224e790a7503b776e6d5650325890439081b0ba34c5aca99b9bd93afe6d007", + "rules": 54 + }, + "modelHistory": { + "version": 1, + "checksum": "98447f88cd71fa57404321b1da6fad4df63a2784d8ace86de0aa3eb84d1f64f8" + }, + "dev": { + "overall": { + "conversations": 508, + "inScope": 358, + "aiIntentLike": 128, + "aiOther": 22, + "correct": 338, + "wrong": 0, + "missed": 20, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 336, + "relabelled": 1, + "coverage": 0.9441340782122905, + "coverageOfAll": 0.6653543307086615, + "precision": 1, + "intentLikeCoverage": 0.6954732510288066, + "rubricPass": 0.9940828402366864, + "qualityCoverage": 0.9385474860335196, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 338, + "n": 358, + "rate": 0.9441340782122905, + "low": 0.9152902660679959, + "high": 0.963547667640371 + }, + "coverageOfAll": { + "k": 338, + "n": 508, + "rate": 0.6653543307086615, + "low": 0.6232155603385077, + "high": 0.7050110752823731 + }, + "precision": { + "k": 338, + "n": 338, + "rate": 1, + "low": 0.9887624548703174, + "high": 1 + }, + "rubricPass": { + "k": 336, + "n": 338, + "rate": 0.9940828402366864, + "low": 0.9786853253275402, + "high": 0.9983757987159095 + }, + "execution": { + "k": 338, + "n": 338, + "rate": 1, + "low": 0.9887624548703174, + "high": 1 + } + } + }, + "byFamily": { + "ai:advice_or_analysis": { + "conversations": 12, + "inScope": 0, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 12, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.24249400665524085 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:ambiguous_day": { + "conversations": 6, + "inScope": 0, + "aiIntentLike": 6, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 6, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.3903342879021653 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 23, + "inScope": 0, + "aiIntentLike": 23, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 23, + "rate": 0, + "low": 0, + "high": 0.14311661845042686 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 11, + "inScope": 0, + "aiIntentLike": 11, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 11, + "rate": 0, + "low": 0, + "high": 0.2588329669680317 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 22, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 22, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 22, + "rate": 0, + "low": 0, + "high": 0.14865487460862087 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 21, + "inScope": 0, + "aiIntentLike": 21, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 21, + "rate": 0, + "low": 0, + "high": 0.154639018924847 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 13, + "inScope": 0, + "aiIntentLike": 13, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 13, + "rate": 0, + "low": 0, + "high": 0.22809537235419838 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:negation_or_quote": { + "conversations": 13, + "inScope": 0, + "aiIntentLike": 13, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 13, + "rate": 0, + "low": 0, + "high": 0.22809537235419838 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 8, + "inScope": 0, + "aiIntentLike": 8, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 8, + "rate": 0, + "low": 0, + "high": 0.32440756488388023 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 7, + "inScope": 0, + "aiIntentLike": 7, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 7, + "rate": 0, + "low": 5.551115123125783e-17, + "high": 0.35433043506668743 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 13, + "inScope": 0, + "aiIntentLike": 13, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 13, + "rate": 0, + "low": 0, + "high": 0.22809537235419838 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "calc.evaluate": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "calendar.count": { + "conversations": 5, + "inScope": 5, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "coverageOfAll": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "calendar.day": { + "conversations": 47, + "inScope": 47, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 46, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 46, + "relabelled": 0, + "coverage": 0.9787234042553191, + "coverageOfAll": 0.9787234042553191, + "precision": 1, + "intentLikeCoverage": 0.9787234042553191, + "rubricPass": 1, + "qualityCoverage": 0.9787234042553191, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 46, + "n": 47, + "rate": 0.9787234042553191, + "low": 0.8888701895837066, + "high": 0.996234231961403 + }, + "coverageOfAll": { + "k": 46, + "n": 47, + "rate": 0.9787234042553191, + "low": 0.8888701895837066, + "high": 0.996234231961403 + }, + "precision": { + "k": 46, + "n": 46, + "rate": 1, + "low": 0.9229264369144198, + "high": 1 + }, + "rubricPass": { + "k": 46, + "n": 46, + "rate": 1, + "low": 0.9229264369144198, + "high": 1 + }, + "execution": { + "k": 46, + "n": 46, + "rate": 1, + "low": 0.9229264369144198, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 0.5, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 4, + "rate": 0.5, + "low": 0.15003898915214947, + "high": 0.8499610108478506 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 14, + "inScope": 14, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 14, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 14, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "coverageOfAll": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "precision": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "rubricPass": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "execution": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + } + } + }, + "calendar.search": { + "conversations": 6, + "inScope": 6, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 6, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 6, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "coverageOfAll": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "precision": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "rubricPass": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "execution": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + } + } + }, + "calendar.upcoming": { + "conversations": 8, + "inScope": 8, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 8, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 8, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "coverageOfAll": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "precision": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "rubricPass": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "execution": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 10, + "inScope": 10, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 10, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 10, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "precision": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "execution": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + } + } + }, + "contacts.find": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 0.6666666666666666, + "coverageOfAll": 0.6666666666666666, + "precision": 1, + "intentLikeCoverage": 0.6666666666666666, + "rubricPass": 1, + "qualityCoverage": 0.6666666666666666, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "coverageOfAll": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "event.create": { + "conversations": 80, + "inScope": 80, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 73, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 73, + "relabelled": 0, + "coverage": 0.9125, + "coverageOfAll": 0.9125, + "precision": 1, + "intentLikeCoverage": 0.9125, + "rubricPass": 1, + "qualityCoverage": 0.9125, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 73, + "n": 80, + "rate": 0.9125, + "low": 0.8302320449152499, + "high": 0.9569679982179401 + }, + "coverageOfAll": { + "k": 73, + "n": 80, + "rate": 0.9125, + "low": 0.8302320449152499, + "high": 0.9569679982179401 + }, + "precision": { + "k": 73, + "n": 73, + "rate": 1, + "low": 0.9500079920442688, + "high": 0.9999999999999998 + }, + "rubricPass": { + "k": 73, + "n": 73, + "rate": 1, + "low": 0.9500079920442688, + "high": 0.9999999999999998 + }, + "execution": { + "k": 73, + "n": 73, + "rate": 1, + "low": 0.9500079920442688, + "high": 0.9999999999999998 + } + } + }, + "event.create_invite": { + "conversations": 26, + "inScope": 26, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 19, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 19, + "relabelled": 0, + "coverage": 0.7307692307692307, + "coverageOfAll": 0.7307692307692307, + "precision": 1, + "intentLikeCoverage": 0.7307692307692307, + "rubricPass": 1, + "qualityCoverage": 0.7307692307692307, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 19, + "n": 26, + "rate": 0.7307692307692307, + "low": 0.5391695962082381, + "high": 0.8629555167798162 + }, + "coverageOfAll": { + "k": 19, + "n": 26, + "rate": 0.7307692307692307, + "low": 0.5391695962082381, + "high": 0.8629555167798162 + }, + "precision": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "rubricPass": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "execution": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + } + } + }, + "event.create_range": { + "conversations": 10, + "inScope": 10, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 8, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 8, + "relabelled": 0, + "coverage": 0.8, + "coverageOfAll": 0.8, + "precision": 1, + "intentLikeCoverage": 0.8, + "rubricPass": 1, + "qualityCoverage": 0.8, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 8, + "n": 10, + "rate": 0.8, + "low": 0.49016247153664183, + "high": 0.9433178485456247 + }, + "coverageOfAll": { + "k": 8, + "n": 10, + "rate": 0.8, + "low": 0.49016247153664183, + "high": 0.9433178485456247 + }, + "precision": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "rubricPass": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "execution": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 32, + "inScope": 32, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 32, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 32, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 32, + "n": 32, + "rate": 1, + "low": 0.8928208017449293, + "high": 1 + }, + "coverageOfAll": { + "k": 32, + "n": 32, + "rate": 1, + "low": 0.8928208017449293, + "high": 1 + }, + "precision": { + "k": 32, + "n": 32, + "rate": 1, + "low": 0.8928208017449293, + "high": 1 + }, + "rubricPass": { + "k": 32, + "n": 32, + "rate": 1, + "low": 0.8928208017449293, + "high": 1 + }, + "execution": { + "k": 32, + "n": 32, + "rate": 1, + "low": 0.8928208017449293, + "high": 1 + } + } + }, + "event.hide": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "event.rename": { + "conversations": 5, + "inScope": 5, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "coverageOfAll": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 25, + "inScope": 24, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 24, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 24, + "relabelled": 1, + "coverage": 1, + "coverageOfAll": 0.96, + "precision": 1, + "intentLikeCoverage": 0.96, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 24, + "n": 24, + "rate": 1, + "low": 0.8620237953250198, + "high": 1 + }, + "coverageOfAll": { + "k": 24, + "n": 25, + "rate": 0.96, + "low": 0.8045593626380625, + "high": 0.9929037664980127 + }, + "precision": { + "k": 24, + "n": 24, + "rate": 1, + "low": 0.8620237953250198, + "high": 1 + }, + "rubricPass": { + "k": 24, + "n": 24, + "rate": 1, + "low": 0.8620237953250198, + "high": 1 + }, + "execution": { + "k": 24, + "n": 24, + "rate": 1, + "low": 0.8620237953250198, + "high": 1 + } + } + }, + "event.set_detail": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "event.show": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "event.snooze": { + "conversations": 6, + "inScope": 6, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 0.8333333333333334, + "coverageOfAll": 0.8333333333333334, + "precision": 1, + "intentLikeCoverage": 0.8333333333333334, + "rubricPass": 1, + "qualityCoverage": 0.8333333333333334, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 6, + "rate": 0.8333333333333334, + "low": 0.43649717781352987, + "high": 0.9699466302516935 + }, + "coverageOfAll": { + "k": 5, + "n": 6, + "rate": 0.8333333333333334, + "low": 0.43649717781352987, + "high": 0.9699466302516935 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "google.status": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "holidays.upcoming": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "invite.send": { + "conversations": 8, + "inScope": 8, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 8, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 8, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "coverageOfAll": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "precision": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "rubricPass": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "execution": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "reminder.after": { + "conversations": 10, + "inScope": 10, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 10, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 10, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "precision": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "execution": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + } + } + }, + "reminder.clear": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "reminder.list": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "reminder.set": { + "conversations": 7, + "inScope": 7, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 7, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 7, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "coverageOfAll": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "precision": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "rubricPass": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + }, + "execution": { + "k": 7, + "n": 7, + "rate": 1, + "low": 0.6456695649333126, + "high": 1 + } + } + }, + "settings.view": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "slots.check_time": { + "conversations": 9, + "inScope": 9, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 8, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 8, + "relabelled": 0, + "coverage": 0.8888888888888888, + "coverageOfAll": 0.8888888888888888, + "precision": 1, + "intentLikeCoverage": 0.8888888888888888, + "rubricPass": 1, + "qualityCoverage": 0.8888888888888888, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 8, + "n": 9, + "rate": 0.8888888888888888, + "low": 0.5650002944233439, + "high": 0.9801091123614551 + }, + "coverageOfAll": { + "k": 8, + "n": 9, + "rate": 0.8888888888888888, + "low": 0.5650002944233439, + "high": 0.9801091123614551 + }, + "precision": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "rubricPass": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "execution": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + } + } + }, + "slots.day": { + "conversations": 8, + "inScope": 8, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 8, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 8, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "coverageOfAll": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "precision": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "rubricPass": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "execution": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + } + } + }, + "time.now": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + } + }, + "byStyle": { + "typo": { + "conversations": 29, + "inScope": 28, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 25, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 25, + "relabelled": 0, + "coverage": 0.8928571428571429, + "coverageOfAll": 0.8620689655172413, + "precision": 1, + "intentLikeCoverage": 0.8620689655172413, + "rubricPass": 1, + "qualityCoverage": 0.8928571428571429, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 25, + "n": 28, + "rate": 0.8928571428571429, + "low": 0.7280414845132785, + "high": 0.9628816331707 + }, + "coverageOfAll": { + "k": 25, + "n": 29, + "rate": 0.8620689655172413, + "low": 0.6944100627964798, + "high": 0.9450256003588804 + }, + "precision": { + "k": 25, + "n": 25, + "rate": 1, + "low": 0.8668077490609515, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 25, + "n": 25, + "rate": 1, + "low": 0.8668077490609515, + "high": 0.9999999999999999 + }, + "execution": { + "k": 25, + "n": 25, + "rate": 1, + "low": 0.8668077490609515, + "high": 0.9999999999999999 + } + } + }, + "plain": { + "conversations": 131, + "inScope": 69, + "aiIntentLike": 49, + "aiOther": 13, + "correct": 62, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 62, + "relabelled": 0, + "coverage": 0.8985507246376812, + "coverageOfAll": 0.4732824427480916, + "precision": 1, + "intentLikeCoverage": 0.5254237288135594, + "rubricPass": 1, + "qualityCoverage": 0.8985507246376812, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 62, + "n": 69, + "rate": 0.8985507246376812, + "low": 0.8050811424505202, + "high": 0.9499833618915401 + }, + "coverageOfAll": { + "k": 62, + "n": 131, + "rate": 0.4732824427480916, + "low": 0.3897677343642162, + "high": 0.5583194484561042 + }, + "precision": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + }, + "execution": { + "k": 62, + "n": 62, + "rate": 1, + "low": 0.9416559278986275, + "high": 0.9999999999999999 + } + } + }, + "no_punct": { + "conversations": 155, + "inScope": 124, + "aiIntentLike": 26, + "aiOther": 5, + "correct": 121, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 119, + "relabelled": 0, + "coverage": 0.9758064516129032, + "coverageOfAll": 0.7806451612903226, + "precision": 1, + "intentLikeCoverage": 0.8066666666666666, + "rubricPass": 0.9834710743801653, + "qualityCoverage": 0.9596774193548387, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 121, + "n": 124, + "rate": 0.9758064516129032, + "low": 0.9312798718154085, + "high": 0.9917383813281402 + }, + "coverageOfAll": { + "k": 121, + "n": 155, + "rate": 0.7806451612903226, + "low": 0.7091483707205343, + "high": 0.8385675760223666 + }, + "precision": { + "k": 121, + "n": 121, + "rate": 1, + "low": 0.9692293020525218, + "high": 1 + }, + "rubricPass": { + "k": 119, + "n": 121, + "rate": 0.9834710743801653, + "low": 0.9417332575142658, + "high": 0.9954554064538752 + }, + "execution": { + "k": 121, + "n": 121, + "rate": 1, + "low": 0.9692293020525218, + "high": 1 + } + } + }, + "caps": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "trailing_punct": { + "conversations": 57, + "inScope": 43, + "aiIntentLike": 10, + "aiOther": 4, + "correct": 42, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 1, + "coverage": 0.9767441860465116, + "coverageOfAll": 0.7368421052631579, + "precision": 1, + "intentLikeCoverage": 0.7924528301886793, + "rubricPass": 1, + "qualityCoverage": 0.9767441860465116, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 43, + "rate": 0.9767441860465116, + "low": 0.8794100655939282, + "high": 0.9958829127256662 + }, + "coverageOfAll": { + "k": 42, + "n": 57, + "rate": 0.7368421052631579, + "low": 0.6102342073651618, + "high": 0.8335421340202692 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "terse": { + "conversations": 148, + "inScope": 104, + "aiIntentLike": 41, + "aiOther": 3, + "correct": 103, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 102, + "relabelled": 0, + "coverage": 0.9903846153846154, + "coverageOfAll": 0.6959459459459459, + "precision": 1, + "intentLikeCoverage": 0.7103448275862069, + "rubricPass": 0.9902912621359223, + "qualityCoverage": 0.9807692307692307, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 103, + "n": 104, + "rate": 0.9903846153846154, + "low": 0.9475322752920542, + "high": 0.9983006271503486 + }, + "coverageOfAll": { + "k": 103, + "n": 148, + "rate": 0.6959459459459459, + "low": 0.6176538029488693, + "high": 0.7643235598245665 + }, + "precision": { + "k": 103, + "n": 103, + "rate": 1, + "low": 0.964045241771352, + "high": 1 + }, + "rubricPass": { + "k": 102, + "n": 103, + "rate": 0.9902912621359223, + "low": 0.9470418081674632, + "high": 0.99828410852095 + }, + "execution": { + "k": 103, + "n": 103, + "rate": 1, + "low": 0.964045241771352, + "high": 1 + } + } + }, + "verbless": { + "conversations": 53, + "inScope": 50, + "aiIntentLike": 3, + "aiOther": 0, + "correct": 48, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 47, + "relabelled": 0, + "coverage": 0.96, + "coverageOfAll": 0.9056603773584906, + "precision": 1, + "intentLikeCoverage": 0.9056603773584906, + "rubricPass": 0.9791666666666666, + "qualityCoverage": 0.94, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 48, + "n": 50, + "rate": 0.96, + "low": 0.8653990931249297, + "high": 0.9889611156723802 + }, + "coverageOfAll": { + "k": 48, + "n": 53, + "rate": 0.9056603773584906, + "low": 0.7974627262370373, + "high": 0.9590273585027496 + }, + "precision": { + "k": 48, + "n": 48, + "rate": 1, + "low": 0.9258998703338824, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 47, + "n": 48, + "rate": 0.9791666666666666, + "low": 0.8910078200474814, + "high": 0.9963128890224893 + }, + "execution": { + "k": 48, + "n": 48, + "rate": 1, + "low": 0.9258998703338824, + "high": 0.9999999999999999 + } + } + }, + "slang": { + "conversations": 26, + "inScope": 25, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 23, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 22, + "relabelled": 0, + "coverage": 0.92, + "coverageOfAll": 0.8846153846153846, + "precision": 1, + "intentLikeCoverage": 0.8846153846153846, + "rubricPass": 0.9565217391304348, + "qualityCoverage": 0.88, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 23, + "n": 25, + "rate": 0.92, + "low": 0.7503389104960468, + "high": 0.9777795987151524 + }, + "coverageOfAll": { + "k": 23, + "n": 26, + "rate": 0.8846153846153846, + "low": 0.7102409678828635, + "high": 0.9599675537638938 + }, + "precision": { + "k": 23, + "n": 23, + "rate": 1, + "low": 0.8568833815495731, + "high": 1 + }, + "rubricPass": { + "k": 22, + "n": 23, + "rate": 0.9565217391304348, + "low": 0.7900884492974114, + "high": 0.9922833338565469 + }, + "execution": { + "k": 23, + "n": 23, + "rate": 1, + "low": 0.8568833815495731, + "high": 1 + } + } + }, + "polite": { + "conversations": 24, + "inScope": 23, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 19, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 19, + "relabelled": 0, + "coverage": 0.8260869565217391, + "coverageOfAll": 0.7916666666666666, + "precision": 1, + "intentLikeCoverage": 0.7916666666666666, + "rubricPass": 1, + "qualityCoverage": 0.8260869565217391, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 19, + "n": 23, + "rate": 0.8260869565217391, + "low": 0.6286235235977385, + "high": 0.9302134643693744 + }, + "coverageOfAll": { + "k": 19, + "n": 24, + "rate": 0.7916666666666666, + "low": 0.5952954674681133, + "high": 0.9075517464714814 + }, + "precision": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "rubricPass": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "execution": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + } + } + }, + "long": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "mixed_lang": { + "conversations": 12, + "inScope": 11, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 10, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 10, + "relabelled": 0, + "coverage": 0.9090909090909091, + "coverageOfAll": 0.8333333333333334, + "precision": 1, + "intentLikeCoverage": 0.8333333333333334, + "rubricPass": 1, + "qualityCoverage": 0.9090909090909091, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 10, + "n": 11, + "rate": 0.9090909090909091, + "low": 0.6226415635484043, + "high": 0.9837678271141151 + }, + "coverageOfAll": { + "k": 10, + "n": 12, + "rate": 0.8333333333333334, + "low": 0.5519691377470266, + "high": 0.9530348578161463 + }, + "precision": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "execution": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + } + } + } + }, + "byRule": { + "basis.event.create": 81, + "basis.calendar.day": 46, + "basis.event.delete": 32, + "basis.event.reschedule": 24, + "basis.event.create_invite": 19, + "basis.calendar.period": 14, + "basis.reminder.after": 10, + "basis.calendar.when": 10, + "basis.slots.check_time": 8, + "basis.slots.day": 8, + "basis.invite.send": 8, + "basis.calendar.upcoming": 7, + "basis.reminder.set": 7, + "basis.calendar.search": 6, + "basis.event.rename": 5, + "basis.event.snooze": 5, + "basis.calendar.count": 5, + "basis.time.now": 4, + "basis.invite.status": 4, + "basis.event.set_detail": 4, + "basis.calendar.image": 4, + "basis.event.show": 3, + "basis.event.hide": 3, + "basis.holidays.upcoming": 3, + "basis.calc.evaluate": 3, + "basis.contacts.find": 2, + "basis.settings.view": 2, + "basis.bot.info": 2, + "basis.reminder.list": 2, + "basis.google.status": 2, + "basis.reminder.clear": 2, + "basis.contacts.list": 2, + "basis.calendar.next": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 20 + }, + "rubricFindings": { + "RAW_ISO_DATE": 2 + } + }, + "heldout": { + "overall": { + "conversations": 247, + "inScope": 185, + "aiIntentLike": 57, + "aiOther": 5, + "correct": 153, + "wrong": 0, + "missed": 32, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 153, + "relabelled": 0, + "coverage": 0.827027027027027, + "coverageOfAll": 0.6194331983805668, + "precision": 1, + "intentLikeCoverage": 0.6322314049586777, + "rubricPass": 1, + "qualityCoverage": 0.827027027027027, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 153, + "n": 185, + "rate": 0.827027027027027, + "low": 0.7660212684254935, + "high": 0.8747278592059029 + }, + "coverageOfAll": { + "k": 153, + "n": 247, + "rate": 0.6194331983805668, + "low": 0.5574920153260318, + "high": 0.6777163121864239 + }, + "precision": { + "k": 153, + "n": 153, + "rate": 1, + "low": 0.9755073763686055, + "high": 1 + }, + "rubricPass": { + "k": 153, + "n": 153, + "rate": 1, + "low": 0.9755073763686055, + "high": 1 + }, + "execution": { + "k": 153, + "n": 153, + "rate": 1, + "low": 0.9755073763686055, + "high": 1 + } + } + }, + "byFamily": { + "ai:advice_or_analysis": { + "conversations": 3, + "inScope": 0, + "aiIntentLike": 3, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 3, + "rate": 0, + "low": 5.551115123125783e-17, + "high": 0.5614970317550454 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:ambiguous_day": { + "conversations": 1, + "inScope": 0, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 17, + "inScope": 0, + "aiIntentLike": 17, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 17, + "rate": 0, + "low": 0, + "high": 0.1843181350088518 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 2, + "inScope": 0, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 2, + "rate": 0, + "low": 0, + "high": 0.6576197724933469 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 5, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 5, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 5, + "rate": 0, + "low": 0, + "high": 0.43448246478317476 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 8, + "inScope": 0, + "aiIntentLike": 8, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 8, + "rate": 0, + "low": 0, + "high": 0.32440756488388023 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 7, + "inScope": 0, + "aiIntentLike": 7, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 7, + "rate": 0, + "low": 5.551115123125783e-17, + "high": 0.35433043506668743 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:negation_or_quote": { + "conversations": 7, + "inScope": 0, + "aiIntentLike": 7, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 7, + "rate": 0, + "low": 5.551115123125783e-17, + "high": 0.35433043506668743 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 4, + "inScope": 0, + "aiIntentLike": 4, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 4, + "rate": 0, + "low": 0, + "high": 0.4898908364545973 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 4, + "inScope": 0, + "aiIntentLike": 4, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 4, + "rate": 0, + "low": 0, + "high": 0.4898908364545973 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 4, + "inScope": 0, + "aiIntentLike": 4, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 4, + "rate": 0, + "low": 0, + "high": 0.4898908364545973 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "calc.evaluate": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.count": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 0.75, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.75, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "coverageOfAll": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "calendar.day": { + "conversations": 24, + "inScope": 24, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 18, + "wrong": 0, + "missed": 6, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 18, + "relabelled": 0, + "coverage": 0.75, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.75, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 18, + "n": 24, + "rate": 0.75, + "low": 0.55100555994846, + "high": 0.8800063377140499 + }, + "coverageOfAll": { + "k": 18, + "n": 24, + "rate": 0.75, + "low": 0.55100555994846, + "high": 0.8800063377140499 + }, + "precision": { + "k": 18, + "n": 18, + "rate": 1, + "low": 0.8241207763533425, + "high": 1 + }, + "rubricPass": { + "k": 18, + "n": 18, + "rate": 1, + "low": 0.8241207763533425, + "high": 1 + }, + "execution": { + "k": 18, + "n": 18, + "rate": 1, + "low": 0.8241207763533425, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 7, + "inScope": 7, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 0.7142857142857143, + "coverageOfAll": 0.7142857142857143, + "precision": 1, + "intentLikeCoverage": 0.7142857142857143, + "rubricPass": 1, + "qualityCoverage": 0.7142857142857143, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 7, + "rate": 0.7142857142857143, + "low": 0.35893445183261935, + "high": 0.9177810759959433 + }, + "coverageOfAll": { + "k": 5, + "n": 7, + "rate": 0.7142857142857143, + "low": 0.35893445183261935, + "high": 0.9177810759959433 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "calendar.search": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 0.75, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.75, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "coverageOfAll": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "calendar.upcoming": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 5, + "inScope": 5, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "coverageOfAll": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "contacts.find": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.create": { + "conversations": 45, + "inScope": 45, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 41, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 41, + "relabelled": 0, + "coverage": 0.9111111111111111, + "coverageOfAll": 0.9111111111111111, + "precision": 1, + "intentLikeCoverage": 0.9111111111111111, + "rubricPass": 1, + "qualityCoverage": 0.9111111111111111, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 41, + "n": 45, + "rate": 0.9111111111111111, + "low": 0.7926642466571064, + "high": 0.9648888832736299 + }, + "coverageOfAll": { + "k": 41, + "n": 45, + "rate": 0.9111111111111111, + "low": 0.7926642466571064, + "high": 0.9648888832736299 + }, + "precision": { + "k": 41, + "n": 41, + "rate": 1, + "low": 0.9143324298155681, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 41, + "n": 41, + "rate": 1, + "low": 0.9143324298155681, + "high": 0.9999999999999999 + }, + "execution": { + "k": 41, + "n": 41, + "rate": 1, + "low": 0.9143324298155681, + "high": 0.9999999999999999 + } + } + }, + "event.create_invite": { + "conversations": 17, + "inScope": 17, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 10, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 10, + "relabelled": 0, + "coverage": 0.5882352941176471, + "coverageOfAll": 0.5882352941176471, + "precision": 1, + "intentLikeCoverage": 0.5882352941176471, + "rubricPass": 1, + "qualityCoverage": 0.5882352941176471, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 10, + "n": 17, + "rate": 0.5882352941176471, + "low": 0.3600544264299978, + "high": 0.7838894320978518 + }, + "coverageOfAll": { + "k": 10, + "n": 17, + "rate": 0.5882352941176471, + "low": 0.3600544264299978, + "high": 0.7838894320978518 + }, + "precision": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "execution": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + } + } + }, + "event.create_range": { + "conversations": 6, + "inScope": 6, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 6, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 6, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "coverageOfAll": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "precision": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "rubricPass": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "execution": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 15, + "inScope": 15, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 15, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 15, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "coverageOfAll": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "precision": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "rubricPass": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "execution": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + } + } + }, + "event.hide": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.rename": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 18, + "inScope": 18, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 14, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 14, + "relabelled": 0, + "coverage": 0.7777777777777778, + "coverageOfAll": 0.7777777777777778, + "precision": 1, + "intentLikeCoverage": 0.7777777777777778, + "rubricPass": 1, + "qualityCoverage": 0.7777777777777778, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 14, + "n": 18, + "rate": 0.7777777777777778, + "low": 0.5478541568378739, + "high": 0.9099907189139831 + }, + "coverageOfAll": { + "k": 14, + "n": 18, + "rate": 0.7777777777777778, + "low": 0.5478541568378739, + "high": 0.9099907189139831 + }, + "precision": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "rubricPass": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "execution": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + } + } + }, + "event.set_detail": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.show": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "event.snooze": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "google.status": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 0, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "coverageOfAll": { + "k": 0, + "n": 1, + "rate": 0, + "low": 0, + "high": 0.7934506856227626 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "holidays.upcoming": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "invite.send": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "reminder.after": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 0.75, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.75, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "coverageOfAll": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "reminder.clear": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "reminder.list": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "reminder.set": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "slots.check_time": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 0.6666666666666666, + "coverageOfAll": 0.6666666666666666, + "precision": 1, + "intentLikeCoverage": 0.6666666666666666, + "rubricPass": 1, + "qualityCoverage": 0.6666666666666666, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "coverageOfAll": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "slots.day": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.3333333333333333, + "coverageOfAll": 0.3333333333333333, + "precision": 1, + "intentLikeCoverage": 0.3333333333333333, + "rubricPass": 1, + "qualityCoverage": 0.3333333333333333, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 3, + "rate": 0.3333333333333333, + "low": 0.06149194472039621, + "high": 0.7923403991979522 + }, + "coverageOfAll": { + "k": 1, + "n": 3, + "rate": 0.3333333333333333, + "low": 0.06149194472039621, + "high": 0.7923403991979522 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + } + }, + "byStyle": { + "plain": { + "conversations": 118, + "inScope": 67, + "aiIntentLike": 48, + "aiOther": 3, + "correct": 59, + "wrong": 0, + "missed": 8, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 59, + "relabelled": 0, + "coverage": 0.8805970149253731, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5130434782608696, + "rubricPass": 1, + "qualityCoverage": 0.8805970149253731, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 59, + "n": 67, + "rate": 0.8805970149253731, + "low": 0.7816797416192777, + "high": 0.9382376746649551 + }, + "coverageOfAll": { + "k": 59, + "n": 118, + "rate": 0.5, + "low": 0.41121889415183216, + "high": 0.5887811058481678 + }, + "precision": { + "k": 59, + "n": 59, + "rate": 1, + "low": 0.9388706294732116, + "high": 1 + }, + "rubricPass": { + "k": 59, + "n": 59, + "rate": 1, + "low": 0.9388706294732116, + "high": 1 + }, + "execution": { + "k": 59, + "n": 59, + "rate": 1, + "low": 0.9388706294732116, + "high": 1 + } + } + }, + "verbless": { + "conversations": 29, + "inScope": 29, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 25, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 25, + "relabelled": 0, + "coverage": 0.8620689655172413, + "coverageOfAll": 0.8620689655172413, + "precision": 1, + "intentLikeCoverage": 0.8620689655172413, + "rubricPass": 1, + "qualityCoverage": 0.8620689655172413, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 25, + "n": 29, + "rate": 0.8620689655172413, + "low": 0.6944100627964798, + "high": 0.9450256003588804 + }, + "coverageOfAll": { + "k": 25, + "n": 29, + "rate": 0.8620689655172413, + "low": 0.6944100627964798, + "high": 0.9450256003588804 + }, + "precision": { + "k": 25, + "n": 25, + "rate": 1, + "low": 0.8668077490609515, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 25, + "n": 25, + "rate": 1, + "low": 0.8668077490609515, + "high": 0.9999999999999999 + }, + "execution": { + "k": 25, + "n": 25, + "rate": 1, + "low": 0.8668077490609515, + "high": 0.9999999999999999 + } + } + }, + "terse": { + "conversations": 58, + "inScope": 52, + "aiIntentLike": 5, + "aiOther": 1, + "correct": 46, + "wrong": 0, + "missed": 6, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 46, + "relabelled": 0, + "coverage": 0.8846153846153846, + "coverageOfAll": 0.7931034482758621, + "precision": 1, + "intentLikeCoverage": 0.8070175438596491, + "rubricPass": 1, + "qualityCoverage": 0.8846153846153846, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 46, + "n": 52, + "rate": 0.8846153846153846, + "low": 0.7702833565509788, + "high": 0.946030307969127 + }, + "coverageOfAll": { + "k": 46, + "n": 58, + "rate": 0.7931034482758621, + "low": 0.6723078207851659, + "high": 0.8774851603050945 + }, + "precision": { + "k": 46, + "n": 46, + "rate": 1, + "low": 0.9229264369144198, + "high": 1 + }, + "rubricPass": { + "k": 46, + "n": 46, + "rate": 1, + "low": 0.9229264369144198, + "high": 1 + }, + "execution": { + "k": 46, + "n": 46, + "rate": 1, + "low": 0.9229264369144198, + "high": 1 + } + } + }, + "typo": { + "conversations": 7, + "inScope": 6, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 0.8333333333333334, + "coverageOfAll": 0.7142857142857143, + "precision": 1, + "intentLikeCoverage": 0.7142857142857143, + "rubricPass": 1, + "qualityCoverage": 0.8333333333333334, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 6, + "rate": 0.8333333333333334, + "low": 0.43649717781352987, + "high": 0.9699466302516935 + }, + "coverageOfAll": { + "k": 5, + "n": 7, + "rate": 0.7142857142857143, + "low": 0.35893445183261935, + "high": 0.9177810759959433 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "slang": { + "conversations": 27, + "inScope": 25, + "aiIntentLike": 1, + "aiOther": 1, + "correct": 15, + "wrong": 0, + "missed": 10, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 15, + "relabelled": 0, + "coverage": 0.6, + "coverageOfAll": 0.5555555555555556, + "precision": 1, + "intentLikeCoverage": 0.5769230769230769, + "rubricPass": 1, + "qualityCoverage": 0.6, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 15, + "n": 25, + "rate": 0.6, + "low": 0.40739457358967, + "high": 0.7659669762225203 + }, + "coverageOfAll": { + "k": 15, + "n": 27, + "rate": 0.5555555555555556, + "low": 0.3731302451322448, + "high": 0.72414141822977 + }, + "precision": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "rubricPass": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "execution": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + } + } + }, + "no_punct": { + "conversations": 14, + "inScope": 13, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 11, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 11, + "relabelled": 0, + "coverage": 0.8461538461538461, + "coverageOfAll": 0.7857142857142857, + "precision": 1, + "intentLikeCoverage": 0.7857142857142857, + "rubricPass": 1, + "qualityCoverage": 0.8461538461538461, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 11, + "n": 13, + "rate": 0.8461538461538461, + "low": 0.5776536898051745, + "high": 0.9567418216419186 + }, + "coverageOfAll": { + "k": 11, + "n": 14, + "rate": 0.7857142857142857, + "low": 0.524107694133997, + "high": 0.9242861328730683 + }, + "precision": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "rubricPass": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "execution": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + } + } + }, + "caps": { + "conversations": 1, + "inScope": 1, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "coverageOfAll": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + }, + "polite": { + "conversations": 8, + "inScope": 8, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 0.625, + "coverageOfAll": 0.625, + "precision": 1, + "intentLikeCoverage": 0.625, + "rubricPass": 1, + "qualityCoverage": 0.625, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 8, + "rate": 0.625, + "low": 0.30574239460262737, + "high": 0.8631557141764026 + }, + "coverageOfAll": { + "k": 5, + "n": 8, + "rate": 0.625, + "low": 0.30574239460262737, + "high": 0.8631557141764026 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "trailing_punct": { + "conversations": 14, + "inScope": 14, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 10, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 10, + "relabelled": 0, + "coverage": 0.7142857142857143, + "coverageOfAll": 0.7142857142857143, + "precision": 1, + "intentLikeCoverage": 0.7142857142857143, + "rubricPass": 1, + "qualityCoverage": 0.7142857142857143, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 10, + "n": 14, + "rate": 0.7142857142857143, + "low": 0.4535091567010681, + "high": 0.8827862135542308 + }, + "coverageOfAll": { + "k": 10, + "n": 14, + "rate": 0.7142857142857143, + "low": 0.4535091567010681, + "high": 0.8827862135542308 + }, + "precision": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "execution": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + } + } + }, + "long": { + "conversations": 7, + "inScope": 6, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 3, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.42857142857142855, + "precision": 1, + "intentLikeCoverage": 0.42857142857142855, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 6, + "rate": 0.5, + "low": 0.1876163064826506, + "high": 0.8123836935173494 + }, + "coverageOfAll": { + "k": 3, + "n": 7, + "rate": 0.42857142857142855, + "low": 0.15821985525146975, + "high": 0.7495416354723428 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "mixed_lang": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 1, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1, + "relabelled": 0, + "coverage": 0.5, + "coverageOfAll": 0.5, + "precision": 1, + "intentLikeCoverage": 0.5, + "rubricPass": 1, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "coverageOfAll": { + "k": 1, + "n": 2, + "rate": 0.5, + "low": 0.09453120573423074, + "high": 0.9054687942657693 + }, + "precision": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "rubricPass": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + }, + "execution": { + "k": 1, + "n": 1, + "rate": 1, + "low": 0.20654931437723745, + "high": 1 + } + } + } + }, + "byRule": { + "basis.event.create": 48, + "basis.calendar.day": 18, + "basis.event.delete": 15, + "basis.event.reschedule": 14, + "basis.event.create_invite": 10, + "basis.calendar.when": 5, + "basis.calendar.period": 5, + "basis.reminder.set": 4, + "basis.invite.send": 3, + "basis.reminder.after": 3, + "basis.calendar.count": 3, + "basis.event.snooze": 3, + "basis.calendar.upcoming": 3, + "basis.calendar.search": 3, + "basis.invite.status": 2, + "basis.contacts.find": 1, + "basis.contacts.list": 1, + "basis.calc.evaluate": 1, + "basis.reminder.list": 1, + "basis.calendar.image": 1, + "basis.event.rename": 1, + "basis.slots.day": 1, + "basis.reminder.clear": 1, + "basis.event.show": 1, + "basis.slots.check_time": 1, + "basis.event.set_detail": 1, + "basis.holidays.upcoming": 1, + "basis.event.hide": 1, + "basis.calendar.next": 1 + }, + "failureClasses": { + "missed:fell_through_to_ai": 32 + }, + "rubricFindings": {} + }, + "all": { + "overall": { + "conversations": 755, + "inScope": 543, + "aiIntentLike": 185, + "aiOther": 27, + "correct": 491, + "wrong": 0, + "missed": 52, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 489, + "relabelled": 1, + "coverage": 0.9042357274401474, + "coverageOfAll": 0.6503311258278146, + "precision": 1, + "intentLikeCoverage": 0.6744505494505495, + "rubricPass": 0.9959266802443992, + "qualityCoverage": 0.9005524861878453, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 491, + "n": 543, + "rate": 0.9042357274401474, + "low": 0.8765693128318065, + "high": 0.926222781510743 + }, + "coverageOfAll": { + "k": 491, + "n": 755, + "rate": 0.6503311258278146, + "low": 0.6156327894728296, + "high": 0.6835074292927316 + }, + "precision": { + "k": 491, + "n": 491, + "rate": 1, + "low": 0.9922369907528583, + "high": 1 + }, + "rubricPass": { + "k": 489, + "n": 491, + "rate": 0.9959266802443992, + "low": 0.9852713565290308, + "high": 0.9988822371504846 + }, + "execution": { + "k": 491, + "n": 491, + "rate": 1, + "low": 0.9922369907528583, + "high": 1 + } + } + }, + "byFamily": { + "ai:advice_or_analysis": { + "conversations": 15, + "inScope": 0, + "aiIntentLike": 15, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 15, + "rate": 0, + "low": 0, + "high": 0.20388330103584862 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:ambiguous_day": { + "conversations": 7, + "inScope": 0, + "aiIntentLike": 7, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 7, + "rate": 0, + "low": 5.551115123125783e-17, + "high": 0.35433043506668743 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 40, + "inScope": 0, + "aiIntentLike": 40, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 40, + "rate": 0, + "low": 0, + "high": 0.08762160119728664 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 13, + "inScope": 0, + "aiIntentLike": 13, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 13, + "rate": 0, + "low": 0, + "high": 0.22809537235419838 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 27, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 27, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 27, + "rate": 0, + "low": 0, + "high": 0.12455502974186705 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 29, + "inScope": 0, + "aiIntentLike": 29, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 29, + "rate": 0, + "low": 0, + "high": 0.11696979849974075 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 20, + "inScope": 0, + "aiIntentLike": 20, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 20, + "rate": 0, + "low": 0, + "high": 0.16112515805281938 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:negation_or_quote": { + "conversations": 20, + "inScope": 0, + "aiIntentLike": 20, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 20, + "rate": 0, + "low": 0, + "high": 0.16112515805281938 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 12, + "inScope": 0, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 12, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.24249400665524085 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:recurrence": { + "conversations": 11, + "inScope": 0, + "aiIntentLike": 11, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 11, + "rate": 0, + "low": 0, + "high": 0.2588329669680317 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 17, + "inScope": 0, + "aiIntentLike": 17, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 17, + "rate": 0, + "low": 0, + "high": 0.1843181350088518 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "bot.info": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "calc.evaluate": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "calendar.count": { + "conversations": 9, + "inScope": 9, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 8, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 8, + "relabelled": 0, + "coverage": 0.8888888888888888, + "coverageOfAll": 0.8888888888888888, + "precision": 1, + "intentLikeCoverage": 0.8888888888888888, + "rubricPass": 1, + "qualityCoverage": 0.8888888888888888, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 8, + "n": 9, + "rate": 0.8888888888888888, + "low": 0.5650002944233439, + "high": 0.9801091123614551 + }, + "coverageOfAll": { + "k": 8, + "n": 9, + "rate": 0.8888888888888888, + "low": 0.5650002944233439, + "high": 0.9801091123614551 + }, + "precision": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "rubricPass": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "execution": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + } + } + }, + "calendar.day": { + "conversations": 71, + "inScope": 71, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 64, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 64, + "relabelled": 0, + "coverage": 0.9014084507042254, + "coverageOfAll": 0.9014084507042254, + "precision": 1, + "intentLikeCoverage": 0.9014084507042254, + "rubricPass": 1, + "qualityCoverage": 0.9014084507042254, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 64, + "n": 71, + "rate": 0.9014084507042254, + "low": 0.8101927775940715, + "high": 0.9514171762760307 + }, + "coverageOfAll": { + "k": 64, + "n": 71, + "rate": 0.9014084507042254, + "low": 0.8101927775940715, + "high": 0.9514171762760307 + }, + "precision": { + "k": 64, + "n": 64, + "rate": 1, + "low": 0.9433759402071946, + "high": 1 + }, + "rubricPass": { + "k": 64, + "n": 64, + "rate": 1, + "low": 0.9433759402071946, + "high": 1 + }, + "execution": { + "k": 64, + "n": 64, + "rate": 1, + "low": 0.9433759402071946, + "high": 1 + } + } + }, + "calendar.image": { + "conversations": 6, + "inScope": 6, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 0.8333333333333334, + "coverageOfAll": 0.8333333333333334, + "precision": 1, + "intentLikeCoverage": 0.8333333333333334, + "rubricPass": 0.6, + "qualityCoverage": 0.5, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 6, + "rate": 0.8333333333333334, + "low": 0.43649717781352987, + "high": 0.9699466302516935 + }, + "coverageOfAll": { + "k": 5, + "n": 6, + "rate": 0.8333333333333334, + "low": 0.43649717781352987, + "high": 0.9699466302516935 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 5, + "rate": 0.6, + "low": 0.2307242812760128, + "high": 0.882379225767352 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "calendar.period": { + "conversations": 21, + "inScope": 21, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 19, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 19, + "relabelled": 0, + "coverage": 0.9047619047619048, + "coverageOfAll": 0.9047619047619048, + "precision": 1, + "intentLikeCoverage": 0.9047619047619048, + "rubricPass": 1, + "qualityCoverage": 0.9047619047619048, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 19, + "n": 21, + "rate": 0.9047619047619048, + "low": 0.7108586092608176, + "high": 0.9734812325619255 + }, + "coverageOfAll": { + "k": 19, + "n": 21, + "rate": 0.9047619047619048, + "low": 0.7108586092608176, + "high": 0.9734812325619255 + }, + "precision": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "rubricPass": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + }, + "execution": { + "k": 19, + "n": 19, + "rate": 1, + "low": 0.8318207759473837, + "high": 1 + } + } + }, + "calendar.search": { + "conversations": 10, + "inScope": 10, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 9, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 9, + "relabelled": 0, + "coverage": 0.9, + "coverageOfAll": 0.9, + "precision": 1, + "intentLikeCoverage": 0.9, + "rubricPass": 1, + "qualityCoverage": 0.9, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 9, + "n": 10, + "rate": 0.9, + "low": 0.5958499732047615, + "high": 0.9821237869049271 + }, + "coverageOfAll": { + "k": 9, + "n": 10, + "rate": 0.9, + "low": 0.5958499732047615, + "high": 0.9821237869049271 + }, + "precision": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "rubricPass": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "execution": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + } + } + }, + "calendar.upcoming": { + "conversations": 12, + "inScope": 12, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 12, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 12, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 12, + "n": 12, + "rate": 1, + "low": 0.7575059933447592, + "high": 1 + }, + "coverageOfAll": { + "k": 12, + "n": 12, + "rate": 1, + "low": 0.7575059933447592, + "high": 1 + }, + "precision": { + "k": 12, + "n": 12, + "rate": 1, + "low": 0.7575059933447592, + "high": 1 + }, + "rubricPass": { + "k": 12, + "n": 12, + "rate": 1, + "low": 0.7575059933447592, + "high": 1 + }, + "execution": { + "k": 12, + "n": 12, + "rate": 1, + "low": 0.7575059933447592, + "high": 1 + } + } + }, + "calendar.when": { + "conversations": 15, + "inScope": 15, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 15, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 15, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "coverageOfAll": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "precision": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "rubricPass": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + }, + "execution": { + "k": 15, + "n": 15, + "rate": 1, + "low": 0.7961166989641515, + "high": 1 + } + } + }, + "contacts.find": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 0.75, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.75, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "coverageOfAll": { + "k": 3, + "n": 4, + "rate": 0.75, + "low": 0.30064184258240184, + "high": 0.9544127391902995 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "contacts.list": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "event.create": { + "conversations": 125, + "inScope": 125, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 114, + "wrong": 0, + "missed": 11, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 114, + "relabelled": 0, + "coverage": 0.912, + "coverageOfAll": 0.912, + "precision": 1, + "intentLikeCoverage": 0.912, + "rubricPass": 1, + "qualityCoverage": 0.912, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 114, + "n": 125, + "rate": 0.912, + "low": 0.8492803613696875, + "high": 0.9501517542146516 + }, + "coverageOfAll": { + "k": 114, + "n": 125, + "rate": 0.912, + "low": 0.8492803613696875, + "high": 0.9501517542146516 + }, + "precision": { + "k": 114, + "n": 114, + "rate": 1, + "low": 0.9674014658411585, + "high": 1 + }, + "rubricPass": { + "k": 114, + "n": 114, + "rate": 1, + "low": 0.9674014658411585, + "high": 1 + }, + "execution": { + "k": 114, + "n": 114, + "rate": 1, + "low": 0.9674014658411585, + "high": 1 + } + } + }, + "event.create_invite": { + "conversations": 43, + "inScope": 43, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 29, + "wrong": 0, + "missed": 14, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 29, + "relabelled": 0, + "coverage": 0.6744186046511628, + "coverageOfAll": 0.6744186046511628, + "precision": 1, + "intentLikeCoverage": 0.6744186046511628, + "rubricPass": 1, + "qualityCoverage": 0.6744186046511628, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 29, + "n": 43, + "rate": 0.6744186046511628, + "low": 0.5251621096752948, + "high": 0.7950670287343129 + }, + "coverageOfAll": { + "k": 29, + "n": 43, + "rate": 0.6744186046511628, + "low": 0.5251621096752948, + "high": 0.7950670287343129 + }, + "precision": { + "k": 29, + "n": 29, + "rate": 1, + "low": 0.8830302015002593, + "high": 1 + }, + "rubricPass": { + "k": 29, + "n": 29, + "rate": 1, + "low": 0.8830302015002593, + "high": 1 + }, + "execution": { + "k": 29, + "n": 29, + "rate": 1, + "low": 0.8830302015002593, + "high": 1 + } + } + }, + "event.create_range": { + "conversations": 16, + "inScope": 16, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 14, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 14, + "relabelled": 0, + "coverage": 0.875, + "coverageOfAll": 0.875, + "precision": 1, + "intentLikeCoverage": 0.875, + "rubricPass": 1, + "qualityCoverage": 0.875, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 14, + "n": 16, + "rate": 0.875, + "low": 0.639771727342413, + "high": 0.9650225122567595 + }, + "coverageOfAll": { + "k": 14, + "n": 16, + "rate": 0.875, + "low": 0.639771727342413, + "high": 0.9650225122567595 + }, + "precision": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "rubricPass": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + }, + "execution": { + "k": 14, + "n": 14, + "rate": 1, + "low": 0.7846891972623643, + "high": 1 + } + } + }, + "event.delete": { + "conversations": 47, + "inScope": 47, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 47, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 47, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 47, + "n": 47, + "rate": 1, + "low": 0.9244423958360037, + "high": 1 + }, + "coverageOfAll": { + "k": 47, + "n": 47, + "rate": 1, + "low": 0.9244423958360037, + "high": 1 + }, + "precision": { + "k": 47, + "n": 47, + "rate": 1, + "low": 0.9244423958360037, + "high": 1 + }, + "rubricPass": { + "k": 47, + "n": 47, + "rate": 1, + "low": 0.9244423958360037, + "high": 1 + }, + "execution": { + "k": 47, + "n": 47, + "rate": 1, + "low": 0.9244423958360037, + "high": 1 + } + } + }, + "event.hide": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "event.rename": { + "conversations": 6, + "inScope": 6, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 6, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 6, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "coverageOfAll": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "precision": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "rubricPass": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "execution": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + } + } + }, + "event.reschedule": { + "conversations": 43, + "inScope": 42, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 38, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 38, + "relabelled": 1, + "coverage": 0.9047619047619048, + "coverageOfAll": 0.8837209302325582, + "precision": 1, + "intentLikeCoverage": 0.8837209302325582, + "rubricPass": 1, + "qualityCoverage": 0.9047619047619048, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 38, + "n": 42, + "rate": 0.9047619047619048, + "low": 0.7793489419142897, + "high": 0.9623377511944261 + }, + "coverageOfAll": { + "k": 38, + "n": 43, + "rate": 0.8837209302325582, + "low": 0.7552083818991787, + "high": 0.9492957226019584 + }, + "precision": { + "k": 38, + "n": 38, + "rate": 1, + "low": 0.908190131774416, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 38, + "n": 38, + "rate": 1, + "low": 0.908190131774416, + "high": 0.9999999999999999 + }, + "execution": { + "k": 38, + "n": 38, + "rate": 1, + "low": 0.908190131774416, + "high": 0.9999999999999999 + } + } + }, + "event.set_detail": { + "conversations": 6, + "inScope": 6, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 0.8333333333333334, + "coverageOfAll": 0.8333333333333334, + "precision": 1, + "intentLikeCoverage": 0.8333333333333334, + "rubricPass": 1, + "qualityCoverage": 0.8333333333333334, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 6, + "rate": 0.8333333333333334, + "low": 0.43649717781352987, + "high": 0.9699466302516935 + }, + "coverageOfAll": { + "k": 5, + "n": 6, + "rate": 0.8333333333333334, + "low": 0.43649717781352987, + "high": 0.9699466302516935 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "event.show": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "event.snooze": { + "conversations": 9, + "inScope": 9, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 8, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 8, + "relabelled": 0, + "coverage": 0.8888888888888888, + "coverageOfAll": 0.8888888888888888, + "precision": 1, + "intentLikeCoverage": 0.8888888888888888, + "rubricPass": 1, + "qualityCoverage": 0.8888888888888888, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 8, + "n": 9, + "rate": 0.8888888888888888, + "low": 0.5650002944233439, + "high": 0.9801091123614551 + }, + "coverageOfAll": { + "k": 8, + "n": 9, + "rate": 0.8888888888888888, + "low": 0.5650002944233439, + "high": 0.9801091123614551 + }, + "precision": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "rubricPass": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + }, + "execution": { + "k": 8, + "n": 8, + "rate": 1, + "low": 0.6755924351161198, + "high": 1 + } + } + }, + "google.status": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 0.6666666666666666, + "coverageOfAll": 0.6666666666666666, + "precision": 1, + "intentLikeCoverage": 0.6666666666666666, + "rubricPass": 1, + "qualityCoverage": 0.6666666666666666, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "coverageOfAll": { + "k": 2, + "n": 3, + "rate": 0.6666666666666666, + "low": 0.2076596008020477, + "high": 0.9385080552796037 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "holidays.upcoming": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + }, + "invite.send": { + "conversations": 11, + "inScope": 11, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 11, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 11, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "coverageOfAll": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "precision": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "rubricPass": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "execution": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + } + } + }, + "invite.status": { + "conversations": 6, + "inScope": 6, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 6, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 6, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "coverageOfAll": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "precision": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "rubricPass": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + }, + "execution": { + "k": 6, + "n": 6, + "rate": 1, + "low": 0.6096657120978346, + "high": 1 + } + } + }, + "reminder.after": { + "conversations": 14, + "inScope": 14, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 13, + "wrong": 0, + "missed": 1, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 13, + "relabelled": 0, + "coverage": 0.9285714285714286, + "coverageOfAll": 0.9285714285714286, + "precision": 1, + "intentLikeCoverage": 0.9285714285714286, + "rubricPass": 1, + "qualityCoverage": 0.9285714285714286, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 13, + "n": 14, + "rate": 0.9285714285714286, + "low": 0.6853129557584889, + "high": 0.9872777847521093 + }, + "coverageOfAll": { + "k": 13, + "n": 14, + "rate": 0.9285714285714286, + "low": 0.6853129557584889, + "high": 0.9872777847521093 + }, + "precision": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + }, + "execution": { + "k": 13, + "n": 13, + "rate": 1, + "low": 0.7719046276458016, + "high": 0.9999999999999999 + } + } + }, + "reminder.clear": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "reminder.list": { + "conversations": 3, + "inScope": 3, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "coverageOfAll": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "reminder.set": { + "conversations": 11, + "inScope": 11, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 11, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 11, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "coverageOfAll": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "precision": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "rubricPass": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "execution": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + } + } + }, + "settings.view": { + "conversations": 2, + "inScope": 2, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 2, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 2, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "coverageOfAll": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "precision": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "rubricPass": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + }, + "execution": { + "k": 2, + "n": 2, + "rate": 1, + "low": 0.34238022750665303, + "high": 1 + } + } + }, + "slots.check_time": { + "conversations": 12, + "inScope": 12, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 10, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 10, + "relabelled": 0, + "coverage": 0.8333333333333334, + "coverageOfAll": 0.8333333333333334, + "precision": 1, + "intentLikeCoverage": 0.8333333333333334, + "rubricPass": 1, + "qualityCoverage": 0.8333333333333334, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 10, + "n": 12, + "rate": 0.8333333333333334, + "low": 0.5519691377470266, + "high": 0.9530348578161463 + }, + "coverageOfAll": { + "k": 10, + "n": 12, + "rate": 0.8333333333333334, + "low": 0.5519691377470266, + "high": 0.9530348578161463 + }, + "precision": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + }, + "execution": { + "k": 10, + "n": 10, + "rate": 1, + "low": 0.7224672001371107, + "high": 0.9999999999999999 + } + } + }, + "slots.day": { + "conversations": 11, + "inScope": 11, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 9, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 9, + "relabelled": 0, + "coverage": 0.8181818181818182, + "coverageOfAll": 0.8181818181818182, + "precision": 1, + "intentLikeCoverage": 0.8181818181818182, + "rubricPass": 1, + "qualityCoverage": 0.8181818181818182, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 9, + "n": 11, + "rate": 0.8181818181818182, + "low": 0.523019438039156, + "high": 0.9486323102539149 + }, + "coverageOfAll": { + "k": 9, + "n": 11, + "rate": 0.8181818181818182, + "low": 0.523019438039156, + "high": 0.9486323102539149 + }, + "precision": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "rubricPass": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + }, + "execution": { + "k": 9, + "n": 9, + "rate": 1, + "low": 0.7008549515804559, + "high": 1 + } + } + }, + "time.now": { + "conversations": 4, + "inScope": 4, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 4, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 4, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "coverageOfAll": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "precision": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "rubricPass": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + }, + "execution": { + "k": 4, + "n": 4, + "rate": 1, + "low": 0.5101091635454027, + "high": 1 + } + } + } + }, + "byStyle": { + "typo": { + "conversations": 36, + "inScope": 34, + "aiIntentLike": 2, + "aiOther": 0, + "correct": 30, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 30, + "relabelled": 0, + "coverage": 0.8823529411764706, + "coverageOfAll": 0.8333333333333334, + "precision": 1, + "intentLikeCoverage": 0.8333333333333334, + "rubricPass": 1, + "qualityCoverage": 0.8823529411764706, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 30, + "n": 34, + "rate": 0.8823529411764706, + "low": 0.7337915722962817, + "high": 0.9532855333511766 + }, + "coverageOfAll": { + "k": 30, + "n": 36, + "rate": 0.8333333333333334, + "low": 0.6810920568872054, + "high": 0.9212955240498188 + }, + "precision": { + "k": 30, + "n": 30, + "rate": 1, + "low": 0.8864866068260312, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 30, + "n": 30, + "rate": 1, + "low": 0.8864866068260312, + "high": 0.9999999999999999 + }, + "execution": { + "k": 30, + "n": 30, + "rate": 1, + "low": 0.8864866068260312, + "high": 0.9999999999999999 + } + } + }, + "plain": { + "conversations": 249, + "inScope": 136, + "aiIntentLike": 97, + "aiOther": 16, + "correct": 121, + "wrong": 0, + "missed": 15, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 121, + "relabelled": 0, + "coverage": 0.8897058823529411, + "coverageOfAll": 0.4859437751004016, + "precision": 1, + "intentLikeCoverage": 0.51931330472103, + "rubricPass": 1, + "qualityCoverage": 0.8897058823529411, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 121, + "n": 136, + "rate": 0.8897058823529411, + "low": 0.825989068198359, + "high": 0.9320121775975252 + }, + "coverageOfAll": { + "k": 121, + "n": 249, + "rate": 0.4859437751004016, + "low": 0.42455106847387564, + "high": 0.5477635984614858 + }, + "precision": { + "k": 121, + "n": 121, + "rate": 1, + "low": 0.9692293020525218, + "high": 1 + }, + "rubricPass": { + "k": 121, + "n": 121, + "rate": 1, + "low": 0.9692293020525218, + "high": 1 + }, + "execution": { + "k": 121, + "n": 121, + "rate": 1, + "low": 0.9692293020525218, + "high": 1 + } + } + }, + "no_punct": { + "conversations": 169, + "inScope": 137, + "aiIntentLike": 27, + "aiOther": 5, + "correct": 132, + "wrong": 0, + "missed": 5, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 130, + "relabelled": 0, + "coverage": 0.9635036496350365, + "coverageOfAll": 0.7810650887573964, + "precision": 1, + "intentLikeCoverage": 0.8048780487804879, + "rubricPass": 0.9848484848484849, + "qualityCoverage": 0.948905109489051, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 132, + "n": 137, + "rate": 0.9635036496350365, + "low": 0.9174111125832916, + "high": 0.9843120097374185 + }, + "coverageOfAll": { + "k": 132, + "n": 169, + "rate": 0.7810650887573964, + "low": 0.7128537569078812, + "high": 0.8367828907355126 + }, + "precision": { + "k": 132, + "n": 132, + "rate": 1, + "low": 0.9717210132013917, + "high": 1 + }, + "rubricPass": { + "k": 130, + "n": 132, + "rate": 0.9848484848484849, + "low": 0.9464398960595944, + "high": 0.9958350258326641 + }, + "execution": { + "k": 132, + "n": 132, + "rate": 1, + "low": 0.9717210132013917, + "high": 1 + } + } + }, + "caps": { + "conversations": 5, + "inScope": 5, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 5, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 5, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "coverageOfAll": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "precision": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "rubricPass": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + }, + "execution": { + "k": 5, + "n": 5, + "rate": 1, + "low": 0.5655175352168251, + "high": 1 + } + } + }, + "trailing_punct": { + "conversations": 71, + "inScope": 57, + "aiIntentLike": 10, + "aiOther": 4, + "correct": 52, + "wrong": 0, + "missed": 5, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 52, + "relabelled": 1, + "coverage": 0.9122807017543859, + "coverageOfAll": 0.7323943661971831, + "precision": 1, + "intentLikeCoverage": 0.7761194029850746, + "rubricPass": 1, + "qualityCoverage": 0.9122807017543859, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 52, + "n": 57, + "rate": 0.9122807017543859, + "low": 0.8105512290044132, + "high": 0.9619483282220778 + }, + "coverageOfAll": { + "k": 52, + "n": 71, + "rate": 0.7323943661971831, + "low": 0.6194601134214525, + "high": 0.8214719651349224 + }, + "precision": { + "k": 52, + "n": 52, + "rate": 1, + "low": 0.9312077638761376, + "high": 1 + }, + "rubricPass": { + "k": 52, + "n": 52, + "rate": 1, + "low": 0.9312077638761376, + "high": 1 + }, + "execution": { + "k": 52, + "n": 52, + "rate": 1, + "low": 0.9312077638761376, + "high": 1 + } + } + }, + "terse": { + "conversations": 206, + "inScope": 156, + "aiIntentLike": 46, + "aiOther": 4, + "correct": 149, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 148, + "relabelled": 0, + "coverage": 0.9551282051282052, + "coverageOfAll": 0.7233009708737864, + "precision": 1, + "intentLikeCoverage": 0.7376237623762376, + "rubricPass": 0.9932885906040269, + "qualityCoverage": 0.9487179487179487, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 149, + "n": 156, + "rate": 0.9551282051282052, + "low": 0.9102836178517977, + "high": 0.9780966625064529 + }, + "coverageOfAll": { + "k": 149, + "n": 206, + "rate": 0.7233009708737864, + "low": 0.6585459173659068, + "high": 0.7798803141187512 + }, + "precision": { + "k": 149, + "n": 149, + "rate": 1, + "low": 0.9748663821299904, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 148, + "n": 149, + "rate": 0.9932885906040269, + "low": 0.9629666399496506, + "high": 0.9988142873866486 + }, + "execution": { + "k": 149, + "n": 149, + "rate": 1, + "low": 0.9748663821299904, + "high": 0.9999999999999999 + } + } + }, + "verbless": { + "conversations": 82, + "inScope": 79, + "aiIntentLike": 3, + "aiOther": 0, + "correct": 73, + "wrong": 0, + "missed": 6, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 72, + "relabelled": 0, + "coverage": 0.9240506329113924, + "coverageOfAll": 0.8902439024390244, + "precision": 1, + "intentLikeCoverage": 0.8902439024390244, + "rubricPass": 0.9863013698630136, + "qualityCoverage": 0.9113924050632911, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 73, + "n": 79, + "rate": 0.9240506329113924, + "low": 0.8440458309717235, + "high": 0.9647279515496333 + }, + "coverageOfAll": { + "k": 73, + "n": 82, + "rate": 0.8902439024390244, + "low": 0.804387633588628, + "high": 0.9411728552386109 + }, + "precision": { + "k": 73, + "n": 73, + "rate": 1, + "low": 0.9500079920442688, + "high": 0.9999999999999998 + }, + "rubricPass": { + "k": 72, + "n": 73, + "rate": 0.9863013698630136, + "low": 0.9264026272903616, + "high": 0.997577748533516 + }, + "execution": { + "k": 73, + "n": 73, + "rate": 1, + "low": 0.9500079920442688, + "high": 0.9999999999999998 + } + } + }, + "slang": { + "conversations": 53, + "inScope": 50, + "aiIntentLike": 2, + "aiOther": 1, + "correct": 38, + "wrong": 0, + "missed": 12, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 37, + "relabelled": 0, + "coverage": 0.76, + "coverageOfAll": 0.7169811320754716, + "precision": 1, + "intentLikeCoverage": 0.7307692307692307, + "rubricPass": 0.9736842105263158, + "qualityCoverage": 0.74, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 38, + "n": 50, + "rate": 0.76, + "low": 0.6258731624205708, + "high": 0.8570260860300828 + }, + "coverageOfAll": { + "k": 38, + "n": 53, + "rate": 0.7169811320754716, + "low": 0.5842971358358297, + "high": 0.8203370955366142 + }, + "precision": { + "k": 38, + "n": 38, + "rate": 1, + "low": 0.908190131774416, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 37, + "n": 38, + "rate": 0.9736842105263158, + "low": 0.8650512388404787, + "high": 0.9953394123142312 + }, + "execution": { + "k": 38, + "n": 38, + "rate": 1, + "low": 0.908190131774416, + "high": 0.9999999999999999 + } + } + }, + "polite": { + "conversations": 32, + "inScope": 31, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 24, + "wrong": 0, + "missed": 7, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 24, + "relabelled": 0, + "coverage": 0.7741935483870968, + "coverageOfAll": 0.75, + "precision": 1, + "intentLikeCoverage": 0.75, + "rubricPass": 1, + "qualityCoverage": 0.7741935483870968, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 24, + "n": 31, + "rate": 0.7741935483870968, + "low": 0.6018758161665794, + "high": 0.886048644387375 + }, + "coverageOfAll": { + "k": 24, + "n": 32, + "rate": 0.75, + "low": 0.5789344100575552, + "high": 0.8674759908149096 + }, + "precision": { + "k": 24, + "n": 24, + "rate": 1, + "low": 0.8620237953250198, + "high": 1 + }, + "rubricPass": { + "k": 24, + "n": 24, + "rate": 1, + "low": 0.8620237953250198, + "high": 1 + }, + "execution": { + "k": 24, + "n": 24, + "rate": 1, + "low": 0.8620237953250198, + "high": 1 + } + } + }, + "long": { + "conversations": 8, + "inScope": 7, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 3, + "wrong": 0, + "missed": 4, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 3, + "relabelled": 0, + "coverage": 0.42857142857142855, + "coverageOfAll": 0.375, + "precision": 1, + "intentLikeCoverage": 0.375, + "rubricPass": 1, + "qualityCoverage": 0.42857142857142855, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 3, + "n": 7, + "rate": 0.42857142857142855, + "low": 0.15821985525146975, + "high": 0.7495416354723428 + }, + "coverageOfAll": { + "k": 3, + "n": 8, + "rate": 0.375, + "low": 0.13684428582359742, + "high": 0.6942576053973727 + }, + "precision": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "rubricPass": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + }, + "execution": { + "k": 3, + "n": 3, + "rate": 1, + "low": 0.4385029682449546, + "high": 1 + } + } + }, + "mixed_lang": { + "conversations": 14, + "inScope": 13, + "aiIntentLike": 1, + "aiOther": 0, + "correct": 11, + "wrong": 0, + "missed": 2, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 11, + "relabelled": 0, + "coverage": 0.8461538461538461, + "coverageOfAll": 0.7857142857142857, + "precision": 1, + "intentLikeCoverage": 0.7857142857142857, + "rubricPass": 1, + "qualityCoverage": 0.8461538461538461, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 11, + "n": 13, + "rate": 0.8461538461538461, + "low": 0.5776536898051745, + "high": 0.9567418216419186 + }, + "coverageOfAll": { + "k": 11, + "n": 14, + "rate": 0.7857142857142857, + "low": 0.524107694133997, + "high": 0.9242861328730683 + }, + "precision": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "rubricPass": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + }, + "execution": { + "k": 11, + "n": 11, + "rate": 1, + "low": 0.7411670330319684, + "high": 1 + } + } + } + }, + "byRule": { + "basis.event.create": 129, + "basis.calendar.day": 64, + "basis.event.delete": 47, + "basis.event.reschedule": 38, + "basis.event.create_invite": 29, + "basis.calendar.period": 19, + "basis.calendar.when": 15, + "basis.reminder.after": 13, + "basis.invite.send": 11, + "basis.reminder.set": 11, + "basis.calendar.upcoming": 10, + "basis.calendar.search": 9, + "basis.slots.check_time": 9, + "basis.slots.day": 9, + "basis.event.snooze": 8, + "basis.calendar.count": 8, + "basis.event.rename": 6, + "basis.invite.status": 6, + "basis.event.set_detail": 5, + "basis.calendar.image": 5, + "basis.time.now": 4, + "basis.event.show": 4, + "basis.event.hide": 4, + "basis.holidays.upcoming": 4, + "basis.calc.evaluate": 4, + "basis.contacts.find": 3, + "basis.reminder.list": 3, + "basis.reminder.clear": 3, + "basis.contacts.list": 3, + "basis.settings.view": 2, + "basis.bot.info": 2, + "basis.google.status": 2, + "basis.calendar.next": 2 + }, + "failureClasses": { + "missed:fell_through_to_ai": 52 + }, + "rubricFindings": { + "RAW_ISO_DATE": 2 + } + }, + "matrix": { + "overall": { + "conversations": 3156, + "inScope": 2130, + "aiIntentLike": 900, + "aiOther": 126, + "correct": 1920, + "wrong": 0, + "missed": 210, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 1878, + "relabelled": 54, + "coverage": 0.9014084507042254, + "coverageOfAll": 0.6083650190114068, + "precision": 1, + "intentLikeCoverage": 0.6336633663366337, + "rubricPass": 0.978125, + "qualityCoverage": 0.8816901408450705, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 1920, + "n": 2130, + "rate": 0.9014084507042254, + "low": 0.8880164354639921, + "high": 0.9133551907790662 + }, + "coverageOfAll": { + "k": 1920, + "n": 3156, + "rate": 0.6083650190114068, + "low": 0.5912136014275136, + "high": 0.6252529551747534 + }, + "precision": { + "k": 1920, + "n": 1920, + "rate": 1, + "low": 0.9980032352442133, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 1878, + "n": 1920, + "rate": 0.978125, + "low": 0.9705646020352869, + "high": 0.9837759916669921 + }, + "execution": { + "k": 1920, + "n": 1920, + "rate": 1, + "low": 0.9980032352442133, + "high": 0.9999999999999999 + } + } + }, + "byFamily": { + "ai:advice_or_analysis": { + "conversations": 84, + "inScope": 0, + "aiIntentLike": 84, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 84, + "rate": 0, + "low": 0, + "high": 0.04373172841465989 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:ambiguous_day": { + "conversations": 6, + "inScope": 0, + "aiIntentLike": 6, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 6, + "rate": 0, + "low": 2.7755575615628914e-17, + "high": 0.3903342879021653 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bare_hour": { + "conversations": 126, + "inScope": 0, + "aiIntentLike": 126, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 126, + "rate": 0, + "low": 1.734723475976807e-18, + "high": 0.02958576448219845 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:bulk": { + "conversations": 126, + "inScope": 0, + "aiIntentLike": 126, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 126, + "rate": 0, + "low": 1.734723475976807e-18, + "high": 0.02958576448219845 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:chitchat": { + "conversations": 126, + "inScope": 0, + "aiIntentLike": 0, + "aiOther": 126, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": null, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 126, + "rate": 0, + "low": 1.734723475976807e-18, + "high": 0.02958576448219845 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:missing_slot": { + "conversations": 210, + "inScope": 0, + "aiIntentLike": 210, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 210, + "rate": 0, + "low": 0, + "high": 0.017964050759283236 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:multi_request": { + "conversations": 42, + "inScope": 0, + "aiIntentLike": 42, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:negation_or_quote": { + "conversations": 126, + "inScope": 0, + "aiIntentLike": 126, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 126, + "rate": 0, + "low": 1.734723475976807e-18, + "high": 0.02958576448219845 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:past_or_invalid": { + "conversations": 42, + "inScope": 0, + "aiIntentLike": 42, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "ai:unsupported": { + "conversations": 84, + "inScope": 0, + "aiIntentLike": 84, + "aiOther": 0, + "correct": 0, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": null, + "coverageOfAll": 0, + "precision": null, + "intentLikeCoverage": 0, + "rubricPass": null, + "qualityCoverage": null, + "aiFallbackCorrect": 1, + "execution": null, + "intervals": { + "coverage": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "coverageOfAll": { + "k": 0, + "n": 84, + "rate": 0, + "low": 0, + "high": 0.04373172841465989 + }, + "precision": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "rubricPass": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + }, + "execution": { + "k": 0, + "n": 0, + "rate": null, + "low": null, + "high": null + } + } + }, + "calc.evaluate": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "calendar.day": { + "conversations": 420, + "inScope": 402, + "aiIntentLike": 18, + "aiOther": 0, + "correct": 360, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 360, + "relabelled": 18, + "coverage": 0.8955223880597015, + "coverageOfAll": 0.8571428571428571, + "precision": 1, + "intentLikeCoverage": 0.8571428571428571, + "rubricPass": 1, + "qualityCoverage": 0.8955223880597015, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 360, + "n": 402, + "rate": 0.8955223880597015, + "low": 0.861784942681361, + "high": 0.9217722644212688 + }, + "coverageOfAll": { + "k": 360, + "n": 420, + "rate": 0.8571428571428571, + "low": 0.8204352436514356, + "high": 0.8873765896570062 + }, + "precision": { + "k": 360, + "n": 360, + "rate": 1, + "low": 0.9894419431113063, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 360, + "n": 360, + "rate": 1, + "low": 0.9894419431113063, + "high": 0.9999999999999999 + }, + "execution": { + "k": 360, + "n": 360, + "rate": 1, + "low": 0.9894419431113063, + "high": 0.9999999999999999 + } + } + }, + "calendar.image": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 0, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 0, + "qualityCoverage": 0, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 0, + "n": 42, + "rate": 0, + "low": 0, + "high": 0.08379879086570391 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "calendar.period": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "contacts.find": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "contacts.list": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "event.create": { + "conversations": 504, + "inScope": 486, + "aiIntentLike": 18, + "aiOther": 0, + "correct": 402, + "wrong": 0, + "missed": 84, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 402, + "relabelled": 18, + "coverage": 0.8271604938271605, + "coverageOfAll": 0.7976190476190477, + "precision": 1, + "intentLikeCoverage": 0.7976190476190477, + "rubricPass": 1, + "qualityCoverage": 0.8271604938271605, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 402, + "n": 486, + "rate": 0.8271604938271605, + "low": 0.7910127146172659, + "high": 0.8581769247521602 + }, + "coverageOfAll": { + "k": 402, + "n": 504, + "rate": 0.7976190476190477, + "low": 0.7603518109989148, + "high": 0.8303837321303483 + }, + "precision": { + "k": 402, + "n": 402, + "rate": 1, + "low": 0.9905345825637017, + "high": 1 + }, + "rubricPass": { + "k": 402, + "n": 402, + "rate": 1, + "low": 0.9905345825637017, + "high": 1 + }, + "execution": { + "k": 402, + "n": 402, + "rate": 1, + "low": 0.9905345825637017, + "high": 1 + } + } + }, + "event.create_invite": { + "conversations": 210, + "inScope": 204, + "aiIntentLike": 6, + "aiOther": 0, + "correct": 126, + "wrong": 0, + "missed": 78, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 126, + "relabelled": 6, + "coverage": 0.6176470588235294, + "coverageOfAll": 0.6, + "precision": 1, + "intentLikeCoverage": 0.6, + "rubricPass": 1, + "qualityCoverage": 0.6176470588235294, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 126, + "n": 204, + "rate": 0.6176470588235294, + "low": 0.5493698847799827, + "high": 0.6815753764415037 + }, + "coverageOfAll": { + "k": 126, + "n": 210, + "rate": 0.6, + "low": 0.5325179857434847, + "high": 0.6638892041046585 + }, + "precision": { + "k": 126, + "n": 126, + "rate": 1, + "low": 0.9704142355178014, + "high": 0.9999999999999998 + }, + "rubricPass": { + "k": 126, + "n": 126, + "rate": 1, + "low": 0.9704142355178014, + "high": 0.9999999999999998 + }, + "execution": { + "k": 126, + "n": 126, + "rate": 1, + "low": 0.9704142355178014, + "high": 0.9999999999999998 + } + } + }, + "event.delete": { + "conversations": 336, + "inScope": 336, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 336, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 336, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + }, + "precision": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + }, + "execution": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + } + } + }, + "event.rename": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "event.reschedule": { + "conversations": 294, + "inScope": 282, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 276, + "wrong": 0, + "missed": 6, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 276, + "relabelled": 12, + "coverage": 0.9787234042553191, + "coverageOfAll": 0.9387755102040817, + "precision": 1, + "intentLikeCoverage": 0.9387755102040817, + "rubricPass": 1, + "qualityCoverage": 0.9787234042553191, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 276, + "n": 282, + "rate": 0.9787234042553191, + "low": 0.9543664300048462, + "high": 0.9902131320850907 + }, + "coverageOfAll": { + "k": 276, + "n": 294, + "rate": 0.9387755102040817, + "low": 0.9053074648870244, + "high": 0.9609251980254973 + }, + "precision": { + "k": 276, + "n": 276, + "rate": 1, + "low": 0.9862727315785058, + "high": 0.9999999999999998 + }, + "rubricPass": { + "k": 276, + "n": 276, + "rate": 1, + "low": 0.9862727315785058, + "high": 0.9999999999999998 + }, + "execution": { + "k": 276, + "n": 276, + "rate": 1, + "low": 0.9862727315785058, + "high": 0.9999999999999998 + } + } + }, + "event.snooze": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "invite.status": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "reminder.set": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "settings.view": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + } + }, + "byStyle": { + "typo": { + "conversations": 42, + "inScope": 42, + "aiIntentLike": 0, + "aiOther": 0, + "correct": 42, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 42, + "relabelled": 0, + "coverage": 1, + "coverageOfAll": 1, + "precision": 1, + "intentLikeCoverage": 1, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": null, + "execution": 1, + "intervals": { + "coverage": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "coverageOfAll": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "precision": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + }, + "execution": { + "k": 42, + "n": 42, + "rate": 1, + "low": 0.916201209134296, + "high": 0.9999999999999999 + } + } + }, + "trailing_punct": { + "conversations": 378, + "inScope": 282, + "aiIntentLike": 96, + "aiOther": 0, + "correct": 282, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 282, + "relabelled": 12, + "coverage": 1, + "coverageOfAll": 0.746031746031746, + "precision": 1, + "intentLikeCoverage": 0.746031746031746, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 282, + "n": 282, + "rate": 1, + "low": 0.9865608759606008, + "high": 1 + }, + "coverageOfAll": { + "k": 282, + "n": 378, + "rate": 0.746031746031746, + "low": 0.6998273601630959, + "high": 0.7872858008185729 + }, + "precision": { + "k": 282, + "n": 282, + "rate": 1, + "low": 0.9865608759606008, + "high": 1 + }, + "rubricPass": { + "k": 282, + "n": 282, + "rate": 1, + "low": 0.9865608759606008, + "high": 1 + }, + "execution": { + "k": 282, + "n": 282, + "rate": 1, + "low": 0.9865608759606008, + "high": 1 + } + } + }, + "plain": { + "conversations": 1056, + "inScope": 498, + "aiIntentLike": 474, + "aiOther": 84, + "correct": 336, + "wrong": 0, + "missed": 162, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 336, + "relabelled": 6, + "coverage": 0.6746987951807228, + "coverageOfAll": 0.3181818181818182, + "precision": 1, + "intentLikeCoverage": 0.345679012345679, + "rubricPass": 1, + "qualityCoverage": 0.6746987951807228, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 336, + "n": 498, + "rate": 0.6746987951807228, + "low": 0.6323511583604485, + "high": 0.7143718892109746 + }, + "coverageOfAll": { + "k": 336, + "n": 1056, + "rate": 0.3181818181818182, + "low": 0.2907916917908576, + "high": 0.34688996634385616 + }, + "precision": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + }, + "execution": { + "k": 336, + "n": 336, + "rate": 1, + "low": 0.9886963208255265, + "high": 0.9999999999999999 + } + } + }, + "terse": { + "conversations": 966, + "inScope": 738, + "aiIntentLike": 228, + "aiOther": 0, + "correct": 738, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 696, + "relabelled": 18, + "coverage": 1, + "coverageOfAll": 0.7639751552795031, + "precision": 1, + "intentLikeCoverage": 0.7639751552795031, + "rubricPass": 0.943089430894309, + "qualityCoverage": 0.943089430894309, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 738, + "n": 738, + "rate": 1, + "low": 0.9948217253497791, + "high": 1 + }, + "coverageOfAll": { + "k": 738, + "n": 966, + "rate": 0.7639751552795031, + "low": 0.7361842198929713, + "high": 0.7896749247760437 + }, + "precision": { + "k": 738, + "n": 738, + "rate": 1, + "low": 0.9948217253497791, + "high": 1 + }, + "rubricPass": { + "k": 696, + "n": 738, + "rate": 0.943089430894309, + "low": 0.923966696194483, + "high": 0.9576232880585733 + }, + "execution": { + "k": 738, + "n": 738, + "rate": 1, + "low": 0.9948217253497791, + "high": 1 + } + } + }, + "slang": { + "conversations": 168, + "inScope": 156, + "aiIntentLike": 12, + "aiOther": 0, + "correct": 114, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 114, + "relabelled": 12, + "coverage": 0.7307692307692307, + "coverageOfAll": 0.6785714285714286, + "precision": 1, + "intentLikeCoverage": 0.6785714285714286, + "rubricPass": 1, + "qualityCoverage": 0.7307692307692307, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 114, + "n": 156, + "rate": 0.7307692307692307, + "low": 0.6562367115839256, + "high": 0.794209627752652 + }, + "coverageOfAll": { + "k": 114, + "n": 168, + "rate": 0.6785714285714286, + "low": 0.6046383896622172, + "high": 0.7445206573556873 + }, + "precision": { + "k": 114, + "n": 114, + "rate": 1, + "low": 0.9674014658411585, + "high": 1 + }, + "rubricPass": { + "k": 114, + "n": 114, + "rate": 1, + "low": 0.9674014658411585, + "high": 1 + }, + "execution": { + "k": 114, + "n": 114, + "rate": 1, + "low": 0.9674014658411585, + "high": 1 + } + } + }, + "mixed_lang": { + "conversations": 126, + "inScope": 120, + "aiIntentLike": 6, + "aiOther": 0, + "correct": 78, + "wrong": 0, + "missed": 42, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 78, + "relabelled": 6, + "coverage": 0.65, + "coverageOfAll": 0.6190476190476191, + "precision": 1, + "intentLikeCoverage": 0.6190476190476191, + "rubricPass": 1, + "qualityCoverage": 0.65, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 78, + "n": 120, + "rate": 0.65, + "low": 0.561213245977243, + "high": 0.729481004097146 + }, + "coverageOfAll": { + "k": 78, + "n": 126, + "rate": 0.6190476190476191, + "low": 0.5319219933564636, + "high": 0.6991290151001559 + }, + "precision": { + "k": 78, + "n": 78, + "rate": 1, + "low": 0.9530621902878059, + "high": 1 + }, + "rubricPass": { + "k": 78, + "n": 78, + "rate": 1, + "low": 0.9530621902878059, + "high": 1 + }, + "execution": { + "k": 78, + "n": 78, + "rate": 1, + "low": 0.9530621902878059, + "high": 1 + } + } + }, + "polite": { + "conversations": 126, + "inScope": 120, + "aiIntentLike": 6, + "aiOther": 0, + "correct": 120, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 120, + "relabelled": 6, + "coverage": 1, + "coverageOfAll": 0.9523809523809523, + "precision": 1, + "intentLikeCoverage": 0.9523809523809523, + "rubricPass": 1, + "qualityCoverage": 1, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 120, + "n": 120, + "rate": 1, + "low": 0.9689808335812966, + "high": 1 + }, + "coverageOfAll": { + "k": 120, + "n": 126, + "rate": 0.9523809523809523, + "low": 0.8999983205231048, + "high": 0.9779955116120489 + }, + "precision": { + "k": 120, + "n": 120, + "rate": 1, + "low": 0.9689808335812966, + "high": 1 + }, + "rubricPass": { + "k": 120, + "n": 120, + "rate": 1, + "low": 0.9689808335812966, + "high": 1 + }, + "execution": { + "k": 120, + "n": 120, + "rate": 1, + "low": 0.9689808335812966, + "high": 1 + } + } + }, + "no_punct": { + "conversations": 714, + "inScope": 582, + "aiIntentLike": 90, + "aiOther": 42, + "correct": 576, + "wrong": 0, + "missed": 6, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 534, + "relabelled": 6, + "coverage": 0.9896907216494846, + "coverageOfAll": 0.8067226890756303, + "precision": 1, + "intentLikeCoverage": 0.8571428571428571, + "rubricPass": 0.9270833333333334, + "qualityCoverage": 0.9175257731958762, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 576, + "n": 582, + "rate": 0.9896907216494846, + "low": 0.9776926382546434, + "high": 0.9952668398052005 + }, + "coverageOfAll": { + "k": 576, + "n": 714, + "rate": 0.8067226890756303, + "low": 0.7761487623998028, + "high": 0.83401382241324 + }, + "precision": { + "k": 576, + "n": 576, + "rate": 1, + "low": 0.9933749842094647, + "high": 0.9999999999999999 + }, + "rubricPass": { + "k": 534, + "n": 576, + "rate": 0.9270833333333334, + "low": 0.9029031240390288, + "high": 0.9456046749732223 + }, + "execution": { + "k": 576, + "n": 576, + "rate": 1, + "low": 0.9933749842094647, + "high": 0.9999999999999999 + } + } + }, + "verbless": { + "conversations": 420, + "inScope": 402, + "aiIntentLike": 18, + "aiOther": 0, + "correct": 402, + "wrong": 0, + "missed": 0, + "falseMatch": 0, + "harmfulFalseMatch": 0, + "unconfirmedWrites": 0, + "exceptions": 0, + "rubricClean": 360, + "relabelled": 18, + "coverage": 1, + "coverageOfAll": 0.9571428571428572, + "precision": 1, + "intentLikeCoverage": 0.9571428571428572, + "rubricPass": 0.8955223880597015, + "qualityCoverage": 0.8955223880597015, + "aiFallbackCorrect": 1, + "execution": 1, + "intervals": { + "coverage": { + "k": 402, + "n": 402, + "rate": 1, + "low": 0.9905345825637017, + "high": 1 + }, + "coverageOfAll": { + "k": 402, + "n": 420, + "rate": 0.9571428571428572, + "low": 0.9332776936660707, + "high": 0.9727214529687349 + }, + "precision": { + "k": 402, + "n": 402, + "rate": 1, + "low": 0.9905345825637017, + "high": 1 + }, + "rubricPass": { + "k": 360, + "n": 402, + "rate": 0.8955223880597015, + "low": 0.861784942681361, + "high": 0.9217722644212688 + }, + "execution": { + "k": 402, + "n": 402, + "rate": 1, + "low": 0.9905345825637017, + "high": 1 + } + } + } + }, + "byRule": { + "basis.event.create": 402, + "basis.calendar.day": 360, + "basis.event.delete": 336, + "basis.event.reschedule": 276, + "basis.event.create_invite": 126, + "basis.contacts.find": 42, + "basis.event.snooze": 42, + "basis.settings.view": 42, + "basis.event.rename": 42, + "basis.calendar.period": 42, + "basis.calendar.image": 42, + "basis.invite.status": 42, + "basis.calc.evaluate": 42, + "basis.contacts.list": 42, + "basis.reminder.set": 42 + }, + "failureClasses": { + "missed:fell_through_to_ai": 210 + }, + "rubricFindings": { + "RAW_ISO_DATE": 42 + }, + "cells": 3156, + "anchors": 11 + }, + "real": null, + "latency": { + "matcherMicrosP50": 29.280000017024577, + "matcherMicrosP95": 122.47000000206754, + "matcherMicrosMax": 393.8719999860041, + "layerMsP50": 15.090523999999277, + "layerMsP95": 24.47257699999318, + "samples": 2265 + }, + "historyKind": "synthetic", + "generatedAt": "2026-11-02T09:47:31.310Z", + "source": { + "sha": "1e81955372ac0053687c140f95c68e03a94c2c64", + "dirty": true, + "engineDigest": "6935a57b68b65dd6f414637a134039f280b5c1ff1b2344d609fa21a6ecd1112b" + }, + "checksum": "d741a79e7906aad07810c464d769214a949fa5cd586ff3accb93130c9ed6f78e" +} diff --git a/docs/intents/kpi-gate.json b/docs/intents/kpi-gate.json index 39e196f83..2a258ff2c 100644 --- a/docs/intents/kpi-gate.json +++ b/docs/intents/kpi-gate.json @@ -62,7 +62,7 @@ "version": "1", "checksum": "98447f88cd71fa57404321b1da6fad4df63a2784d8ace86de0aa3eb84d1f64f8" }, - "benchmarkResult": "docs/intents/benchmarks/20261102T053634Z-synthetic-current.json" + "benchmarkResult": "docs/intents/benchmarks/20261102T094731Z-synthetic-current.json" }, "informational": { "intervals95": { @@ -100,7 +100,7 @@ "labelsChecksum": "19ee899c60362252cd89a08b71fc5621f417f2ece970dbf58222792ffff05f7e", "splitChecksum": "310707bb8c752d46692ceb9b50cc86bda217b681ac54867263ca3e8dda8354f0" }, - "engineDigest": "736154156269de819a0a775de5e6162f364a300052aca709ea53273c941d4d2f", - "benchmarkResult": "docs/intents/benchmarks/20261102T053443Z-real-current.json", + "engineDigest": "6935a57b68b65dd6f414637a134039f280b5c1ff1b2344d609fa21a6ecd1112b", + "benchmarkResult": "docs/intents/benchmarks/20261102T094538Z-real-current.json", "autoActivate": true } diff --git a/src/services/intent/intent-executor.ts b/src/services/intent/intent-executor.ts index 11136e5f2..7f3bc95cf 100644 --- a/src/services/intent/intent-executor.ts +++ b/src/services/intent/intent-executor.ts @@ -456,7 +456,9 @@ function respondWithText( if (writeReports.some((report) => 'existing' in report)) { const { timezone, language } = env.userCtx; const lines = writeReports.map((report) => - 'existing' in report ? renderToolAnswer(report.existing, { now: new Date(), timezone, language }).text : report.text, + 'existing' in report + ? renderToolAnswer(report.existing, { now: new Date(), timezone, language }).text + : report.text, ); return { success: true, response: lines.join('\n'), mentionedEventId }; } @@ -619,8 +621,7 @@ async function runLevel2Step( if (step.abstain === true) return { success: false, errorCode: 'ABSTAIN' }; // Respond with text and optionally stop - if (step.respond !== undefined) - return respondWithText(step.respond, env, state.mentionedEventId, state.writeReports); + if (step.respond !== undefined) return respondWithText(step.respond, env, state.mentionedEventId, state.writeReports); // No call — nothing to execute in this step if (step.call === undefined) return null; diff --git a/test/services/ai/chat-run-gate.test.ts b/test/services/ai/chat-run-gate.test.ts index f9d7183e5..846e96a16 100644 --- a/test/services/ai/chat-run-gate.test.ts +++ b/test/services/ai/chat-run-gate.test.ts @@ -329,7 +329,12 @@ describe('ChatRunGate', () => { test('a run whose place expired still owns its message and is placed behind the runs in between', async () => { const time = new ManualTime(); - const expiring = new ChatRunGate({ leaseMs: LEASE_MS, reservationMs: 1000, now: time.now, schedule: time.schedule }); + const expiring = new ChatRunGate({ + leaseMs: LEASE_MS, + reservationMs: 1000, + now: time.now, + schedule: time.schedule, + }); const reservation = expiring.reserve(1, 10); time.advance(1000); const slot = await expiring.run(1, { reservation }, async (s) => s); From 705cd9fcdc9314ffc8042450c0d995f9fa8fe984 Mon Sep 17 00:00:00 2001 From: Alex Ultra <111590326+alex-mextner@users.noreply.github.com> Date: Wed, 4 Nov 2026 03:17:50 +0100 Subject: [PATCH 4/4] chore(release): bump the version to 0.3.1 for the per-chat serialization and idempotent create_event fix (Refs #929, #520) Reviewed as a single-seat review via omp Opus (claude-opus-5-5, three role invocations, no findings); the fixed Opus+Sol roster is refused until Sol returns 2026-11-09T10:20Z, the full-roster review stays owed in #961. --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f20f13a28..0b0947dd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "hypercalendarbot" -version = "0.3.0" +version = "0.3.1" requires-python = ">=3.12" dependencies = [ "pyrofork>=2.3", diff --git a/uv.lock b/uv.lock index 4b8624bee..656397f86 100644 --- a/uv.lock +++ b/uv.lock @@ -335,7 +335,7 @@ wheels = [ [[package]] name = "hypercalendarbot" -version = "0.3.0" +version = "0.3.1" source = { virtual = "." } dependencies = [ { name = "numpy" },