From 0f4981f44c1c76d191bf8610098b0b91ccaaac29 Mon Sep 17 00:00:00 2001 From: alexdancer Date: Thu, 30 Jul 2026 16:16:17 -0500 Subject: [PATCH] feat: add bounded teacher guidance action card --- README.md | 24 ++- apps/web/app/board/evidence-desk.tsx | 72 ++++++-- apps/web/app/globals.css | 161 ++++++++++++++++++ apps/web/lib/operations.ts | 84 ++++++++- apps/web/lib/teacher-guidance-policy.ts | 7 + apps/web/test/evidence-desk-rendering.test.ts | 25 +++ apps/web/test/evidence-desk-state.test.ts | 4 + apps/web/test/teacher-guidance-policy.test.ts | 10 ++ db/migrations/011_teacher_guidance_card.sql | 27 +++ packages/application/src/board-reader.ts | 8 + .../evidence-reader-implementation.test.ts | 4 + packages/db/src/boards.ts | 38 ++++- packages/db/src/scoped.ts | 19 ++- .../db/test/operations-migrations.test.ts | 13 +- packages/narrator/src/batch.ts | 15 +- packages/narrator/src/catalog.ts | 79 ++++++++- packages/narrator/src/index.ts | 6 +- packages/narrator/src/prompt.ts | 61 +++++-- packages/narrator/src/render.ts | 8 +- packages/narrator/src/request.ts | 65 ++++--- .../narrator/test/fallback-catalog.test.ts | 12 ++ packages/narrator/test/no-pii.test.ts | 23 ++- .../narrator/test/request-contract.test.ts | 29 ++++ .../contracts/application-interfaces.md | 50 +++--- specs/001-huddle-triage-board/plan.md | 14 +- specs/001-huddle-triage-board/spec.md | 73 ++++---- 26 files changed, 777 insertions(+), 154 deletions(-) create mode 100644 apps/web/lib/teacher-guidance-policy.ts create mode 100644 apps/web/test/teacher-guidance-policy.test.ts create mode 100644 db/migrations/011_teacher_guidance_card.sql diff --git a/README.md b/README.md index 20d5a76..6746cf3 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,13 @@ acknowledged. It is not a diagnosis product and makes **no accuracy claim**. ## Why this is useful at TSA - **Guide utility:** one deterministic queue turns attempt-level activity into a short morning plan. -- **Coach-like intervention:** every report includes a concrete opener, not another analytics chart. +- **Coach-like intervention:** every report provides a visible teacher decision map, not another + analytics chart. - **Responsible student data:** the roster, activity, names, and evaluation corpus are visibly synthetic; real student data is rejected and remains out of scope. -- **Trustworthy AI boundary:** rules classify and rank without a model. Optional AI may select only - reviewed language-catalog IDs and evidence slots; deterministic fallback always keeps the board - usable. +- **Trustworthy AI boundary:** rules classify and rank without a model. Only a manual synthetic + refresh may draft the explanation, first question, and teaching move from a bounded packet; a + complete deterministic card always keeps the board usable. ## Architecture and trust boundaries @@ -139,14 +140,19 @@ walkthrough below; the smoke command does not pretend to replace it. validate. Call out the separate received/accepted/duplicate/unmapped/rejected counts and that validation stores no activity. 3. **0:40–0:55 — Commit and refresh.** Commit the same bytes, then choose **Refresh board now**. The - deterministic engine publishes one immutable ranked run; no model key is needed. -4. **0:55–1:30 — Use the Evidence Desk.** Open the top report. Read the cause-specific opener, - compare it with the student's own baseline, then expand **Show exact contributing evidence**. + deterministic engine publishes one immutable ranked run. With `ANTHROPIC_API_KEY`, this manual + synthetic refresh may draft only the explanation, first question, and teaching move; without it, + the complete deterministic card publishes instead. Nightly refresh never invokes AI. +4. **0:55–1:30 — Use the Evidence Desk.** Open the top report. The visible teacher-decision map + presents a hypothesis, first question, confident/struggling fork, listen-for, and teaching move. + Only the three draftable fields are labeled AI-drafted wording; the fork and listen-for are + deterministic teacher support. Expand **Show exact contributing evidence** for the sole detail. Exact attempt/session results, selected answers, and timing-aware durations remain readable in seconds without exposing internal record IDs, additional causes, or rule-quality internals. Severity determines rank; evidence confidence is shown separately. -5. **1:30–1:45 — Show trust behavior.** Point to the concise “Deterministic fallback” wording cue - and the visible “Seen” acknowledgment. Refresh/back navigation keeps the report and seen state. +5. **1:30–1:45 — Show trust behavior.** Point to the teacher-reviewed-draft framing, a visible + `Working hypothesis` when evidence is weak, and the visible “Seen” acknowledgment. Refresh/back + navigation keeps the report and seen state. 6. **1:45–2:00 — Show engineering evidence.** Run `npm run eval:portfolio`: eight fixed cases, hard-fail grounding injections, zero model calls, and an explicit no-accuracy-claim posture. diff --git a/apps/web/app/board/evidence-desk.tsx b/apps/web/app/board/evidence-desk.tsx index 423257b..34f8dfe 100644 --- a/apps/web/app/board/evidence-desk.tsx +++ b/apps/web/app/board/evidence-desk.tsx @@ -96,7 +96,7 @@ function Freshness({ const narration = state.narration.status === 'degraded' ? `${state.narration.degradedCount} deterministic fallback${state.narration.degradedCount === 1 ? '' : 's'}` - : 'Conversation openers ready'; + : 'Conversation plans ready'; return (
@@ -326,6 +326,64 @@ export function ReportContext({ ); } +export function TeacherGuidanceCard({ entry }: { entry: BoardEntryView }) { + const weakEvidence = entry.finalConfidence < 0.55; + return ( +
+
+
+

AI-assisted conversation plan

+

+ Teacher-reviewed draft. Use your professional judgment before speaking with the student. +

+
+
+
+

+ Likely explanation to test AI-drafted wording +

+

{entry.diagnosis}

+
+
+

+ Ask first AI-drafted wording +

+

{entry.opener}

+
+
+
+

+ If they explain confidently Teacher support +

+

{entry.confidentFollowUp}

+
+
+

+ If they struggle Teacher support +

+

{entry.strugglingFollowUp}

+
+
+
+
+

+ Listen for Teacher support +

+

{entry.listenFor}

+
+
+

+ Suggested teaching move AI-drafted wording +

+

{entry.teachingMove}

+
+
+
+ ); +} + function Workspace({ evidence, grant, @@ -347,17 +405,7 @@ function Workspace({
-
-

Why Huddle ranked this

-

{entry.diagnosis}

-
- Conversation opener -
- {entry.opener} -
-
- - +
); diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index dee1927..5f4801f 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -659,6 +659,144 @@ button:hover { border-top: 1px solid var(--huddle-line); } +.teacher-guidance-card { + overflow: hidden; + margin-bottom: 20px; + border: 1px solid var(--huddle-line); + border-radius: 12px; +} + +.teacher-guidance-heading { + display: flex; + justify-content: space-between; + gap: 16px; + padding: 16px; + border-bottom: 1px solid var(--huddle-line); + background: var(--huddle-surface); +} + +.teacher-guidance-heading h3, +.guidance-field h4 { + margin: 0; +} + +.teacher-guidance-heading h3 { + font-size: 1.1rem; +} + +.teacher-guidance-heading p { + margin: 5px 0 0; + color: var(--huddle-muted); + font-size: 0.84rem; +} + +.guidance-field { + min-width: 0; + padding: 15px 16px; +} + +.guidance-field h4 { + display: flex; + flex-wrap: wrap; + gap: 6px; + align-items: center; + color: var(--huddle-muted); + font-size: 0.72rem; + letter-spacing: 0.055em; + text-transform: uppercase; +} + +.guidance-field h4 span { + padding: 2px 5px; + border-radius: 4px; + background: #efedf0; + color: #554b4f; + font-size: 0.58rem; + letter-spacing: 0.02em; + text-transform: none; +} + +.guidance-explanation h4 span, +.guidance-ask h4 span, +.guidance-move h4 span { + background: #f2dbe3; + color: var(--huddle-brand-dark); +} + +.guidance-field > p { + margin: 8px 0 0; + overflow-wrap: anywhere; +} + +.guidance-explanation, +.guidance-ask { + border-bottom: 1px solid var(--huddle-line); +} + +.guidance-explanation.working-hypothesis { + border-left: 4px solid var(--huddle-warn); + background: #fff7e7; +} + +.guidance-ask { + background: #fff2f6; +} + +.guidance-ask > p, +.guidance-move > p { + font-weight: 760; +} + +.guidance-fork, +.guidance-footer { + display: grid; + grid-template-columns: 1fr 1fr; +} + +.guidance-fork { + position: relative; + gap: 20px; + padding: 34px 16px 16px; + border-bottom: 1px solid var(--huddle-line); +} + +.guidance-fork::before { + position: absolute; + top: 13px; + right: 25%; + left: 25%; + height: 18px; + border-top: 2px solid #c6a5b0; + border-right: 2px solid #c6a5b0; + border-left: 2px solid #c6a5b0; + content: ''; +} + +.guidance-fork .guidance-field, +.guidance-footer { + border: 1px solid var(--huddle-line); + border-radius: 9px; +} + +.guidance-confident { + background: var(--huddle-green-soft); +} +.guidance-struggle { + background: var(--huddle-blue-soft); +} +.guidance-footer { + overflow: hidden; + margin: 16px; +} + +.guidance-footer .guidance-field + .guidance-field { + border-left: 1px solid var(--huddle-line); +} + +.guidance-move { + background: #f5eef1; +} + .finding-intro h3, .report-context h3, .teacher-evidence h3 { @@ -957,6 +1095,29 @@ blockquote { grid-template-columns: 1fr; } + .guidance-fork, + .guidance-footer { + grid-template-columns: 1fr; + } + + .guidance-fork { + gap: 10px; + padding: 14px; + } + + .guidance-fork::before { + display: none; + } + + .guidance-footer { + margin: 14px; + } + + .guidance-footer .guidance-field + .guidance-field { + border-top: 1px solid var(--huddle-line); + border-left: 0; + } + .teacher-evidence-item { grid-template-columns: 1fr; gap: 6px; diff --git a/apps/web/lib/operations.ts b/apps/web/lib/operations.ts index 6786861..eb1cbb1 100644 --- a/apps/web/lib/operations.ts +++ b/apps/web/lib/operations.ts @@ -12,7 +12,12 @@ import { } from '@huddle/application'; import type { Attempt, EvidenceBundle, LearningSession } from '@huddle/core'; import { items, skillPrereqs, skills } from '@huddle/core/seed'; -import { deterministicFallback } from '@huddle/narrator/catalog.js'; +import { + NARRATION_TIMEOUT_MS, + Narrator, + deterministicFallback, + type TeacherGuidanceDraft, +} from '@huddle/narrator'; import { allRules, compareSignals, runEngine } from '@huddle/signal-engine'; import { RULE_CONFIG } from '@huddle/signal-engine/config/thresholds.js'; import { @@ -30,6 +35,7 @@ import { type PoolClient, } from '@huddle/db'; import type { GuideAccess } from '@huddle/application'; +import { mayDraftTeacherGuidance } from './teacher-guidance-policy'; /** Server-only composition. Browser inputs provide bytes only; roster/scope comes from GuideAccess. */ export async function importerFor(access: GuideAccess): Promise { @@ -47,6 +53,33 @@ export async function importerFor(access: GuideAccess): Promise { ); } +type GeneratedGuidance = { draft: TeacherGuidanceDraft; fingerprint: string }; +type GuidanceOutcome = { + generated: GeneratedGuidance | null; + degradedReason: 'model-unavailable' | 'timeout' | 'provider-error' | 'grounding-rejected'; +}; + +async function manualSyntheticGuidance(bundle: EvidenceBundle): Promise { + if (!process.env.ANTHROPIC_API_KEY) return null; + const narrator = new Narrator(); + let timeoutId: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timeoutId = setTimeout( + () => reject(Object.assign(new Error('Narration timed out.'), { code: 'timeout' })), + NARRATION_TIMEOUT_MS + ); + }); + try { + const draft = await Promise.race([narrator.render(bundle), timeout]); + return { + draft, + fingerprint: createHash('sha256').update(JSON.stringify(draft)).digest('hex'), + }; + } finally { + if (timeoutId) clearTimeout(timeoutId); + } +} + async function compileExactSnapshot( claim: Parameters[0]['publish']>>[0], client: PoolClient @@ -169,6 +202,30 @@ async function compileExactSnapshot( .digest('hex'); return fallback; }); + const generatedBySignal: GuidanceOutcome[] = []; + for (const fallback of fallbackBySignal) { + if (!mayDraftTeacherGuidance(claim.trigger, process.env.ANTHROPIC_API_KEY)) { + generatedBySignal.push({ generated: null, degradedReason: 'model-unavailable' }); + continue; + } + try { + const generated = await manualSyntheticGuidance(fallback.bundle); + generatedBySignal.push({ + generated, + degradedReason: 'model-unavailable', + }); + } catch (error) { + const message = error instanceof Error ? error.message : ''; + generatedBySignal.push({ + generated: null, + degradedReason: message.includes('timed out') + ? 'timeout' + : message.includes('bounded packet') + ? 'grounding-rejected' + : 'provider-error', + }); + } + } const byStudent = new Map(); engineSignals.forEach((signal, index) => byStudent.set(signal.studentId, [...(byStudent.get(signal.studentId) ?? []), index]) @@ -201,6 +258,10 @@ async function compileExactSnapshot( fallbackNarrationSelection: fallback.selection, fallbackDiagnosis: fallback.diagnosis, fallbackOpener: fallback.opener, + fallbackConfidentFollowUp: fallback.confidentFollowUp, + fallbackStrugglingFollowUp: fallback.strugglingFollowUp, + fallbackListenFor: fallback.listenFor, + fallbackTeachingMove: fallback.teachingMove, windowStart: signal.windowStart, windowEnd: signal.windowEnd, computedAt: signal.computedAt, @@ -208,6 +269,8 @@ async function compileExactSnapshot( }); const entries: CompiledEntry[] = grouped.map(({ studentId, dominant, additional }, index) => { const language = fallbackBySignal[dominant]!; + const outcome = generatedBySignal[dominant]!; + const generated = outcome.generated; return { studentId, dominant, @@ -223,12 +286,19 @@ async function compileExactSnapshot( ) .digest('hex'), catalogVersion: language.catalogVersion, - renderVersion: language.renderVersion, - languageFingerprint: language.languageFingerprint, - narrationFingerprint: language.narrationFingerprint, - narrationSelection: language.selection, - diagnosis: language.diagnosis, - opener: language.opener, + renderVersion: generated ? 'manual-teacher-guidance-v1' : language.renderVersion, + languageFingerprint: generated ? generated.fingerprint : language.languageFingerprint, + narrationFingerprint: generated ? generated.fingerprint : language.narrationFingerprint, + narrationSelection: generated ? { generated: generated.draft } : language.selection, + narrationMode: generated ? 'generated' : 'deterministic-fallback', + narrationStatus: generated ? 'complete' : 'degraded', + narrationDegradedReason: generated ? null : outcome.degradedReason, + diagnosis: generated?.draft.likelyExplanation ?? language.diagnosis, + opener: generated?.draft.askFirst ?? language.opener, + confidentFollowUp: language.confidentFollowUp, + strugglingFollowUp: language.strugglingFollowUp, + listenFor: language.listenFor, + teachingMove: generated?.draft.suggestedTeachingMove ?? language.teachingMove, }; }); const mastery: CompiledMastery[] = snapshot.currentMastery.map((row) => ({ diff --git a/apps/web/lib/teacher-guidance-policy.ts b/apps/web/lib/teacher-guidance-policy.ts new file mode 100644 index 0000000..806de6b --- /dev/null +++ b/apps/web/lib/teacher-guidance-policy.ts @@ -0,0 +1,7 @@ +/** The portfolio's only model path: an authenticated manual refresh of synthetic data. */ +export function mayDraftTeacherGuidance( + trigger: 'manual' | 'nightly', + apiKey: string | undefined +): boolean { + return trigger === 'manual' && Boolean(apiKey); +} diff --git a/apps/web/test/evidence-desk-rendering.test.ts b/apps/web/test/evidence-desk-rendering.test.ts index e5e40ee..75262f4 100644 --- a/apps/web/test/evidence-desk-rendering.test.ts +++ b/apps/web/test/evidence-desk-rendering.test.ts @@ -9,6 +9,7 @@ import { ReportContext, ReportHeading, StatusCue, + TeacherGuidanceCard, TeacherEvidenceList, refreshDescription, } from '../app/board/evidence-desk'; @@ -32,6 +33,10 @@ const entry = { finalConfidence: 0.5796, diagnosis: 'Evidence-backed diagnosis.', opener: 'Show me your first step.', + confidentFollowUp: 'Ask for one more example.', + strugglingFollowUp: 'Work through one opening step.', + listenFor: 'A connection between the strategy and problem.', + teachingMove: 'Model one useful next step.', narration: { mode: 'deterministic-fallback', status: 'degraded', @@ -216,6 +221,26 @@ describe('Evidence Desk teacher presentation', () => { expect(html).toContain('aria-label="Rank #1 today. Urgent priority. Medium confidence."'); }); + it('renders the selected Variant B teacher decision map with all six fields in contract order', () => { + const html = renderToStaticMarkup(createElement(TeacherGuidanceCard, { entry })); + const positions = [ + 'Likely explanation to test', + 'Ask first', + 'If they explain confidently', + 'If they struggle', + 'Listen for', + 'Suggested teaching move', + ].map((label) => html.indexOf(label)); + + expect(positions.every((position) => position >= 0)).toBe(true); + expect(positions).toEqual([...positions].sort((a, b) => a - b)); + expect(html).toContain('AI-assisted conversation plan'); + expect(html).toContain('AI-drafted wording'); + expect(html).toContain('Teacher support'); + expect(html).not.toContain(' { const report = teacherReportView(evidence); const html = renderToStaticMarkup( diff --git a/apps/web/test/evidence-desk-state.test.ts b/apps/web/test/evidence-desk-state.test.ts index 5dd9cc4..8958cc6 100644 --- a/apps/web/test/evidence-desk-state.test.ts +++ b/apps/web/test/evidence-desk-state.test.ts @@ -36,6 +36,10 @@ const board = (boardRunId = 'run-1'): Exclude finalConfidence: 0.9, diagnosis: 'Evidence-backed diagnosis.', opener: 'Show me your first step.', + confidentFollowUp: 'Ask for one more example.', + strugglingFollowUp: 'Work through one opening step.', + listenFor: 'A connection between the strategy and problem.', + teachingMove: 'Model one useful next step.', narration: { mode: 'deterministic-fallback', status: 'degraded', diff --git a/apps/web/test/teacher-guidance-policy.test.ts b/apps/web/test/teacher-guidance-policy.test.ts new file mode 100644 index 0000000..e154d9d --- /dev/null +++ b/apps/web/test/teacher-guidance-policy.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, it } from 'vitest'; +import { mayDraftTeacherGuidance } from '../lib/teacher-guidance-policy'; + +describe('manual synthetic teacher-guidance policy', () => { + it('permits the model only for a manual refresh with a configured key', () => { + expect(mayDraftTeacherGuidance('manual', 'key')).toBe(true); + expect(mayDraftTeacherGuidance('manual', undefined)).toBe(false); + expect(mayDraftTeacherGuidance('nightly', 'key')).toBe(false); + }); +}); diff --git a/db/migrations/011_teacher_guidance_card.sql b/db/migrations/011_teacher_guidance_card.sql new file mode 100644 index 0000000..15954cc --- /dev/null +++ b/db/migrations/011_teacher_guidance_card.sql @@ -0,0 +1,27 @@ +-- Narrow teacher-guidance card: three AI-drafted fields plus deterministic support/fallback fields. +ALTER TABLE signal + ADD COLUMN fallback_confident_follow_up text NOT NULL DEFAULT 'Ask the student to explain the strategy on one new problem.', + ADD COLUMN fallback_struggling_follow_up text NOT NULL DEFAULT 'Work through one opening step together.', + ADD COLUMN fallback_listen_for text NOT NULL DEFAULT 'A clear connection between the strategy and the problem.', + ADD COLUMN fallback_teaching_move text NOT NULL DEFAULT 'Name one useful next step before continuing.'; + +ALTER TABLE triage_entry + ADD COLUMN confident_follow_up text NOT NULL DEFAULT 'Ask the student to explain the strategy on one new problem.', + ADD COLUMN struggling_follow_up text NOT NULL DEFAULT 'Work through one opening step together.', + ADD COLUMN listen_for text NOT NULL DEFAULT 'A clear connection between the strategy and the problem.', + ADD COLUMN suggested_teaching_move text NOT NULL DEFAULT 'Name one useful next step before continuing.'; + +ALTER TABLE signal + ADD CONSTRAINT signal_fallback_guidance_complete CHECK ( + length(btrim(fallback_confident_follow_up)) > 0 + AND length(btrim(fallback_struggling_follow_up)) > 0 + AND length(btrim(fallback_listen_for)) > 0 + AND length(btrim(fallback_teaching_move)) > 0 + ); +ALTER TABLE triage_entry + ADD CONSTRAINT triage_entry_guidance_complete CHECK ( + length(btrim(confident_follow_up)) > 0 + AND length(btrim(struggling_follow_up)) > 0 + AND length(btrim(listen_for)) > 0 + AND length(btrim(suggested_teaching_move)) > 0 + ); diff --git a/packages/application/src/board-reader.ts b/packages/application/src/board-reader.ts index fafa8fe..bbaeb9b 100644 --- a/packages/application/src/board-reader.ts +++ b/packages/application/src/board-reader.ts @@ -43,8 +43,16 @@ export interface BoardEntryView { scope: EvidenceScope; severity: number; finalConfidence: number; + /** AI-drafted likely explanation to test (or deterministic fallback). */ diagnosis: string; + /** AI-drafted ask first (or deterministic fallback). */ opener: string; + /** Cause-specific deterministic support. */ + confidentFollowUp: string; + strugglingFollowUp: string; + listenFor: string; + /** AI-drafted suggested teaching move (or deterministic fallback). */ + teachingMove: string; narration: NarrationView; acknowledgedAt: string | null; additionalCauseCount: number; diff --git a/packages/application/test/evidence-reader-implementation.test.ts b/packages/application/test/evidence-reader-implementation.test.ts index 9d9d279..2b34298 100644 --- a/packages/application/test/evidence-reader-implementation.test.ts +++ b/packages/application/test/evidence-reader-implementation.test.ts @@ -49,6 +49,10 @@ const evidence: EvidenceView = { finalConfidence: 0.9, diagnosis: 'A deterministic diagnosis.', opener: 'Show me your first step.', + confidentFollowUp: 'Ask for one more example.', + strugglingFollowUp: 'Work through one opening step.', + listenFor: 'A connection between the strategy and problem.', + teachingMove: 'Model one useful next step.', narration: { mode: 'deterministic-fallback', status: 'degraded', diff --git a/packages/db/src/boards.ts b/packages/db/src/boards.ts index 98014db..b121c66 100644 --- a/packages/db/src/boards.ts +++ b/packages/db/src/boards.ts @@ -23,6 +23,10 @@ export type CompiledSignal = { fallbackNarrationSelection: unknown; fallbackDiagnosis: string; fallbackOpener: string; + fallbackConfidentFollowUp: string; + fallbackStrugglingFollowUp: string; + fallbackListenFor: string; + fallbackTeachingMove: string; windowStart: Date; windowEnd: Date; computedAt: Date; @@ -35,6 +39,13 @@ export type CompiledEntry = { additional: number[]; diagnosis: string; opener: string; + confidentFollowUp: string; + strugglingFollowUp: string; + listenFor: string; + teachingMove: string; + narrationMode: 'generated' | 'deterministic-fallback'; + narrationStatus: 'complete' | 'degraded'; + narrationDegradedReason: string | null; catalogVersion: string; renderVersion: string; languageFingerprint: string; @@ -206,9 +217,10 @@ export async function publishCompiledBoardRun( behavior_fingerprint,intensity,severity,raw_confidence,final_confidence,confidence_breakdown, evidence,evidence_fingerprint,fallback_catalog_version,fallback_render_version, fallback_language_fingerprint,fallback_narration_fingerprint,fallback_narration_selection, - fallback_diagnosis,fallback_opener,window_start,window_end,computed_at) + fallback_diagnosis,fallback_opener,fallback_confident_follow_up,fallback_struggling_follow_up, + fallback_listen_for,fallback_teaching_move,window_start,window_end,computed_at) VALUES($1::bigint,$2::uuid,$3::scope_kind,$4,$5::root_cause,$6,$7,$8,$9,$9,$10,$11, - '{}'::jsonb,$12::jsonb,$13,$14,$15,$16,$17,$18::jsonb,$19,$20,$21,$22,$23) + '{}'::jsonb,$12::jsonb,$13,$14,$15,$16,$17,$18::jsonb,$19,$20,$21,$22,$23,$24,$25,$26,$27) RETURNING id::text`, [ run.rows[0]!.id, @@ -231,6 +243,10 @@ export async function publishCompiledBoardRun( JSON.stringify(s.fallbackNarrationSelection), s.fallbackDiagnosis, s.fallbackOpener, + s.fallbackConfidentFollowUp, + s.fallbackStrugglingFollowUp, + s.fallbackListenFor, + s.fallbackTeachingMove, s.windowStart, s.windowEnd, s.computedAt, @@ -255,10 +271,12 @@ export async function publishCompiledBoardRun( await client.query( `INSERT INTO triage_entry(board_run_id,student_id,dominant_signal_id,finding_fingerprint,rank, additional_causes,catalog_version,language_fingerprint,expected_narration_fingerprint, - narration_selection,narration_mode,narration_status,narration_degraded_reason,render_version, - diagnosis,opener,language_rendered_at) - VALUES($1::bigint,$2::uuid,$3::bigint,$4,$5,$6::jsonb,$7,$8,$9,$10::jsonb, - 'deterministic_fallback','degraded','model-unavailable',$11,$12,$13,now())`, + narration_fingerprint,narration_selection,narration_mode,narration_status,narration_degraded_reason, + render_version,diagnosis,opener,confident_follow_up,struggling_follow_up,listen_for, + suggested_teaching_move,language_rendered_at,narration_completed_at) + VALUES($1::bigint,$2::uuid,$3::bigint,$4,$5,$6::jsonb,$7,$8,$9,$10,$11::jsonb, + $12::narration_mode,$13::narration_status,$14::narration_degraded_reason,$15,$16,$17,$18,$19,$20,$21,now(), + CASE WHEN $12='generated' THEN now() ELSE NULL END)`, [ run.rows[0]!.id, e.studentId, @@ -269,10 +287,18 @@ export async function publishCompiledBoardRun( e.catalogVersion, e.languageFingerprint, e.narrationFingerprint, + e.narrationMode === 'generated' ? e.narrationFingerprint : null, JSON.stringify(e.narrationSelection), + e.narrationMode === 'deterministic-fallback' ? 'deterministic_fallback' : 'generated', + e.narrationStatus, + e.narrationDegradedReason, e.renderVersion, e.diagnosis, e.opener, + e.confidentFollowUp, + e.strugglingFollowUp, + e.listenFor, + e.teachingMove, ] ); } diff --git a/packages/db/src/scoped.ts b/packages/db/src/scoped.ts index e53129e..1a5086a 100644 --- a/packages/db/src/scoped.ts +++ b/packages/db/src/scoped.ts @@ -36,6 +36,10 @@ export interface TriageBoardRow { skillName: string; diagnosis: string | null; opener: string | null; + confidentFollowUp: string | null; + strugglingFollowUp: string | null; + listenFor: string | null; + teachingMove: string | null; generatedAt: Date | null; additionalCauses: Array<{ cause: RootCause; severity: number }>; } @@ -159,6 +163,10 @@ export async function getTriageBoardForGuide( COALESCE(sk.id || ' — ' || sk.name, '—') AS "skillName", te.diagnosis, te.opener, + te.confident_follow_up AS "confidentFollowUp", + te.struggling_follow_up AS "strugglingFollowUp", + te.listen_for AS "listenFor", + te.suggested_teaching_move AS "teachingMove", te.language_rendered_at AS "generatedAt", te.additional_causes AS "additionalCauses" FROM board_head head @@ -246,6 +254,10 @@ export async function getBoardViewForGuide( final_confidence: number; diagnosis: string; opener: string; + confident_follow_up: string; + struggling_follow_up: string; + listen_for: string; + suggested_teaching_move: string; narration_mode: 'generated' | 'deterministic_fallback'; narration_status: 'complete' | 'degraded'; narration_degraded_reason: @@ -266,7 +278,8 @@ export async function getBoardViewForGuide( `SELECT entry.id::text AS triage_entry_id,entry.finding_fingerprint, student.id::text AS student_id,student.first_name,entry.rank,signal.kind AS cause, signal.skill_id,skill.name AS skill_name,signal.severity,signal.final_confidence, - entry.diagnosis,entry.opener,entry.narration_mode,entry.narration_status, + entry.diagnosis,entry.opener,entry.confident_follow_up,entry.struggling_follow_up, + entry.listen_for,entry.suggested_teaching_move,entry.narration_mode,entry.narration_status, entry.narration_degraded_reason,entry.catalog_version,entry.render_version, acknowledgement.acknowledged_at, jsonb_array_length(entry.additional_causes) AS additional_cause_count, @@ -300,6 +313,10 @@ export async function getBoardViewForGuide( finalConfidence: row.final_confidence, diagnosis: row.diagnosis, opener: row.opener, + confidentFollowUp: row.confident_follow_up, + strugglingFollowUp: row.struggling_follow_up, + listenFor: row.listen_for, + teachingMove: row.suggested_teaching_move, narration: { mode: row.narration_mode === 'deterministic_fallback' diff --git a/packages/db/test/operations-migrations.test.ts b/packages/db/test/operations-migrations.test.ts index 140f80a..5852486 100644 --- a/packages/db/test/operations-migrations.test.ts +++ b/packages/db/test/operations-migrations.test.ts @@ -8,6 +8,9 @@ const migration8 = fileURLToPath( const migration9 = fileURLToPath( new URL('../../../db/migrations/009_board_run_head.sql', import.meta.url) ); +const migration11 = fileURLToPath( + new URL('../../../db/migrations/011_teacher_guidance_card.sql', import.meta.url) +); describe('Operations migration contract', () => { it('orders receipt/session lineage before immutable board publication and keeps raw rejected rows out', async () => { @@ -46,7 +49,10 @@ describe('Operations migration contract', () => { }); it('makes runs/heads immutable and requires deterministic fallback text before publication', async () => { - const nine = await readFile(migration9, 'utf8'); + const [nine, eleven] = await Promise.all([ + readFile(migration9, 'utf8'), + readFile(migration11, 'utf8'), + ]); expect(nine).toMatch( /CREATE TABLE board_run[\s\S]*input_receipt_set_fingerprint[\s\S]*behavior_fingerprint/ ); @@ -63,5 +69,10 @@ describe('Operations migration contract', () => { 'FOREIGN KEY (current_board_run_id, application_scope_id, guide_id, studio_id, board_date)' ); expect(nine).toContain('ALTER TABLE mastery_snapshot RENAME TO legacy_mastery_snapshot'); + expect(eleven).toMatch( + /ADD COLUMN confident_follow_up text NOT NULL[\s\S]*suggested_teaching_move text NOT NULL/ + ); + expect(eleven).toMatch(/fallback_confident_follow_up[\s\S]*fallback_teaching_move/); + expect(eleven).toContain('triage_entry_guidance_complete'); }); }); diff --git a/packages/narrator/src/batch.ts b/packages/narrator/src/batch.ts index 07bd800..1d58638 100644 --- a/packages/narrator/src/batch.ts +++ b/packages/narrator/src/batch.ts @@ -1,6 +1,6 @@ import Anthropic from '@anthropic-ai/sdk'; -import type { EvidenceBundle, NarrationResult } from '@huddle/core'; -import { makeNarrationRequest, parseNarration } from './request.js'; +import type { EvidenceBundle } from '@huddle/core'; +import { makeNarrationRequest, parseNarration, type TeacherGuidanceDraft } from './request.js'; const POLL_INTERVAL_MS = 2000; const MAX_POLL_INTERVAL_MS = 60_000; @@ -12,7 +12,7 @@ export interface BatchEntry { bundle: EvidenceBundle; } -export type BatchResult = { customId: string; result: NarrationResult | null; error?: string }; +export type BatchResult = { customId: string; result: TeacherGuidanceDraft | null; error?: string }; /** Contract identity: a provider custom_id is exactly the persisted signal id. */ export function customIdForSignal(signalId: number): string { @@ -76,7 +76,10 @@ export async function renderBatch( })); } - const resultsByCustomId = new Map(); + const bundleByCustomId = new Map( + entries.map((entry) => [customIdForSignal(entry.signalId), entry.bundle]) + ); + const resultsByCustomId = new Map(); const errorsByCustomId = new Map(); const decoder = await client.beta.messages.batches.results(batch.id); for await (const line of decoder) { @@ -90,7 +93,9 @@ export async function renderBatch( continue; } try { - resultsByCustomId.set(line.custom_id, parseNarration(JSON.parse(text.text))); + const bundle = bundleByCustomId.get(line.custom_id); + if (!bundle) throw new Error('batch result does not match a requested evidence bundle'); + resultsByCustomId.set(line.custom_id, parseNarration(bundle, JSON.parse(text.text))); } catch (error) { errorsByCustomId.set( line.custom_id, diff --git a/packages/narrator/src/catalog.ts b/packages/narrator/src/catalog.ts index a505a40..5c5cd61 100644 --- a/packages/narrator/src/catalog.ts +++ b/packages/narrator/src/catalog.ts @@ -16,8 +16,16 @@ export interface NarrationSelection { export interface DeterministicNarration { bundle: EvidenceBundle; selection: NarrationSelection; + /** Deterministic fallback for AI-drafted likely explanation to test. */ diagnosis: string; + /** Deterministic fallback for AI-drafted ask first. */ opener: string; + /** Deterministic branch/listening support; never drafted by AI. */ + confidentFollowUp: string; + strugglingFollowUp: string; + listenFor: string; + /** Deterministic fallback for AI-drafted suggested teaching move. */ + teachingMove: string; catalogVersion: string; renderVersion: string; languageFingerprint: string; @@ -43,6 +51,65 @@ const option = (id: string, template: string, eligibleCause: RootCause): Catalog slots: studentSlot, }); +const SUPPORT: Record< + RootCause, + Pick< + DeterministicNarration, + 'confidentFollowUp' | 'strugglingFollowUp' | 'listenFor' | 'teachingMove' + > +> = { + guessing: { + confidentFollowUp: 'Ask the student to explain the strategy on one new problem.', + strugglingFollowUp: + 'Invite the student to name one choice they can rule out before trying again.', + listenFor: 'A reason connected to the problem instead of a quick guess.', + teachingMove: 'Pause for one worked think-aloud before the next independent attempt.', + }, + prerequisite_gap: { + confidentFollowUp: 'Ask the student to use the prerequisite in a nearby example.', + strugglingFollowUp: 'Rebuild the prerequisite with one concrete example and name each step.', + listenFor: 'Whether the student can connect the prerequisite to the current problem.', + teachingMove: 'Use a brief prerequisite example, then return to the current task.', + }, + grinding: { + confidentFollowUp: 'Ask the student to name the changed strategy before another attempt.', + strugglingFollowUp: 'Choose one different strategy together before retrying.', + listenFor: 'A specific change in approach rather than repeating the same attempt.', + teachingMove: 'Model one alternate strategy, then release the next step to the student.', + }, + hint_farming: { + confidentFollowUp: 'Ask the student to explain an opening step without a hint.', + strugglingFollowUp: 'Cover the hint and identify one clue already present in the problem.', + listenFor: 'An independently chosen first step.', + teachingMove: 'Prompt for one independent step before offering another hint.', + }, + no_read_retry: { + confidentFollowUp: 'Ask the student to name what changed after reading the feedback.', + strugglingFollowUp: + 'Read the feedback together and point to one detail that changes the next attempt.', + listenFor: 'A connection between the feedback and a changed response.', + teachingMove: 'Make the feedback-to-next-step connection explicit before retrying.', + }, + decay: { + confidentFollowUp: 'Ask the student to apply the earlier idea in the current problem.', + strugglingFollowUp: 'Revisit one earlier example and identify the part that transfers.', + listenFor: 'Recognition of the earlier skill in the present task.', + teachingMove: 'Use one short retrieval example before continuing.', + }, + disengagement: { + confidentFollowUp: 'Ask the student to choose a manageable next step and explain why.', + strugglingFollowUp: 'Offer two small next-step choices and let the student select one.', + listenFor: 'A concrete next step the student is willing to try.', + teachingMove: 'Reduce the next task to one visible, manageable action.', + }, + fine: { + confidentFollowUp: 'Ask the student to explain the strategy they would use next.', + strugglingFollowUp: 'Work through one opening step together.', + listenFor: 'A clear connection between the strategy and the problem.', + teachingMove: 'Name the successful strategy before moving on.', + }, +}; + const CATALOG: Record = { guessing: { diagnosis: option( @@ -160,16 +227,24 @@ export function deterministicFallback(bundle: EvidenceBundle): DeterministicNarr }, }; const rendered = validateAndRenderSelection(groundedBundle, selection); + const guidance = { + ...rendered, + diagnosis: + bundle.finding.finalConfidence < 0.55 + ? `Working hypothesis: ${rendered.diagnosis}` + : rendered.diagnosis, + }; const languageFingerprint = digest({ catalogVersion: FALLBACK_CATALOG_VERSION, renderVersion: FALLBACK_RENDER_VERSION, selection, - ...rendered, + ...guidance, }); return { bundle: groundedBundle, selection, - ...rendered, + ...guidance, + ...SUPPORT[bundle.finding.dominantCause], catalogVersion: FALLBACK_CATALOG_VERSION, renderVersion: FALLBACK_RENDER_VERSION, languageFingerprint, diff --git a/packages/narrator/src/index.ts b/packages/narrator/src/index.ts index a6533fc..d762d6a 100644 --- a/packages/narrator/src/index.ts +++ b/packages/narrator/src/index.ts @@ -5,9 +5,13 @@ export { NARRATION_MODEL, NARRATION_SYSTEM, NarrationSchema, + NARRATION_TIMEOUT_MS, makeNarrationRequest, + parseNarration, } from './request.js'; -export { SYSTEM_PROMPT, makeUserMessage } from './prompt.js'; +export type { TeacherGuidanceDraft } from './request.js'; +export { SYSTEM_PROMPT, makeUserMessage, teacherGuidancePacket } from './prompt.js'; +export type { TeacherGuidancePacket } from './prompt.js'; export { FALLBACK_CATALOG_VERSION, FALLBACK_RENDER_VERSION, diff --git a/packages/narrator/src/prompt.ts b/packages/narrator/src/prompt.ts index 20be40f..b1a7409 100644 --- a/packages/narrator/src/prompt.ts +++ b/packages/narrator/src/prompt.ts @@ -1,27 +1,54 @@ import type { EvidenceBundle } from '@huddle/core'; -// Frozen system prompt. No dates, ids, or volatile content: it is the cacheable prefix. -export const SYSTEM_PROMPT = `You are a guide's assistant in a K-8 learning environment. Your job is to turn the attached evidence bundle into two short strings: +export type TeacherGuidancePacket = { + student: { firstName: string }; + primaryProblem: string; + relatedAttempts: Array<{ + skill: string; + result: 'correct' | 'incorrect'; + chosenAnswer: string | null; + misconception: string | null; + hintsUsed: number; + }>; + prerequisite: string | null; + uncertainty: 'strong-evidence' | 'working-hypothesis'; +}; -1. diagnosis — one or two sentences describing what is happening and why. -2. opener — one sentence the adult can say to the student. +// Frozen and intentionally limited: only these three teacher-facing fields are AI drafted. +export const SYSTEM_PROMPT = `You draft exactly three short pieces of teacher-facing wording for a K-8 guide: +1. likelyExplanation — a tentative explanation to test, not a diagnosis. +2. askFirst — one question the teacher can ask first. +3. suggestedTeachingMove — one concise teaching move. -Root causes you may reference: guessing, prerequisite_gap, grinding, hint_farming, no_read_retry, decay, disengagement, fine. +The teacher reviews this draft before using it. Do not address the student directly except in askFirst. Do not write follow-up branches or listen-for guidance; deterministic support supplies those fields. -Rules: -- Cite ONLY facts that appear in the evidence bundle. If a number, date, skill code, or student attribute is not in the bundle, do not mention it. -- Do not perform arithmetic. Restate numbers from the bundle, or say fewer numbers. -- Use the student's first name only. Do not use last names, full names, or other student attributes. -- Be specific, calm, and actionable. Keep opener under 25 words and diagnosis under 40 words. +Use only the attached packet. Do not invent quantities, dates, skill names, student attributes, or claims. Keep language calm and nonjudgmental. askFirst must end with a question mark. If uncertainty is working-hypothesis, likelyExplanation must begin "Working hypothesis:".`; -Worked example: -{ "diagnosis": "Lily is answering multiplication problems much faster than her own correct-answer baseline while getting most wrong; this looks like guessing.", "opener": "Lily, I see some quick answers here — walk me through how you picked this one." } -`; +/** Extract only the bounded current evidence allowed to reach the model. */ +export function teacherGuidancePacket(bundle: EvidenceBundle): TeacherGuidancePacket { + return { + student: { firstName: bundle.student.firstName }, + primaryProblem: + bundle.scope.kind === 'skill' + ? `${bundle.scope.skill.code} · ${bundle.scope.skill.name}` + : 'Across skills', + relatedAttempts: bundle.attempts.slice(-3).map((attempt) => ({ + skill: `${attempt.skill.code} · ${attempt.skill.name}`, + result: attempt.isCorrect ? 'correct' : 'incorrect', + chosenAnswer: attempt.chosenLabel, + misconception: attempt.misconception, + hintsUsed: attempt.hintsUsed, + })), + prerequisite: bundle.prerequisiteCheck + ? `${bundle.prerequisiteCheck.skillCode} · ${bundle.prerequisiteCheck.skillName}` + : null, + uncertainty: bundle.finding.finalConfidence < 0.55 ? 'working-hypothesis' : 'strong-evidence', + }; +} -/** Sort object keys at every depth while preserving array order for canonical bundle bytes. */ +/** Sort object keys at every depth while preserving array order for canonical packet bytes. */ export function sortDeep(value: unknown): unknown { if (Array.isArray(value)) return value.map(sortDeep); - // A Date has no own enumerable keys, so treating it as a record would erase it to `{}`. if (value instanceof Date) return value.toISOString(); if (value !== null && typeof value === 'object') { const record = value as Record; @@ -34,7 +61,7 @@ export function sortDeep(value: unknown): unknown { return value; } -/** The sole model input: one complete, canonically serialized evidence bundle. */ +/** The sole model input: a bounded, canonically serialized teacher-guidance packet. */ export function makeUserMessage(bundle: EvidenceBundle): string { - return JSON.stringify(sortDeep(bundle)); + return JSON.stringify(sortDeep(teacherGuidancePacket(bundle))); } diff --git a/packages/narrator/src/render.ts b/packages/narrator/src/render.ts index f206f09..aee005d 100644 --- a/packages/narrator/src/render.ts +++ b/packages/narrator/src/render.ts @@ -1,6 +1,6 @@ import Anthropic from '@anthropic-ai/sdk'; -import type { EvidenceBundle, NarrationResult } from '@huddle/core'; -import { makeNarrationRequest, NarrationSchema } from './request.js'; +import type { EvidenceBundle } from '@huddle/core'; +import { makeNarrationRequest, parseNarration, type TeacherGuidanceDraft } from './request.js'; export class Narrator { private readonly client: Anthropic; @@ -11,10 +11,10 @@ export class Narrator { this.client = new Anthropic({ apiKey: key }); } - async render(bundle: EvidenceBundle): Promise { + async render(bundle: EvidenceBundle): Promise { const response = await this.client.messages.parse(makeNarrationRequest(bundle)); if (response.parsed_output == null) throw new Error('Narration response did not contain structured output'); - return NarrationSchema.parse(response.parsed_output); + return parseNarration(bundle, response.parsed_output); } } diff --git a/packages/narrator/src/request.ts b/packages/narrator/src/request.ts index 9d329a6..ededea6 100644 --- a/packages/narrator/src/request.ts +++ b/packages/narrator/src/request.ts @@ -1,24 +1,24 @@ import { zodOutputFormat } from '@anthropic-ai/sdk/helpers/zod'; import { z } from 'zod/v4'; import type { EvidenceBundle } from '@huddle/core'; -import { makeUserMessage, SYSTEM_PROMPT } from './prompt.js'; +import { makeUserMessage, SYSTEM_PROMPT, teacherGuidancePacket } from './prompt.js'; -export const NarrationSchema = z.object({ - diagnosis: z.string(), - opener: z.string(), -}); +export const NarrationSchema = z + .object({ + likelyExplanation: z.string(), + askFirst: z.string(), + suggestedTeachingMove: z.string(), + }) + .strict(); +export type TeacherGuidanceDraft = z.infer; -/** Pinned for reproducible generation/evaluation; overrides are intentionally not supported. */ +/** Pinned for reproducible synthetic manual-refresh generation; overrides are intentionally unsupported. */ export const NARRATION_MODEL = 'claude-opus-5'; -/** Thinking plus two guide-visible strings need materially more than a string-only budget. */ -export const NARRATION_MAX_TOKENS = 4096; +export const NARRATION_MAX_TOKENS = 1024; +export const NARRATION_TIMEOUT_MS = 12_000; export const NARRATION_SYSTEM = [ - { - type: 'text' as const, - text: SYSTEM_PROMPT, - cache_control: { type: 'ephemeral' as const }, - }, + { type: 'text' as const, text: SYSTEM_PROMPT, cache_control: { type: 'ephemeral' as const } }, ]; export function makeNarrationRequest(bundle: EvidenceBundle) { @@ -26,14 +26,41 @@ export function makeNarrationRequest(bundle: EvidenceBundle) { model: NARRATION_MODEL, max_tokens: NARRATION_MAX_TOKENS, system: NARRATION_SYSTEM, - output_config: { - effort: 'low' as const, - format: zodOutputFormat(NarrationSchema), - }, + output_config: { effort: 'low' as const, format: zodOutputFormat(NarrationSchema) }, messages: [{ role: 'user' as const, content: makeUserMessage(bundle) }], }; } -export function parseNarration(value: unknown) { - return NarrationSchema.parse(value); +export function parseNarration(bundle: EvidenceBundle, value: unknown): TeacherGuidanceDraft { + const draft = NarrationSchema.parse(value); + const packet = teacherGuidancePacket(bundle); + const fields = [draft.likelyExplanation, draft.askFirst, draft.suggestedTeachingMove]; + if (fields.some((field) => wordCount(field) === 0 || wordCount(field) > 45)) reject(); + if (!draft.askFirst.trim().endsWith('?')) reject(); + if ( + packet.uncertainty === 'working-hypothesis' && + !draft.likelyExplanation.startsWith('Working hypothesis:') + ) + reject(); + if (fields.some((field) => /\b(?:lazy|stupid|careless|unmotivated|bad at)\b/i.test(field))) + reject(); + const packetText = JSON.stringify(packet); + if ( + fields.some((field) => /\d{4}-\d{2}-\d{2}|\b[A-Z]{2,}[.-]?\d+(?:\.\d+)?[A-Z]?\b/.test(field)) + ) { + const explicit = fields.join(' '); + const matches = + explicit.match(/\d{4}-\d{2}-\d{2}|\b[A-Z]{2,}[.-]?\d+(?:\.\d+)?[A-Z]?\b/g) ?? []; + if (matches.some((value) => !packetText.includes(value))) reject(); + } + const numbers = fields.join(' ').match(/\b\d+(?:\.\d+)?\b/g) ?? []; + if (numbers.some((value) => !packetText.includes(value))) reject(); + return draft; +} + +function wordCount(value: string): number { + return value.match(/[\p{L}\p{N}]+(?:['’][\p{L}\p{N}]*)*/gu)?.length ?? 0; +} +function reject(): never { + throw new Error('Teacher-guidance draft is not permitted by the bounded packet.'); } diff --git a/packages/narrator/test/fallback-catalog.test.ts b/packages/narrator/test/fallback-catalog.test.ts index 583bd89..b6674a6 100644 --- a/packages/narrator/test/fallback-catalog.test.ts +++ b/packages/narrator/test/fallback-catalog.test.ts @@ -79,6 +79,18 @@ describe('deterministic fallback catalog', () => { expect(() => validateFallbackSelection(result.bundle, result.selection)).not.toThrow(); }); + it('labels a low-confidence fallback as a working hypothesis while retaining a complete card', () => { + const weak = deterministicFallback({ + ...bundle('guessing'), + finding: { ...bundle('guessing').finding, finalConfidence: 0.4 }, + }); + expect(weak.diagnosis).toMatch(/^Working hypothesis:/); + expect(weak.confidentFollowUp).not.toBe(''); + expect(weak.strugglingFollowUp).not.toBe(''); + expect(weak.listenFor).not.toBe(''); + expect(weak.teachingMove).not.toBe(''); + }); + it('rejects options that a selection tries to authorize for itself', () => { const result = deterministicFallback(bundle('guessing')); const selfAuthorized = { diff --git a/packages/narrator/test/no-pii.test.ts b/packages/narrator/test/no-pii.test.ts index aa972c5..493658f 100644 --- a/packages/narrator/test/no-pii.test.ts +++ b/packages/narrator/test/no-pii.test.ts @@ -9,7 +9,7 @@ import { import { items, skillPrereqs, skills } from '@huddle/core/seed'; import { allRules, runEngine } from '@huddle/signal-engine'; import { RULE_CONFIG } from '@huddle/signal-engine/config/thresholds.js'; -import { makeUserMessage } from '../src/prompt.js'; +import { makeUserMessage, teacherGuidancePacket } from '../src/prompt.js'; import { makeNarrationRequest } from '../src/request.js'; async function assembledBundle(): Promise { @@ -50,26 +50,25 @@ async function assembledBundle(): Promise { } describe('PII verification', () => { - it('serializes a real assembled bundle without losing nested grounding evidence', async () => { + it('serializes only the bounded teacher-guidance packet from a real assembled bundle', async () => { const bundle = await assembledBundle(); const serialized = makeUserMessage(bundle); + const packet = teacherGuidancePacket(bundle); - expect(JSON.parse(serialized)).toEqual(bundle); + expect(JSON.parse(serialized)).toEqual(packet); expect(makeUserMessage(bundle)).toBe(serialized); - expect(JSON.parse(serialized).student).toEqual({ - id: bundle.student.id, - firstName: bundle.student.firstName, - }); - expect(serialized).not.toMatch(/lastName|guideId|"grade"/); - expect(bundle.attempts.length).toBeGreaterThan(0); - expect(JSON.parse(serialized).attempts).toEqual(bundle.attempts); + expect(packet.student).toEqual({ firstName: bundle.student.firstName }); + expect(packet.relatedAttempts).toHaveLength(Math.min(3, bundle.attempts.length)); + expect(serialized).not.toMatch( + /activityId|attemptId|guideId|studentId|ruleId|severity|confidence|lastName|"grade"/ + ); }); - it('sends only the complete serialized bundle as the narrator user turn', async () => { + it('sends only the bounded serialized packet as the narrator user turn', async () => { const bundle = await assembledBundle(); const request = makeNarrationRequest(bundle); expect(request.messages).toEqual([{ role: 'user', content: makeUserMessage(bundle) }]); - expect(JSON.parse(request.messages[0].content)).toEqual(bundle); + expect(JSON.parse(request.messages[0].content)).toEqual(teacherGuidancePacket(bundle)); }); }); diff --git a/packages/narrator/test/request-contract.test.ts b/packages/narrator/test/request-contract.test.ts index ae84c4d..64fe7c6 100644 --- a/packages/narrator/test/request-contract.test.ts +++ b/packages/narrator/test/request-contract.test.ts @@ -6,6 +6,7 @@ import { NARRATION_MODEL, NARRATION_SYSTEM, makeNarrationRequest, + parseNarration, } from '../src/request.js'; const bundle: EvidenceBundle = { @@ -83,6 +84,34 @@ describe('narration generation contract', () => { expect(request.system.at(-1)?.cache_control).toEqual({ type: 'ephemeral' }); }); + it('accepts only the three AI-drafted fields and requires a weak-evidence hypothesis label', () => { + expect( + parseNarration(bundle, { + likelyExplanation: 'Alex may be choosing an answer without checking the problem.', + askFirst: 'How did you choose this answer?', + suggestedTeachingMove: 'Model one short think-aloud before the next attempt.', + }) + ).toMatchObject({ askFirst: 'How did you choose this answer?' }); + expect(() => + parseNarration( + { ...bundle, finding: { ...bundle.finding, finalConfidence: 0.4 } }, + { + likelyExplanation: 'Alex may be choosing an answer without checking the problem.', + askFirst: 'How did you choose this answer?', + suggestedTeachingMove: 'Model one short think-aloud before the next attempt.', + } + ) + ).toThrow(/bounded packet/); + expect(() => + parseNarration(bundle, { + likelyExplanation: 'Alex may be choosing an answer without checking the problem.', + askFirst: 'How did you choose this answer?', + suggestedTeachingMove: 'Use 99 new examples.', + extra: 'not allowed', + }) + ).toThrow(); + }); + it('uses the signal id as the stable batch identity and keeps the same request contract', () => { const requests = makeBatchRequests([{ signalId: 42, bundle }]); diff --git a/specs/001-huddle-triage-board/contracts/application-interfaces.md b/specs/001-huddle-triage-board/contracts/application-interfaces.md index abb3753..21707de 100644 --- a/specs/001-huddle-triage-board/contracts/application-interfaces.md +++ b/specs/001-huddle-triage-board/contracts/application-interfaces.md @@ -288,19 +288,26 @@ Variant B — **Evidence Desk** is the quick-demo shell: unauthorized, cross-guide, missing, or superseded/unselected runs return the same `not-found` result and expose no student fact. -The teacher-facing report presents: - -1. cause, rank, priority, final confidence, concrete opener, and a concise wording-status cue; -2. personal-baseline comparison and prerequisite check; -3. one accessible **Show exact contributing evidence** disclosure containing only the dominant - finding's deduplicated attempts and sessions: skills, timestamps, results, selected answers, - useful misconceptions, and compact duration context. The presentation names active versus elapsed - time when that distinction is available, renders durations in seconds, keeps missing duration - explicit, and uses record identity only as an unexposed rendering key. +The teacher-facing report presents one fully visible **AI-assisted conversation plan** in Variant B +teacher-decision-map hierarchy: + +1. `likely explanation to test` then `ask first` as the visible stem; +2. `if they explain confidently` and `if they struggle` as the explicit fork; +3. `listen for` and `suggested teaching move` as the closing row. + +The explanation, first question, and teaching move are visibly labeled AI-drafted wording; the fork +and listen-for guidance are visibly labeled deterministic teacher support. The whole card is a +teacher-reviewed draft, not a student instruction or a diagnosis. A weak-evidence explanation is +visibly a `Working hypothesis`. Rank, priority, and confidence stay in the report header. The only +expandable detail is one accessible **Show exact contributing evidence** disclosure containing only +the dominant finding's deduplicated attempts and sessions: skills, timestamps, results, selected +answers, useful misconceptions, and compact duration context. The presentation names active versus +elapsed time when that distinction is available, renders durations in seconds, keeps missing duration +explicit, and uses record identity only as an unexposed rendering key. `EvidenceView.comparison.additionalCauses` and `additionalEvidence` remain intact for deterministic traceability and downstream contracts. The teacher-facing projection does not render them, rule or -quality details, or expandable technical provenance. +quality details, technical provenance, or regeneration controls. Every displayed fact points to a stored record or deterministic bundle path. The report never fetches broader ambient student context. @@ -347,12 +354,14 @@ the finding/evidence fingerprint, the changed report is not silently acknowledge links may disable framework prefetch, but correctness depends on the grant-free `readEntry`/fresh `openEntry` boundary rather than that optimization. -## Narration degradation +## Teacher-guidance degradation -Every board-visible cause has one committed, cause-specific fallback diagnosis and opener mapping in -the same closed catalog used by generated selection. Before board publication, deterministic code -selects the eligible fallback IDs/slots, validates them, and renders a non-empty concrete opener. -Therefore delayed, unavailable, timed-out, or rejected model output never leaves a blank opener. +Every board-visible cause has committed deterministic support for the confident/struggling fork and +listen-for field plus complete deterministic fallback wording for the three AI-drafted fields. Only a +manual refresh of synthetic data may request the strict three-field AI draft from its bounded packet; +nightly refreshes never invoke a model. Before publication, the resulting six-field card is complete. +Therefore a missing key, delayed, unavailable, timed-out, malformed, or rejected model result never +leaves a blank field or changes the deterministic board. Each entry exposes: @@ -367,12 +376,11 @@ export interface NarrationView { } ``` -Fallback language passes the same eligibility, slot, atom, word-count, and traceability gates as -model-selected language. Generated narration may replace a fallback only when board run, entry, -evidence hash, catalog version, and the expected narration fingerprint persisted at publication -still match. The attached result's narration fingerprint is separate provenance. It may update -language and status only; it cannot block publication or change student membership, cause, severity, -confidence, or rank. +The complete deterministic card is available before any manual model attempt. The strict generated +three-field draft is accepted only during that same manual synthetic refresh and is persisted with the +immutable run; it never replaces a published card. Its packet/validation boundary cannot block +publication or change student membership, cause, severity, confidence, rank, primary problem, or +evidence. ## Guide-facing synthetic import diff --git a/specs/001-huddle-triage-board/plan.md b/specs/001-huddle-triage-board/plan.md index 198d3bc..f2d2b3c 100644 --- a/specs/001-huddle-triage-board/plan.md +++ b/specs/001-huddle-triage-board/plan.md @@ -227,10 +227,11 @@ same `BoardCompiler`. There is no public v1 API. `board_refresh_request` owns `idle/queued/running/succeeded/failed` presentation. A compile candidate is invisible until one transaction inserts the immutable `board_run` plus run-scoped mastery, signals, and entries, then updates `board_head`. Failure is recorded separately and leaves the head -unchanged. Deterministic cause-specific fallback narration is present at publication; model-backed -selection may replace only narration/provenance after matching the expected narration fingerprint -persisted at publication and can never move the head or alter ranking, causes, evidence, scope, or -freshness. +unchanged. A complete deterministic cause-specific six-field teacher-guidance card is present at +publication. Only a manual synthetic refresh may draft the likely explanation, first question, and +teaching move from a bounded packet; its strict result is persisted inside that same immutable run. +Nightly refreshes never invoke a model, and no model path can alter ranking, causes, evidence, scope, +or freshness. ### Current implementation reconciliation @@ -250,8 +251,9 @@ The current Evidence Desk presentation is owned by the only the selected immutable head; evidence opens receive signed one-use grants, acquire a current-head reveal lease, and write replay-safe first-open acknowledgments to PostgreSQL. The fixed portfolio corpus verifies deterministic fallback and grounding with zero model calls and no accuracy claim. -Optional model-backed narration attachment, deployment scheduler wiring, and the full -simulator/accuracy gates remain implementation gaps, not alternate contracts. +The manual synthetic-refresh teacher-guidance draft is a constrained portfolio behavior, not an +accuracy claim or a real-data path; deployment scheduler wiring and the full simulator/accuracy gates +remain implementation gaps, not alternate contracts. ### Future gates preserved diff --git a/specs/001-huddle-triage-board/spec.md b/specs/001-huddle-triage-board/spec.md index fe9d222..170048f 100644 --- a/specs/001-huddle-triage-board/spec.md +++ b/specs/001-huddle-triage-board/spec.md @@ -40,10 +40,10 @@ confidence value, and an opening line. Fully testable without any other story im the board, **Then** that student does **not** appear on the board. 4. **Given** two students both flagged, one at higher severity, **When** the guide opens the board, **Then** the higher-severity student is ranked above the lower-severity one. -5. **Given** generated narration is delayed, unavailable, timed out, or rejected by grounding, - **When** the guide opens the board, **Then** every entry still has a non-empty, cause-specific, - evidence-grounded deterministic fallback opener, the entry shows a concise fallback cue, and no - student, cause, confidence, or rank changes. +5. **Given** the manual synthetic-refresh draft is unavailable, timed out, malformed, or rejected, + **When** the guide opens the board, **Then** every entry still has a complete, cause-specific, + deterministic six-field conversation card, and no student, cause, confidence, rank, or evidence + changes. 6. **Given** an authorized synthetic guide, **When** they open the board on desktop, **Then** Variant B — Evidence Desk displays a ranked rail beside the selected evidence workspace; on mobile, the same URL-addressable queue/detail states appear one at a time. Before any report is selected, every @@ -86,12 +86,13 @@ with no other story implemented. against the drill-down evidence, **Then** every factual value and qualitative proposition is authorized by that evidence, including every quantity, date, skill, and student attribute. 4. **Given** a guide selects a ranked entry, **When** the evidence workspace opens, **Then** it shows - the cause, full priority, confidence, concrete opener, student comparison, prerequisite context, - and a concise deterministic-fallback cue when applicable; one clear accessible disclosure reveals - only the dominant finding's exact deduplicated attempt/session evidence. The teacher-facing report - does not expose additional causes/evidence, rule or quality details, expandable technical - provenance, attempt/activity/rule/signal identifiers, ordinals, hashes, confidence math, - multipliers, raw pattern dumps, or conflict-adjustment jargon. + cause, full priority, confidence, and one fully visible teacher-reviewed decision-map card with the + six FR-054 fields in order; only the explanation, first question, and teaching move are AI-drafted. + One clear accessible disclosure reveals only the dominant finding's exact deduplicated + attempt/session evidence. The teacher-facing report does not expose additional causes/evidence, + rule or quality details, technical provenance, regeneration controls, attempt/activity/rule/signal + identifiers, ordinals, hashes, confidence math, multipliers, raw pattern dumps, or + conflict-adjustment jargon. 5. **Given** an authorized report becomes visibly open, **When** the automatic acknowledgment action submits the fresh opening-read-issued short-lived one-use grant and its same-opening renewal capability, **Then** expiry after visibility or in flight uniquely claims the expired source nonce, @@ -278,8 +279,9 @@ denied. delivery does not inflate attempt counts. - **Activity referencing an unknown skill code**: recorded and reported as unmapped rather than silently dropped or guessed into an adjacent skill. -- **Language generation unavailable or slow**: the board still renders with ranking and root causes - intact, a cause-specific grounded fallback opener, and a concise deterministic-fallback cue. +- **Manual synthetic language draft unavailable or slow**: the board still renders with ranking and + root causes intact and a complete cause-specific deterministic six-field card; nightly refresh + never invokes language generation. - **Refresh fails after a prior success**: the current `board_head` remains unchanged and visible; the failed request is shown separately and never becomes an all-clear. - **Refresh fails before any success**: the board reports `not-built`/unavailable, not successful-empty. @@ -374,19 +376,25 @@ denied. - **FR-022**: System MUST assemble, for each board entry, a self-contained evidence bundle containing everything needed to describe the finding and nothing more. -- **FR-023**: Language generation MUST receive only that evidence bundle. It MUST NOT select which - students appear, rank them, or determine root cause. -- **FR-024**: Generated language MUST be selected from a committed closed catalog of evidence-backed - diagnosis propositions and opener templates. The model may return only catalog IDs and authorized - slot references into its one evidence bundle. Unknown IDs, missing or extra slots, unauthorized - references, value mismatches, unsupported qualitative claims, and unsupported student attributes - MUST hard-fail deterministic validation; free factual prose is prohibited. -- **FR-025**: System MUST deterministically render, per entry, a non-empty plain-language explanation - and a concrete opening line from a validated catalog selection. -- **FR-026**: System MUST render the board with ranking and root causes intact when model-backed - selection is delayed or unavailable. Every board-visible cause MUST instead receive its committed, - cause-specific deterministic fallback catalog selection, validated and rendered under the same - evidence/slot/atom rules, with explicit degraded reason; fallback may never block or rerank. +- **FR-023**: Language generation MUST receive only a bounded packet projected from the current + synthetic evidence: first name, primary problem, at most three related attempts with their mapped + answer/misconception and hint facts, relevant prerequisite, and deterministic uncertainty framing. + It MUST NOT receive identifiers, unrelated history, rule/provenance metadata, rank, severity, raw + confidence, or a student attribute beyond first name; it cannot select membership, ordering, + priority, confidence, primary problem, or evidence. +- **FR-024**: The sole AI-drafted fields are `likely explanation to test`, `ask first`, and + `suggested teaching move`. Output MUST use that strict three-field schema with short limits; + `ask first` is a question; weak evidence begins `Working hypothesis:`; explicit quantities, dates, + and skill identifiers must exactly match the packet; and judgmental language is rejected. The + deterministic cause-specific card supplies `if they explain confidently`, `if they struggle`, and + `listen for`. No open-ended validator may attempt to prove every qualitative natural-language claim. +- **FR-025**: Only a manual refresh of synthetic data may invoke the model. It validates and persists + one complete six-field conversation plan with the immutable board run. Nightly refreshes never call + a model. There is no regenerate action, real-data behavior, second model judgment, or + diagnosis-accuracy claim. +- **FR-026**: A missing key, timeout, provider error, malformed output, or rejected output MUST publish + the complete deterministic cause-specific six-field fallback without blocking, partially publishing, + reranking, or changing deterministic evidence. #### The Board @@ -473,12 +481,15 @@ denied. - **FR-054**: The quick demo MUST use Variant B — Evidence Desk: a ranked rail and URL-addressable evidence workspace on desktop, and equivalent queue/detail states on mobile. Every queue card MUST expose rank for today, full priority, and confidence accessibly before selection. Each selected - report MUST retain the cause, concrete opener, student comparison, and concise fallback cue, with - one accessible **Show exact contributing evidence** disclosure for the dominant finding's - deduplicated attempt/session results, answers, and timing. Additional causes/evidence, rule and - quality details, expandable technical provenance, internal IDs, ordinals, hashes, confidence math, - and conflict-adjustment jargon MUST stay out of the teacher-facing report. Returning MUST restore - the selected row's focus and position. + report MUST show one fully visible `AI-assisted conversation plan` in this order: likely explanation + to test; ask first; if they explain confidently; if they struggle; listen for; suggested teaching + move. The first question is the stem and the two conditional fields are an obvious fork. The plan is + a teacher-reviewed draft; weak evidence is a working hypothesis; only the explanation, first + question, and move are visibly AI-drafted. **Show exact contributing evidence** is the only + expandable detail. Additional causes/evidence, rule and quality details, technical provenance, + regeneration controls, internal IDs, ordinals, hashes, confidence math, and conflict-adjustment + jargon MUST stay out of the teacher-facing report. Returning MUST restore the selected row's focus + and position. - **FR-055**: System MUST automatically acknowledge the exact finding after its authorized evidence report is visibly opened. Prefetch MUST create and carry no acknowledgment grant. Actual opening MUST bypass prefetched evidence and perform a fresh authorized read that returns evidence plus a