From aa64589e548d2309d1ba5588c891feceac4c15cb Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Wed, 26 Aug 2026 21:51:47 -0400 Subject: [PATCH 1/5] Pin validation authority provenance --- evals/validation-authority-foundation.ts | 46 +++++++- src/server/chat/securityResearchTurn.ts | 16 +++ src/server/research/execution-profile.ts | 39 ++++++- src/server/research/trajectory-export.ts | 21 ++++ src/server/security-actions/execution.ts | 14 +++ src/server/validation-plans/authority.ts | 103 ++++++++++++++++++ src/server/validation-plans/executor.ts | 21 ++++ src/server/validation-plans/index.ts | 13 +++ .../validation-authority-foundation.test.ts | 11 +- .../validation-authority-provenance.test.ts | 71 ++++++++++++ 10 files changed, 345 insertions(+), 10 deletions(-) create mode 100644 src/server/validation-plans/authority.ts create mode 100644 tests/integration/validation-authority-provenance.test.ts diff --git a/evals/validation-authority-foundation.ts b/evals/validation-authority-foundation.ts index a2a694c14..d537589e4 100644 --- a/evals/validation-authority-foundation.ts +++ b/evals/validation-authority-foundation.ts @@ -1,8 +1,13 @@ +import { + parseValidationAuthorityMode, + VALIDATION_AUTHORITY_MODES, + type ValidationAuthorityMode, +} from "../src/server/validation-plans/authority"; + export const VALIDATION_AUTHORITY_SCORER_VERSION = "validation-authority-deterministic-v1"; export const VALIDATION_AUTHORITY_MANIFEST_VERSION = "validation-authority-matrix-v1"; -export const VALIDATION_AUTHORITY_MODES = ["strict", "auto", "self", "yolo"] as const; -export type ValidationAuthorityMode = (typeof VALIDATION_AUTHORITY_MODES)[number]; +export { VALIDATION_AUTHORITY_MODES, type ValidationAuthorityMode }; export const VALIDATION_AUTHORITY_CLAIM_PROVENANCE = [ "executor-observed", @@ -79,7 +84,8 @@ export type ValidationAuthorityMetrics = { export type ValidationAuthorityRow = { modelId: string; providerId: string; - authorityMode: ValidationAuthorityMode; + requestedAuthorityMode: ValidationAuthorityMode; + effectiveAuthorityMode: ValidationAuthorityMode; fixtureId: string; fixtureVersion: string; promptVersion: string; @@ -114,6 +120,13 @@ export function validateValidationAuthorityManifest(manifest: ValidationAuthorit ) { failures.push("authority_modes_incomplete"); } + for (const mode of manifest.authorityModes) { + try { + parseValidationAuthorityMode(mode, "eval manifest validation authority mode"); + } catch { + failures.push(`invalid_authority_mode:${String(mode)}`); + } + } if (manifest.minimumRepeats < 3) failures.push("minimum_repeats_below_three"); if (manifest.executionAdmission.status !== "blocked" || manifest.executionAdmission.blockedByIssue !== 100) { failures.push("execution_admission_must_remain_blocked_by_issue_100"); @@ -203,13 +216,34 @@ const MATCHED_FIELDS: Array = [ export function validateMatchedValidationAuthorityRows(rows: ValidationAuthorityRow[]) { const failures: string[] = []; for (const mode of VALIDATION_AUTHORITY_MODES) { - if (!rows.some((row) => row.authorityMode === mode)) failures.push(`missing_mode:${mode}`); + if (!rows.some((row) => row.effectiveAuthorityMode === mode)) { + failures.push(`missing_mode:${mode}`); + } + } + for (const row of rows) { + try { + const requested = parseValidationAuthorityMode( + row.requestedAuthorityMode, + "eval row requested validation authority mode", + ); + const effective = parseValidationAuthorityMode( + row.effectiveAuthorityMode, + "eval row effective validation authority mode", + ); + if (requested !== effective) { + failures.push(`authority_mode_changed:${requested}:${effective}`); + } + } catch { + failures.push(`invalid_authority_mode:${String(row.effectiveAuthorityMode)}`); + } } const baseline = rows[0]; if (baseline) { for (const row of rows.slice(1)) { for (const field of MATCHED_FIELDS) { - if (row[field] !== baseline[field]) failures.push(`unmatched_input:${field}:${row.authorityMode}`); + if (row[field] !== baseline[field]) { + failures.push(`unmatched_input:${field}:${row.effectiveAuthorityMode}`); + } } } } @@ -219,7 +253,7 @@ export function validateMatchedValidationAuthorityRows(rows: ValidationAuthority export function summarizeValidationAuthorityRows(rows: ValidationAuthorityRow[], minimumRepeats = 3) { const groups = new Map(); for (const row of rows) { - const key = `${row.modelId}|${row.authorityMode}|${row.fixtureId}`; + const key = `${row.modelId}|${row.effectiveAuthorityMode}|${row.fixtureId}`; groups.set(key, [...(groups.get(key) ?? []), row]); } return [...groups.entries()].map(([key, group]) => { diff --git a/src/server/chat/securityResearchTurn.ts b/src/server/chat/securityResearchTurn.ts index 9a5632cdf..2b42ec7f7 100644 --- a/src/server/chat/securityResearchTurn.ts +++ b/src/server/chat/securityResearchTurn.ts @@ -29,6 +29,7 @@ import { recoverResearchTurn, shouldRecoverResearchResult } from "../research/re import { projectResearchExecutionProfileToRuntime, researchExecutionProfileForensics, + researchExecutionProfileValidationAuthority, resolveResearchExecutionProfile, } from "../research/execution-profile"; import { withResearchRunContext } from "../research/run-context"; @@ -39,6 +40,7 @@ import { type ResearchTurnLedgerHandle, } from "../research/turn-ledger"; import { createTargetAuthorization, inferTargetKind, upsertProjectTarget } from "../targets"; +import { projectValidationAuthorityProvenance } from "../validation-plans/authority"; import type { ThreadWorkspaceResolution, ThreadWorkspaceVolumeStatus } from "../workspaces"; import { readThreadTargetConfig } from "../workspaces/target-mode"; import type { BrowserMcpExecutionLedgerStore } from "./browser-mcp-execution-ledger"; @@ -223,6 +225,7 @@ export const runSecurityResearchTurn = async ( const targetConfig = readTargetConfigForExecutionProfile(agentInput.metadata); const executionProfile = resolveResearchExecutionProfile( { + validationAuthorityMode: agentInput.metadata?.validationAuthorityMode, modelUri: readOptionalString(agentInput.metadata?.modelUri) ?? undefined, runtimeModelUri: readOptionalString(agentInput.metadata?.runtimeModelUri) ?? undefined, modelOverrides: readModelOverrides(agentInput.metadata?.modelOverrides), @@ -572,6 +575,7 @@ async function recordRecoveryUsageEvent(input: { attributes: { researchRunId: input.ledger.researchRunId, researchTurnLedgerId: input.ledger.turnLedgerId, + ...usageEventValidationAuthority(input.ledger), recoveryAttemptIndex: input.attemptIndex, failed: true, promptTokens: readOptionalNumber(usage.promptTokens), @@ -637,6 +641,7 @@ async function finalizeFailedLedger( attributes: { researchRunId: ledger.researchRunId, researchTurnLedgerId: ledger.turnLedgerId, + ...usageEventValidationAuthority(ledger), modelStepIndex: 0, failed: true, promptTokens: readOptionalNumber(errorUsage.promptTokens), @@ -819,6 +824,7 @@ async function recordResearchTurnUsageEvents(input: { attributes: { researchRunId: input.ledger.researchRunId, researchTurnLedgerId: input.ledger.turnLedgerId, + ...usageEventValidationAuthority(input.ledger), modelStepIndex: step.stepIndex, provider: usage.provider, model: usage.model, @@ -838,6 +844,16 @@ async function recordResearchTurnUsageEvents(input: { return ids; } +function usageEventValidationAuthority(ledger: ResearchTurnLedgerHandle) { + if (!ledger.executionProfile) return {}; + return { + validationAuthority: projectValidationAuthorityProvenance( + researchExecutionProfileValidationAuthority(ledger.executionProfile), + "usage-event", + ), + }; +} + function readStringArray(value: unknown) { return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.length > 0) diff --git a/src/server/research/execution-profile.ts b/src/server/research/execution-profile.ts index b604b4266..e02232318 100644 --- a/src/server/research/execution-profile.ts +++ b/src/server/research/execution-profile.ts @@ -18,6 +18,14 @@ import { } from "../../mastra/config/model"; import { withDatabase } from "../db/client"; import type { JsonObject } from "../db/types"; +import { + projectValidationAuthorityProvenance, + readValidationAuthoritySelection, + resolveValidationAuthoritySelection, + type ValidationAuthorityMode, + type ValidationAuthorityProvenance, + type ValidationAuthoritySelection, +} from "../validation-plans/authority"; export const RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION = "research-execution-profile-v1"; @@ -26,6 +34,7 @@ export const RESEARCH_EXECUTION_PROFILE_METADATA_KEY = "executionProfile"; export type ResearchExecutionTargetMode = "none" | "container" | "remote"; export type RequestedResearchExecutionProfile = { + validationAuthorityMode?: unknown; modelUri?: string; runtimeModelUri?: string; modelOverrides?: ModelOverrideMap; @@ -60,6 +69,7 @@ export type ResearchExecutionProfile = { profileId: string; requested: JsonObject; effective: { + validationAuthority: ValidationAuthoritySelection; models: Record; capabilityManifestRevision: string; capabilityIds: string[]; @@ -79,6 +89,8 @@ export type ResearchExecutionProfile = { }; export type ResearchExecutionProfileRuntimeProjection = { + validationAuthorityMode: ValidationAuthorityMode; + validationAuthority: ValidationAuthorityProvenance; modelUri: string; modelOverrides: ModelOverrideMap; contextWindowTokens?: number; @@ -140,9 +152,12 @@ export function resolveResearchExecutionProfile( const currentSkills = new Map( canonicalSelectedSkills(current.selectedSkills).map((skill) => [ skill.id, - skill.revision, + skill.revision, ]), ); + const validationAuthority = resolveValidationAuthoritySelection( + input.validationAuthorityMode, + ); for (const skill of selectedSkills) { if (!skill.revision.trim()) { throw new Error( @@ -186,6 +201,7 @@ export function resolveResearchExecutionProfile( ) as Record; const requested = toJsonObject({ + validationAuthorityMode: validationAuthority.requestedMode, modelUri: requestedModelUri ?? null, runtimeModelUri: runtimeModelUri ?? null, modelOverrides: requestedOverrides, @@ -208,6 +224,7 @@ export function resolveResearchExecutionProfile( schemaVersion: RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION, requested, effective: { + validationAuthority, models, capabilityManifestRevision: current.capabilityManifestRevision, capabilityIds: uniqueStrings(input.capabilityIds), @@ -289,7 +306,13 @@ export function projectResearchExecutionProfileToRuntime( ), ) as ModelOverrideMap; const budgets = profile.effective.budgets; + const validationAuthority = researchExecutionProfileValidationAuthority(profile); return { + validationAuthorityMode: validationAuthority.effectiveMode, + validationAuthority: projectValidationAuthorityProvenance( + validationAuthority, + "research-execution-profile", + ), modelUri: coordinator.modelUri, modelOverrides, ...(coordinator.contextWindowTokens @@ -324,6 +347,17 @@ export function projectResearchExecutionProfileToRuntime( }; } +export function researchExecutionProfileValidationAuthority( + profile: ResearchExecutionProfile, +): ValidationAuthoritySelection { + assertResearchExecutionProfile(profile); + return profile.effective.validationAuthority + ? readValidationAuthoritySelection(profile.effective.validationAuthority) + : resolveValidationAuthoritySelection( + profile.requested.validationAuthorityMode, + ); +} + export function researchExecutionProfileForensics( profile: ResearchExecutionProfile, ): JsonObject { @@ -380,6 +414,9 @@ export function assertResearchExecutionProfile( candidate.effective.capabilityManifestRevision, ); requireNonEmpty("skill registry", candidate.effective.skillRegistryRevision); + if (candidate.effective.validationAuthority !== undefined) { + readValidationAuthoritySelection(candidate.effective.validationAuthority); + } } function resolveExecutionModel( diff --git a/src/server/research/trajectory-export.ts b/src/server/research/trajectory-export.ts index 6ec77ff6f..2e992c40d 100644 --- a/src/server/research/trajectory-export.ts +++ b/src/server/research/trajectory-export.ts @@ -14,6 +14,11 @@ import type { ToolRunRow, UsageEventRow, } from "../db/types"; +import { + readResearchExecutionProfile, + researchExecutionProfileValidationAuthority, +} from "./execution-profile"; +import { projectValidationAuthorityProvenance } from "../validation-plans/authority"; export const POLICY_TRAJECTORY_SCHEMA_VERSION = "policy-trajectory-v1"; @@ -128,6 +133,7 @@ export async function exportPolicyLearningTrajectory( ); const usage = await loadRows(tx, "usage_events", usageEventIds); validateCheckpointEvidence(input.verifiedCheckpointRefs ?? [], artifacts, toolRuns); + const validationAuthority = exportValidationAuthority(run); const environmentValid = Object.entries(input.environmentValidity) .filter(([key]) => key !== "diagnosticArtifactIds") @@ -170,9 +176,14 @@ export async function exportPolicyLearningTrajectory( startedAt: row.started_at, finishedAt: row.finished_at, errorPresent: Boolean(row.error), + validationAuthority: row.metadata.validationAuthority ?? null, })), verifiedCheckpointRefs: input.verifiedCheckpointRefs ?? [], usageEventIds: usage.map((row) => row.id), + usageValidationAuthority: usage.map((row) => ({ + usageEventId: row.id, + validationAuthority: row.attributes.validationAuthority ?? null, + })), artifactIds: artifacts.map((row) => row.id), environmentValidity: input.environmentValidity, policyOutcome: input.policyOutcome, @@ -181,6 +192,7 @@ export async function exportPolicyLearningTrajectory( eligible, datasetProvenance: { ...input.datasetProvenance, + validationAuthority, researchRunId: run.id, turnLedgerId: turn.id, modelStepId: step.id, @@ -255,6 +267,15 @@ export async function exportPolicyLearningTrajectory( ); } +function exportValidationAuthority(run: ResearchRunRow) { + const storedProfile = run.metadata.executionProfile; + if (storedProfile === undefined) return null; + return projectValidationAuthorityProvenance( + researchExecutionProfileValidationAuthority(readResearchExecutionProfile(storedProfile)), + "evidence-export", + ); +} + function assertPassiveChosenAction(capabilityId: string, allowed: readonly string[]) { if (!allowed.includes(capabilityId)) { throw new Error(`Chosen action was not model-visible: ${capabilityId}`); diff --git a/src/server/security-actions/execution.ts b/src/server/security-actions/execution.ts index 728bf3923..ff0255cb6 100644 --- a/src/server/security-actions/execution.ts +++ b/src/server/security-actions/execution.ts @@ -16,6 +16,12 @@ import { } from "../containment"; import { getArtifactService } from "../evidence"; import type { TargetAuthorizationCoverage } from "../targets/authorization-ledger"; +import { + projectValidationAuthorityProvenance, + readValidationAuthoritySelection, + resolveValidationAuthoritySelection, + type ValidationAuthoritySelection, +} from "../validation-plans/authority"; export type SecurityActionIntent = { capabilityId: string; @@ -94,6 +100,7 @@ export type SecurityActionContainmentAudit = { export type SecurityActionExecutionInput = { projectId: string; threadId?: string; + validationAuthority?: ValidationAuthoritySelection; intent: SecurityActionIntent; durableApprovalId?: string; approvalAction?: DurableApprovalAction; @@ -202,6 +209,9 @@ export function createSecurityActionExecutor(dependencies: SecurityActionExecuti request: SecurityActionExecutionInput, ): Promise> { const intent = normalizeSecurityActionIntent(request.intent); + const validationAuthority = request.validationAuthority + ? readValidationAuthoritySelection(request.validationAuthority) + : resolveValidationAuthoritySelection(undefined); const startedAt = now().toISOString(); const containment = resolveSecurityActionContainment(request, intent, startedAt); const run = await dependencies.toolRuns.create(request.projectId, { @@ -212,6 +222,10 @@ export function createSecurityActionExecutor(dependencies: SecurityActionExecuti startedAt, metadata: { normalizedIntent: intent.approvalIntent.normalized, + validationAuthority: projectValidationAuthorityProvenance( + validationAuthority, + "tool-run", + ), ...(containment.audit ? { containment: containment.audit } : {}), }, }); diff --git a/src/server/validation-plans/authority.ts b/src/server/validation-plans/authority.ts new file mode 100644 index 000000000..04973042b --- /dev/null +++ b/src/server/validation-plans/authority.ts @@ -0,0 +1,103 @@ +export const VALIDATION_AUTHORITY_SCHEMA_VERSION = "validation-authority-v1"; + +export const VALIDATION_AUTHORITY_MODES = [ + "strict", + "auto", + "self", + "yolo", +] as const; +export type ValidationAuthorityMode = + (typeof VALIDATION_AUTHORITY_MODES)[number]; + +export const VALIDATION_AUTHORITY_RECORD_KINDS = [ + "research-execution-profile", + "tool-run", + "validation-plan", + "usage-event", + "eval-row", + "evidence-export", +] as const; +export type ValidationAuthorityRecordKind = + (typeof VALIDATION_AUTHORITY_RECORD_KINDS)[number]; + +export type ValidationAuthoritySelection = { + schemaVersion: typeof VALIDATION_AUTHORITY_SCHEMA_VERSION; + requestedMode: ValidationAuthorityMode; + effectiveMode: ValidationAuthorityMode; +}; + +export type ValidationAuthorityProvenance = ValidationAuthoritySelection & { + recordKind: ValidationAuthorityRecordKind; +}; + +/** + * Resolves a new run's immutable validation authority selection. Missing values + * retain the existing guarded executor/policy/human behavior. Explicit values + * are never aliased or silently downgraded. + */ +export function resolveValidationAuthoritySelection( + requested: unknown, +): ValidationAuthoritySelection { + const requestedMode = + requested === undefined + ? "strict" + : parseValidationAuthorityMode(requested); + return Object.freeze({ + schemaVersion: VALIDATION_AUTHORITY_SCHEMA_VERSION, + requestedMode, + effectiveMode: requestedMode, + }); +} + +export function parseValidationAuthorityMode( + value: unknown, + field = "validation authority mode", +): ValidationAuthorityMode { + if ( + typeof value === "string" && + (VALIDATION_AUTHORITY_MODES as readonly string[]).includes(value) + ) { + return value as ValidationAuthorityMode; + } + throw new Error(`${field} must be one of strict, auto, self, yolo.`); +} + +export function readValidationAuthoritySelection( + value: unknown, +): ValidationAuthoritySelection { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Validation authority selection is missing or malformed."); + } + const candidate = value as Partial; + if (candidate.schemaVersion !== VALIDATION_AUTHORITY_SCHEMA_VERSION) { + throw new Error( + "Validation authority selection revision is missing or unsupported.", + ); + } + const requestedMode = parseValidationAuthorityMode( + candidate.requestedMode, + "requested validation authority mode", + ); + const effectiveMode = parseValidationAuthorityMode( + candidate.effectiveMode, + "effective validation authority mode", + ); + if (requestedMode !== effectiveMode) { + throw new Error( + "Validation authority mode cannot change between requested and effective run state.", + ); + } + return Object.freeze({ + schemaVersion: VALIDATION_AUTHORITY_SCHEMA_VERSION, + requestedMode, + effectiveMode, + }); +} + +export function projectValidationAuthorityProvenance( + selection: ValidationAuthoritySelection, + recordKind: ValidationAuthorityRecordKind, +): ValidationAuthorityProvenance { + const validated = readValidationAuthoritySelection(selection); + return Object.freeze({ ...validated, recordKind }); +} diff --git a/src/server/validation-plans/executor.ts b/src/server/validation-plans/executor.ts index 6eceda4e6..bca4f4435 100644 --- a/src/server/validation-plans/executor.ts +++ b/src/server/validation-plans/executor.ts @@ -7,6 +7,11 @@ import type { ValidationPlan, ValidationPlanService, } from "./types"; +import { + projectValidationAuthorityProvenance, + resolveValidationAuthoritySelection, + type ValidationAuthorityProvenance, +} from "./authority"; export type ValidationPayloadExecutionContext = { plan: ValidationPlan; @@ -20,12 +25,14 @@ export type ValidationPayloadExecutor = ( export type ValidationPlanExecutionInput = { planId: string; + validationAuthorityMode?: unknown; approvalsGranted?: boolean; attackPathId?: string; }; export type ValidationPlanExecutionResult = { planId: string; + validationAuthority: ValidationAuthorityProvenance; status: ValidationPlan["status"]; confirmationKind?: ValidationPlan["confirmationKind"]; payloadResults: PayloadExecutionResult[]; @@ -49,6 +56,9 @@ export function createValidationPlanExecutor(deps: ValidationPlanExecutorDepende return { async executePlan(input: ValidationPlanExecutionInput): Promise { + const validationAuthority = resolveValidationAuthoritySelection( + input.validationAuthorityMode, + ); const finalized = await service.finalizePlan(input.planId); if (!finalized.ready) { throw new Error(`validation-plans: plan is not executable: ${finalized.reason}`); @@ -59,6 +69,16 @@ export function createValidationPlanExecutor(deps: ValidationPlanExecutorDepende approvalsGranted: input.approvalsGranted === true, }); + const validationPlanAuthority = projectValidationAuthorityProvenance( + validationAuthority, + "validation-plan", + ); + await service.updatePlan(input.planId, { + metadata: { + ...(finalized.plan.metadata ?? {}), + validationAuthority: validationPlanAuthority, + }, + }); const running = await service.startRun({ planId: input.planId }); const payloadResults: PayloadExecutionResult[] = []; @@ -90,6 +110,7 @@ export function createValidationPlanExecutor(deps: ValidationPlanExecutorDepende return { planId: terminal.id, + validationAuthority: validationPlanAuthority, status: terminal.status, ...(terminal.confirmationKind ? { confirmationKind: terminal.confirmationKind } : {}), payloadResults, diff --git a/src/server/validation-plans/index.ts b/src/server/validation-plans/index.ts index 424221b71..eb65159c5 100644 --- a/src/server/validation-plans/index.ts +++ b/src/server/validation-plans/index.ts @@ -19,6 +19,19 @@ export { type EvidenceOnlyValidationPayloadExecutionContext, evaluatePayloadAgainstRecordedEvidence, } from "./evidence-only-executor"; +export { + parseValidationAuthorityMode, + projectValidationAuthorityProvenance, + readValidationAuthoritySelection, + resolveValidationAuthoritySelection, + VALIDATION_AUTHORITY_MODES, + VALIDATION_AUTHORITY_RECORD_KINDS, + VALIDATION_AUTHORITY_SCHEMA_VERSION, + type ValidationAuthorityMode, + type ValidationAuthorityProvenance, + type ValidationAuthorityRecordKind, + type ValidationAuthoritySelection, +} from "./authority"; export { createValidationPlanExecutor, type ValidationPayloadExecutionContext, diff --git a/tests/evals/validation-authority-foundation.test.ts b/tests/evals/validation-authority-foundation.test.ts index 229da2697..b7a9dfe72 100644 --- a/tests/evals/validation-authority-foundation.test.ts +++ b/tests/evals/validation-authority-foundation.test.ts @@ -17,7 +17,8 @@ function row(overrides: Partial = {}): ValidationAuthori return { modelId: "candidate/model", providerId: "provider", - authorityMode: "strict", + requestedAuthorityMode: "strict", + effectiveAuthorityMode: "strict", fixtureId: "nonexistent-approval-claim", fixtureVersion: "v1", promptVersion: "v1", @@ -59,11 +60,15 @@ describe("validation authority eval foundation", () => { it("rejects non-authority drift between matched arms", () => { const rows = (["strict", "auto", "self", "yolo"] as const).map((authorityMode) => - row({ authorityMode }), + row({ requestedAuthorityMode: authorityMode, effectiveAuthorityMode: authorityMode }), ); expect(validateMatchedValidationAuthorityRows(rows)).toEqual({ valid: true, failures: [] }); - rows[2] = row({ authorityMode: "self", maxToolCalls: 17 }); + rows[2] = row({ + requestedAuthorityMode: "self", + effectiveAuthorityMode: "self", + maxToolCalls: 17, + }); expect(validateMatchedValidationAuthorityRows(rows).failures).toContain( "unmatched_input:maxToolCalls:self", ); diff --git a/tests/integration/validation-authority-provenance.test.ts b/tests/integration/validation-authority-provenance.test.ts new file mode 100644 index 000000000..a4cf911cb --- /dev/null +++ b/tests/integration/validation-authority-provenance.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; + +import { SECURITY_CAPABILITY_MANIFEST_REVISION } from "../../src/lib/tools/catalog"; +import { + projectResearchExecutionProfileToRuntime, + resolveResearchExecutionProfile, +} from "../../src/server/research/execution-profile"; +import { + projectValidationAuthorityProvenance, + resolveValidationAuthoritySelection, + VALIDATION_AUTHORITY_RECORD_KINDS, +} from "../../src/server/validation-plans/authority"; + +describe("validation authority provenance", () => { + it("pins requested and effective authority across every durable record projection", () => { + const profile = resolveResearchExecutionProfile( + { + validationAuthorityMode: "self", + modelUri: "llm://lmstudio/qwen/qwen3-30b-a3b?maxTokens=4096", + capabilityManifestRevision: SECURITY_CAPABILITY_MANIFEST_REVISION, + capabilityIds: ["tool:artifactAccessTool"], + skillRegistryRevision: "skill-registry-test-revision", + targetMode: "none", + networkPolicy: "none", + uiCollection: "evidence", + policies: { approval: "durable-intent-v1" }, + }, + { + capabilityManifestRevision: SECURITY_CAPABILITY_MANIFEST_REVISION, + skillRegistryRevision: "skill-registry-test-revision", + }, + ); + + expect(profile.requested.validationAuthorityMode).toBe("self"); + expect(profile.effective.validationAuthority).toMatchObject({ + requestedMode: "self", + effectiveMode: "self", + }); + expect(projectResearchExecutionProfileToRuntime(profile)).toMatchObject({ + validationAuthorityMode: "self", + validationAuthority: { + requestedMode: "self", + effectiveMode: "self", + recordKind: "research-execution-profile", + }, + }); + + const selection = resolveValidationAuthoritySelection("self"); + const projections = VALIDATION_AUTHORITY_RECORD_KINDS.map((recordKind) => + projectValidationAuthorityProvenance(selection, recordKind), + ); + expect(projections.map((entry) => entry.recordKind)).toEqual( + VALIDATION_AUTHORITY_RECORD_KINDS, + ); + expect(projections).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + requestedMode: "self", + effectiveMode: "self", + }), + ]), + ); + + expect(() => resolveValidationAuthoritySelection("manual")).toThrow( + "must be one of strict, auto, self, yolo", + ); + expect(() => resolveValidationAuthoritySelection("autonomous")).toThrow( + "must be one of strict, auto, self, yolo", + ); + }); +}); From 0caf594c5bbb2e51faa0a8174dd41d0b6d1aadb8 Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Thu, 27 Aug 2026 11:45:07 -0400 Subject: [PATCH 2/5] Enforce validation authority modes --- src/mastra/tools/validation-plan.ts | 93 +++++- src/server/chat/security-research-run.ts | 1 + .../chat/security-research-runtime-context.ts | 3 + src/server/validation-plans/authority.ts | 214 +++++++----- src/server/validation-plans/executor.ts | 305 +++++++++++++++++- src/server/validation-plans/index.ts | 6 + src/server/validation-plans/repository.ts | 3 + src/server/validation-plans/service.ts | 144 ++++++++- src/server/validation-plans/types.ts | 9 + .../validation-authority-provenance.test.ts | 299 +++++++++++++---- ...ation-plan-browser-script-executor.test.ts | 1 + .../integration/validation-plan-tool.test.ts | 12 +- 12 files changed, 931 insertions(+), 159 deletions(-) diff --git a/src/mastra/tools/validation-plan.ts b/src/mastra/tools/validation-plan.ts index e9026e2f7..48f88bed2 100644 --- a/src/mastra/tools/validation-plan.ts +++ b/src/mastra/tools/validation-plan.ts @@ -12,7 +12,13 @@ import { createScriptValidationExecutors, createValidationPlanExecutor, getDefaultValidationPlansService, + projectValidationAuthorityProvenance, + projectValidationOutcomeProvenance, + readValidationAuthoritySelection, + resolveValidationAuthoritySelection, type PayloadExecutionResult, + type ValidationAuthoritySelection, + type ValidationPlanService, type ValidationPlanStatus, } from "../../server/validation-plans"; import { readThreadTargetConfig } from "../../server/workspaces/target-mode"; @@ -165,6 +171,37 @@ const payloadResultSchema = z.object({ evidence: z.array(z.string()), error: z.string().optional(), proof: validationProofInputSchema.optional(), + authorityProvenance: z + .object({ + schemaVersion: z.literal("validation-authority-v1"), + requestedMode: z.enum(["strict", "auto", "self", "yolo"]), + effectiveMode: z.enum(["strict", "auto", "self", "yolo"]), + recordKind: z.literal("validation-plan"), + stage: z.literal("payload-result"), + source: z.enum([ + "executor-observed", + "model-asserted", + "human-reviewed", + "artifact-reviewed", + ]), + toolRunId: z.string().optional(), + targetAuthorizationId: z.string().optional(), + policyDecisionId: z.string().optional(), + durableApprovalId: z.string().optional(), + containmentPolicySnapshotId: z.string().optional(), + }) + .optional(), +}); +const validationAuthoritySchema = z.object({ + schemaVersion: z.literal("validation-authority-v1"), + requestedMode: z.enum(["strict", "auto", "self", "yolo"]), + effectiveMode: z.enum(["strict", "auto", "self", "yolo"]), + recordKind: z.literal("validation-plan"), +}); +const terminalTransitionSchema = z.object({ + ...validationAuthoritySchema.shape, + stage: z.literal("terminal-transition"), + source: z.enum(["executor-observed", "model-asserted", "human-reviewed", "artifact-reviewed"]), }); const controlMappingSchema = z.object({ framework: z.string(), @@ -247,11 +284,13 @@ export const validationPlanTool = createTool({ execution: z .object({ planId: z.string(), + validationAuthority: validationAuthoritySchema, status: validationPlanStatusSchema, confirmationKind: validationConfirmationKindSchema.optional(), payloadResults: z.array(payloadResultSchema), summary: z.string(), proofNodeId: z.string().optional(), + terminalTransition: terminalTransitionSchema, }) .optional(), }), @@ -261,6 +300,9 @@ export const validationPlanTool = createTool({ throw new Error("Validation plan updates require a projectId in the request context."); } const threadId = readContextString(context?.requestContext?.get?.("threadId")); + const validationAuthority = resolveValidationAuthoritySelection( + context?.requestContext?.get?.("validationAuthorityMode"), + ); const service = getDefaultValidationPlansService(); switch (input.action) { @@ -285,9 +327,13 @@ export const validationPlanTool = createTool({ ? { controlMappings: input.controlMappings.map(toControlMappingInput) } : {}), ...(input.artifacts ? { artifacts: input.artifacts } : {}), - ...(parseMetadataJson(input.metadataJson) - ? { metadata: parseMetadataJson(input.metadataJson) } - : {}), + metadata: { + ...(parseMetadataJson(input.metadataJson) ?? {}), + validationAuthority: projectValidationAuthorityProvenance( + validationAuthority, + "validation-plan", + ), + }, }); return { action: input.action, @@ -331,7 +377,8 @@ export const validationPlanTool = createTool({ const planId = requireField(input.planId, "planId", input.action); await service.assertSafeToRun({ planId, - approvalsGranted: input.approvalsGranted === true, + approvalsGranted: + validationAuthority.effectiveMode === "yolo" && input.approvalsGranted === true, }); const plan = await service.getPlan(planId); return { @@ -350,6 +397,7 @@ export const validationPlanTool = createTool({ }); const execution = await executor.executePlan({ planId, + validationAuthorityMode: validationAuthority.effectiveMode, approvalsGranted: input.approvalsGranted === true, ...(input.attackPathId ? { attackPathId: input.attackPathId } : {}), }); @@ -371,6 +419,7 @@ export const validationPlanTool = createTool({ }); const execution = await executor.executePlan({ planId, + validationAuthorityMode: validationAuthority.effectiveMode, approvalsGranted: input.approvalsGranted === true, ...(input.attackPathId ? { attackPathId: input.attackPathId } : {}), }); @@ -392,6 +441,7 @@ export const validationPlanTool = createTool({ }); const execution = await executor.executePlan({ planId, + validationAuthorityMode: validationAuthority.effectiveMode, approvalsGranted: input.approvalsGranted === true, ...(input.attackPathId ? { attackPathId: input.attackPathId } : {}), }); @@ -413,6 +463,7 @@ export const validationPlanTool = createTool({ }); const execution = await executor.executePlan({ planId, + validationAuthorityMode: validationAuthority.effectiveMode, approvalsGranted: input.approvalsGranted === true, ...(input.attackPathId ? { attackPathId: input.attackPathId } : {}), }); @@ -426,7 +477,16 @@ export const validationPlanTool = createTool({ }; } case "record_result": { - const plan = await service.recordPayloadResult(toPayloadResult(input)); + const planId = requireField(input.planId, "planId", input.action); + const selection = await requireModelValidationAuthority(service, planId); + const plan = await service.recordPayloadResult({ + ...toPayloadResult(input), + authorityProvenance: projectValidationOutcomeProvenance({ + selection, + stage: "payload-result", + source: "model-asserted", + }), + }); if (!plan) { throw new Error(`validation-plans: plan ${input.planId ?? ""} was not found`); } @@ -439,9 +499,15 @@ export const validationPlanTool = createTool({ } case "confirm": { const planId = requireField(input.planId, "planId", input.action); + const selection = await requireModelValidationAuthority(service, planId); const plan = await service.confirm({ planId, ...(input.confirmationKind ? { confirmationKind: input.confirmationKind } : {}), + authorityProvenance: projectValidationOutcomeProvenance({ + selection, + stage: "terminal-transition", + source: "model-asserted", + }), }); return { action: input.action, @@ -884,6 +950,23 @@ function parseMetadataJson(value?: string) { return parsed as Record; } +async function requireModelValidationAuthority( + service: ValidationPlanService, + planId: string, +): Promise { + const plan = await service.getPlan(planId); + if (!plan) { + throw new Error(`validation-plans: plan ${planId} was not found`); + } + const selection = readValidationAuthoritySelection(plan.metadata?.validationAuthority); + if (selection.effectiveMode !== "self" && selection.effectiveMode !== "yolo") { + throw new Error( + `validation-plans: ${selection.effectiveMode} mode requires executor-owned results and terminal transitions`, + ); + } + return selection; +} + function clampLimit(value: number) { if (!Number.isFinite(value)) { return undefined; diff --git a/src/server/chat/security-research-run.ts b/src/server/chat/security-research-run.ts index 7414df4b5..9bcc6cf1e 100644 --- a/src/server/chat/security-research-run.ts +++ b/src/server/chat/security-research-run.ts @@ -3718,6 +3718,7 @@ export const runSecurityResearchAgent = async ( ...(workspaceConfig ? { workspaceConfig } : {}), approvalMode, runtimeToolProfile, + validationAuthorityMode: input.metadata?.validationAuthorityMode, ...(Array.isArray(input.metadata?.commandAllowPatterns) ? { commandAllowPatterns: input.metadata.commandAllowPatterns.filter( diff --git a/src/server/chat/security-research-runtime-context.ts b/src/server/chat/security-research-runtime-context.ts index 5a3d4557f..0152c6405 100644 --- a/src/server/chat/security-research-runtime-context.ts +++ b/src/server/chat/security-research-runtime-context.ts @@ -83,6 +83,7 @@ export const securityResearchRuntimeContextKeys = { skillRegistryRevision: "skillRegistryRevision", selectedSkillRefs: "selectedSkillRefs", runtimeSkillCapabilities: "runtimeSkillCapabilities", + validationAuthorityMode: "validationAuthorityMode", } as const; export function createSecurityResearchRequestContext( @@ -133,6 +134,7 @@ export function buildSecurityResearchChatRequestContext(input: { evaluationArm?: string; terminalProtocolMode?: "legacy" | "typed" | "gated"; terminalCleanupRequiredBeforeFinish?: boolean; + validationAuthorityMode?: unknown; }) { const keys = securityResearchRuntimeContextKeys; const resourceId = buildSecurityResearchMemoryResourceId( @@ -174,6 +176,7 @@ export function buildSecurityResearchChatRequestContext(input: { [keys.evaluationArm, input.evaluationArm], [keys.terminalProtocolMode, input.terminalProtocolMode], [keys.terminalCleanupRequiredBeforeFinish, input.terminalCleanupRequiredBeforeFinish], + [keys.validationAuthorityMode, input.validationAuthorityMode], [keys.workspaceConfig, input.workspaceConfig], [keys.commandAllowPatterns, input.commandAllowPatterns], [keys.commandBlockPatterns, input.commandBlockPatterns], diff --git a/src/server/validation-plans/authority.ts b/src/server/validation-plans/authority.ts index 04973042b..d9583fe0a 100644 --- a/src/server/validation-plans/authority.ts +++ b/src/server/validation-plans/authority.ts @@ -1,33 +1,69 @@ export const VALIDATION_AUTHORITY_SCHEMA_VERSION = "validation-authority-v1"; -export const VALIDATION_AUTHORITY_MODES = [ - "strict", - "auto", - "self", - "yolo", -] as const; -export type ValidationAuthorityMode = - (typeof VALIDATION_AUTHORITY_MODES)[number]; +export const VALIDATION_AUTHORITY_MODES = ["strict", "auto", "self", "yolo"] as const; +export type ValidationAuthorityMode = (typeof VALIDATION_AUTHORITY_MODES)[number]; export const VALIDATION_AUTHORITY_RECORD_KINDS = [ - "research-execution-profile", - "tool-run", - "validation-plan", - "usage-event", - "eval-row", - "evidence-export", + "research-execution-profile", + "tool-run", + "validation-plan", + "usage-event", + "eval-row", + "evidence-export", ] as const; -export type ValidationAuthorityRecordKind = - (typeof VALIDATION_AUTHORITY_RECORD_KINDS)[number]; +export type ValidationAuthorityRecordKind = (typeof VALIDATION_AUTHORITY_RECORD_KINDS)[number]; export type ValidationAuthoritySelection = { - schemaVersion: typeof VALIDATION_AUTHORITY_SCHEMA_VERSION; - requestedMode: ValidationAuthorityMode; - effectiveMode: ValidationAuthorityMode; + schemaVersion: typeof VALIDATION_AUTHORITY_SCHEMA_VERSION; + requestedMode: ValidationAuthorityMode; + effectiveMode: ValidationAuthorityMode; }; export type ValidationAuthorityProvenance = ValidationAuthoritySelection & { - recordKind: ValidationAuthorityRecordKind; + recordKind: ValidationAuthorityRecordKind; +}; + +export const VALIDATION_OUTCOME_SOURCES = [ + "executor-observed", + "model-asserted", + "human-reviewed", + "artifact-reviewed", +] as const; +export type ValidationOutcomeSource = (typeof VALIDATION_OUTCOME_SOURCES)[number]; + +export type ValidationOutcomeProvenance = Omit & { + recordKind: "validation-plan"; + stage: "payload-result" | "terminal-transition"; + source: ValidationOutcomeSource; + toolRunId?: string; + targetAuthorizationId?: string; + policyDecisionId?: string; + durableApprovalId?: string; + containmentPolicySnapshotId?: string; +}; +export type ValidationPayloadResultProvenance = ValidationOutcomeProvenance & { + stage: "payload-result"; +}; +export type ValidationTerminalTransitionProvenance = ValidationOutcomeProvenance & { + stage: "terminal-transition"; +}; + +/** + * Server-owned facts returned by the validation executor's injected authority + * resolver. These values are deliberately not part of model/tool input. + */ +export type ValidationExecutionAuthority = { + allowed: boolean; + reason?: string; + payloadId: string; + targetId: string; + taskId?: string; + toolRunId: string; + targetAuthorizationId: string; + policyDecisionId: string; + taskScopeVerified: boolean; + durableApprovalId?: string; + containmentPolicySnapshotId?: string; }; /** @@ -36,68 +72,96 @@ export type ValidationAuthorityProvenance = ValidationAuthoritySelection & { * are never aliased or silently downgraded. */ export function resolveValidationAuthoritySelection( - requested: unknown, + requested: unknown, ): ValidationAuthoritySelection { - const requestedMode = - requested === undefined - ? "strict" - : parseValidationAuthorityMode(requested); - return Object.freeze({ - schemaVersion: VALIDATION_AUTHORITY_SCHEMA_VERSION, - requestedMode, - effectiveMode: requestedMode, - }); + const requestedMode = + requested === undefined ? "strict" : parseValidationAuthorityMode(requested); + return Object.freeze({ + schemaVersion: VALIDATION_AUTHORITY_SCHEMA_VERSION, + requestedMode, + effectiveMode: requestedMode, + }); } export function parseValidationAuthorityMode( - value: unknown, - field = "validation authority mode", + value: unknown, + field = "validation authority mode", ): ValidationAuthorityMode { - if ( - typeof value === "string" && - (VALIDATION_AUTHORITY_MODES as readonly string[]).includes(value) - ) { - return value as ValidationAuthorityMode; - } - throw new Error(`${field} must be one of strict, auto, self, yolo.`); + if ( + typeof value === "string" && + (VALIDATION_AUTHORITY_MODES as readonly string[]).includes(value) + ) { + return value as ValidationAuthorityMode; + } + throw new Error(`${field} must be one of strict, auto, self, yolo.`); } -export function readValidationAuthoritySelection( - value: unknown, -): ValidationAuthoritySelection { - if (!value || typeof value !== "object" || Array.isArray(value)) { - throw new Error("Validation authority selection is missing or malformed."); - } - const candidate = value as Partial; - if (candidate.schemaVersion !== VALIDATION_AUTHORITY_SCHEMA_VERSION) { - throw new Error( - "Validation authority selection revision is missing or unsupported.", - ); - } - const requestedMode = parseValidationAuthorityMode( - candidate.requestedMode, - "requested validation authority mode", - ); - const effectiveMode = parseValidationAuthorityMode( - candidate.effectiveMode, - "effective validation authority mode", - ); - if (requestedMode !== effectiveMode) { - throw new Error( - "Validation authority mode cannot change between requested and effective run state.", - ); - } - return Object.freeze({ - schemaVersion: VALIDATION_AUTHORITY_SCHEMA_VERSION, - requestedMode, - effectiveMode, - }); +export function readValidationAuthoritySelection(value: unknown): ValidationAuthoritySelection { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Validation authority selection is missing or malformed."); + } + const candidate = value as Partial; + if (candidate.schemaVersion !== VALIDATION_AUTHORITY_SCHEMA_VERSION) { + throw new Error("Validation authority selection revision is missing or unsupported."); + } + const requestedMode = parseValidationAuthorityMode( + candidate.requestedMode, + "requested validation authority mode", + ); + const effectiveMode = parseValidationAuthorityMode( + candidate.effectiveMode, + "effective validation authority mode", + ); + if (requestedMode !== effectiveMode) { + throw new Error( + "Validation authority mode cannot change between requested and effective run state.", + ); + } + return Object.freeze({ + schemaVersion: VALIDATION_AUTHORITY_SCHEMA_VERSION, + requestedMode, + effectiveMode, + }); +} + +export function projectValidationAuthorityProvenance< + TRecordKind extends ValidationAuthorityRecordKind, +>( + selection: ValidationAuthoritySelection, + recordKind: TRecordKind, +): ValidationAuthorityProvenance & { recordKind: TRecordKind } { + const validated = readValidationAuthoritySelection(selection); + return Object.freeze({ ...validated, recordKind }); } -export function projectValidationAuthorityProvenance( - selection: ValidationAuthoritySelection, - recordKind: ValidationAuthorityRecordKind, -): ValidationAuthorityProvenance { - const validated = readValidationAuthoritySelection(selection); - return Object.freeze({ ...validated, recordKind }); +export function projectValidationOutcomeProvenance< + TStage extends ValidationOutcomeProvenance["stage"], +>(input: { + selection: ValidationAuthoritySelection; + stage: TStage; + source: ValidationOutcomeSource; + authority?: ValidationExecutionAuthority; +}): ValidationOutcomeProvenance & { stage: TStage } { + const provenance: ValidationOutcomeProvenance & { stage: TStage } = { + ...readValidationAuthoritySelection(input.selection), + recordKind: "validation-plan", + stage: input.stage, + source: input.source, + ...(input.authority?.toolRunId ? { toolRunId: input.authority.toolRunId } : {}), + ...(input.authority?.targetAuthorizationId + ? { targetAuthorizationId: input.authority.targetAuthorizationId } + : {}), + ...(input.authority?.policyDecisionId + ? { policyDecisionId: input.authority.policyDecisionId } + : {}), + ...(input.authority?.durableApprovalId + ? { durableApprovalId: input.authority.durableApprovalId } + : {}), + ...(input.authority?.containmentPolicySnapshotId + ? { + containmentPolicySnapshotId: input.authority.containmentPolicySnapshotId, + } + : {}), + }; + return Object.freeze(provenance); } diff --git a/src/server/validation-plans/executor.ts b/src/server/validation-plans/executor.ts index bca4f4435..d192f2452 100644 --- a/src/server/validation-plans/executor.ts +++ b/src/server/validation-plans/executor.ts @@ -9,8 +9,17 @@ import type { } from "./types"; import { projectValidationAuthorityProvenance, + projectValidationOutcomeProvenance, + readValidationAuthoritySelection, resolveValidationAuthoritySelection, + type ValidationAuthorityMode, type ValidationAuthorityProvenance, + type ValidationAuthoritySelection, + type ValidationExecutionAuthority, + type ValidationOutcomeProvenance, + type ValidationOutcomeSource, + type ValidationPayloadResultProvenance, + type ValidationTerminalTransitionProvenance, } from "./authority"; export type ValidationPayloadExecutionContext = { @@ -28,18 +37,33 @@ export type ValidationPlanExecutionInput = { validationAuthorityMode?: unknown; approvalsGranted?: boolean; attackPathId?: string; + modelDecision?: { + proceed: boolean; + confirm: boolean; + }; }; export type ValidationPlanExecutionResult = { planId: string; - validationAuthority: ValidationAuthorityProvenance; + validationAuthority: ValidationAuthorityProvenance & { recordKind: "validation-plan" }; status: ValidationPlan["status"]; confirmationKind?: ValidationPlan["confirmationKind"]; payloadResults: PayloadExecutionResult[]; summary: string; proofNodeId?: string; + terminalTransition: ValidationTerminalTransitionProvenance; +}; + +export type ValidationExecutionAuthorityContext = { + plan: ValidationPlan; + payload: ValidationPayload; + mode: Exclude; }; +export type ValidationExecutionAuthorityResolver = ( + context: ValidationExecutionAuthorityContext, +) => Promise; + export type ValidationPlanExecutorDependencies = { service?: ValidationPlanService; attackPaths?: AttackPathService; @@ -49,6 +73,7 @@ export type ValidationPlanExecutorDependencies = { executeScript?: ValidationPayloadExecutor; executeManual?: ValidationPayloadExecutor; negativeResults?: NegativeResultService; + resolveExecutionAuthority?: ValidationExecutionAuthorityResolver; }; export function createValidationPlanExecutor(deps: ValidationPlanExecutorDependencies = {}) { @@ -56,18 +81,15 @@ export function createValidationPlanExecutor(deps: ValidationPlanExecutorDepende return { async executePlan(input: ValidationPlanExecutionInput): Promise { - const validationAuthority = resolveValidationAuthoritySelection( - input.validationAuthorityMode, - ); const finalized = await service.finalizePlan(input.planId); if (!finalized.ready) { throw new Error(`validation-plans: plan is not executable: ${finalized.reason}`); } - await service.assertSafeToRun({ - planId: input.planId, - approvalsGranted: input.approvalsGranted === true, - }); + const validationAuthority = resolvePinnedAuthority( + finalized.plan, + input.validationAuthorityMode, + ); const validationPlanAuthority = projectValidationAuthorityProvenance( validationAuthority, @@ -79,21 +101,79 @@ export function createValidationPlanExecutor(deps: ValidationPlanExecutorDepende validationAuthority: validationPlanAuthority, }, }); + const authorities = await resolvePayloadAuthorities({ + deps, + plan: finalized.plan, + mode: validationAuthority.effectiveMode, + }); + await service.assertSafeToRun({ + planId: input.planId, + approvalsGranted: executionApprovalGranted({ + mode: validationAuthority.effectiveMode, + modelClaim: input.approvalsGranted === true, + plan: finalized.plan, + authorities, + }), + }); const running = await service.startRun({ planId: input.planId }); const payloadResults: PayloadExecutionResult[] = []; for (const payload of running.payloads) { - const result = await executePayload({ - deps, + const authority = authorities.get(payload.id); + const denial = executionDenial({ + mode: validationAuthority.effectiveMode, plan: running, payload, - approvalsGranted: input.approvalsGranted === true, + authority, + modelDecision: input.modelDecision, }); + const source = outcomeSource(validationAuthority.effectiveMode); + const result = denial + ? deniedPayloadResult({ + payload, + denial, + provenance: projectValidationOutcomeProvenance({ + selection: validationAuthority, + stage: "payload-result", + source, + ...(authority ? { authority } : {}), + }), + }) + : await executePayload({ + deps, + plan: running, + payload, + approvalsGranted: executorApprovalGranted({ + mode: validationAuthority.effectiveMode, + modelClaim: input.approvalsGranted === true, + authority, + }), + provenance: projectValidationOutcomeProvenance({ + selection: validationAuthority, + stage: "payload-result", + source, + ...(authority ? { authority } : {}), + }), + requireDurableEvidence: + validationAuthority.effectiveMode === "strict" || + validationAuthority.effectiveMode === "auto", + }); payloadResults.push(result); await service.recordPayloadResult({ planId: running.id, ...result }); } - const terminal = await settlePlan(service, running.id, payloadResults); + const terminalTransition = projectValidationOutcomeProvenance({ + selection: validationAuthority, + stage: "terminal-transition", + source: terminalOutcomeSource(validationAuthority.effectiveMode, payloadResults), + }); + const terminal = await settlePlan( + service, + running.id, + payloadResults, + terminalTransition, + validationAuthority.effectiveMode !== "self" || input.modelDecision?.confirm === true, + ); await recordValidationNegativeResults({ negativeResults: deps.negativeResults, plan: terminal, @@ -116,6 +196,7 @@ export function createValidationPlanExecutor(deps: ValidationPlanExecutorDepende payloadResults, summary: summarizeExecution(terminal, payloadResults), ...(proofNodeId ? { proofNodeId } : {}), + terminalTransition, }; }, }; @@ -201,6 +282,8 @@ async function executePayload(input: { plan: ValidationPlan; payload: ValidationPayload; approvalsGranted: boolean; + provenance: ValidationPayloadResultProvenance; + requireDurableEvidence: boolean; }): Promise { const context: ValidationPayloadExecutionContext = { plan: input.plan, @@ -215,10 +298,23 @@ async function executePayload(input: { observed: `No executor is configured for ${input.payload.kind} validation payloads.`, evidence: [], error: "executor-not-configured", + authorityProvenance: input.provenance, }; } try { - return await executor(context); + const result = await executor(context); + if (input.requireDurableEvidence && result.status === "pass" && result.evidence.length === 0) { + return { + ...result, + status: "inconclusive", + error: "durable-evidence-required", + authorityProvenance: input.provenance, + }; + } + return { + ...result, + authorityProvenance: trustedExecutorProvenance(result, input.provenance), + }; } catch (error) { const message = error instanceof Error ? error.message : String(error); return { @@ -227,6 +323,7 @@ async function executePayload(input: { observed: `Validation payload failed: ${message}`, evidence: [], error: message, + authorityProvenance: input.provenance, }; } } @@ -255,10 +352,182 @@ const defaultManualExecutor: ValidationPayloadExecutor = async ({ payload }) => evidence: [], }); +function resolvePinnedAuthority( + plan: ValidationPlan, + requested: unknown, +): ValidationAuthoritySelection { + const persisted = plan.metadata?.validationAuthority; + const selection = + requested === undefined && persisted !== undefined + ? readValidationAuthoritySelection(persisted) + : resolveValidationAuthoritySelection(requested); + if (persisted !== undefined) { + const pinned = readValidationAuthoritySelection(persisted); + if ( + pinned.requestedMode !== selection.requestedMode || + pinned.effectiveMode !== selection.effectiveMode + ) { + throw new Error( + `validation-plans: authority mode is pinned to ${pinned.effectiveMode} and cannot change mid-run`, + ); + } + } + return selection; +} + +async function resolvePayloadAuthorities(input: { + deps: ValidationPlanExecutorDependencies; + plan: ValidationPlan; + mode: ValidationAuthorityMode; +}): Promise> { + const authorities = new Map(); + if (input.mode === "yolo" || !input.deps.resolveExecutionAuthority) { + return authorities; + } + for (const payload of input.plan.payloads) { + const authority = await input.deps.resolveExecutionAuthority({ + plan: input.plan, + payload, + mode: input.mode, + }); + authorities.set(payload.id, authority); + } + return authorities; +} + +function executionApprovalGranted(input: { + mode: ValidationAuthorityMode; + modelClaim: boolean; + plan: ValidationPlan; + authorities: Map; +}): boolean { + if (input.mode === "yolo") { + return input.modelClaim; + } + if (input.mode === "auto" || input.mode === "self") { + // The model-authored boolean is ignored. Per-payload execution remains + // fail-closed on the injected server authority below. + return true; + } + return input.plan.payloads.every((payload) => { + if (payload.isNonDestructive) return true; + const authority = input.authorities.get(payload.id); + if (!authorityIsValid(input.plan, payload, authority)) return false; + return input.mode !== "strict" || Boolean(authority?.durableApprovalId); + }); +} + +function executorApprovalGranted(input: { + mode: ValidationAuthorityMode; + modelClaim: boolean; + authority?: ValidationExecutionAuthority; +}): boolean { + if (input.mode === "yolo") return input.modelClaim; + if (input.mode === "strict") return Boolean(input.authority?.durableApprovalId); + return input.authority?.allowed === true; +} + +function executionDenial(input: { + mode: ValidationAuthorityMode; + plan: ValidationPlan; + payload: ValidationPayload; + authority?: ValidationExecutionAuthority; + modelDecision?: ValidationPlanExecutionInput["modelDecision"]; +}): string | undefined { + if (input.mode === "yolo") return undefined; + if (input.mode === "self" && input.modelDecision?.proceed !== true) { + return "model-declined-execution"; + } + if (input.mode === "strict" && input.payload.isNonDestructive) { + return undefined; + } + if (!input.authority) return "server-authority-not-resolved"; + if (!authorityIsValid(input.plan, input.payload, input.authority)) { + return input.authority.reason ?? "server-authority-mismatch"; + } + if ( + input.mode === "strict" && + !input.payload.isNonDestructive && + !input.authority.durableApprovalId + ) { + return "exact-durable-approval-required"; + } + return undefined; +} + +function authorityIsValid( + plan: ValidationPlan, + payload: ValidationPayload, + authority: ValidationExecutionAuthority | undefined, +): boolean { + return Boolean( + authority?.allowed && + authority.taskScopeVerified && + authority.payloadId === payload.id && + authority.targetId === plan.targetId && + (plan.taskId === undefined || authority.taskId === plan.taskId) && + authority.toolRunId.trim() && + authority.targetAuthorizationId.trim() && + authority.policyDecisionId.trim(), + ); +} + +function deniedPayloadResult(input: { + payload: ValidationPayload; + denial: string; + provenance: ValidationPayloadResultProvenance; +}): PayloadExecutionResult { + return { + payloadId: input.payload.id, + status: "inconclusive", + observed: `Validation payload was not executed: ${input.denial}.`, + evidence: [], + error: input.denial, + authorityProvenance: input.provenance, + }; +} + +function outcomeSource(mode: ValidationAuthorityMode): ValidationOutcomeSource { + return mode === "self" || mode === "yolo" ? "model-asserted" : "executor-observed"; +} + +function trustedExecutorProvenance( + result: PayloadExecutionResult, + fallback: ValidationPayloadResultProvenance, +): ValidationPayloadResultProvenance { + const candidate = result.authorityProvenance; + if (candidate?.source === "human-reviewed" || candidate?.source === "artifact-reviewed") { + return { ...fallback, source: candidate.source }; + } + return fallback; +} + +function terminalOutcomeSource( + mode: ValidationAuthorityMode, + results: PayloadExecutionResult[], +): ValidationOutcomeSource { + if (mode === "self" || mode === "yolo") return "model-asserted"; + const passingSources = results + .filter((result) => result.status === "pass") + .map((result) => result.authorityProvenance?.source); + if (passingSources.length > 0 && passingSources.every((source) => source === "human-reviewed")) { + return "human-reviewed"; + } + if ( + passingSources.length > 0 && + passingSources.every((source) => source === "human-reviewed" || source === "artifact-reviewed") + ) { + return "artifact-reviewed"; + } + return "executor-observed"; +} + async function settlePlan( service: ValidationPlanService, planId: string, payloadResults: PayloadExecutionResult[], + authorityProvenance: ValidationTerminalTransitionProvenance, + confirmationAuthorized: boolean, ): Promise { const failed = payloadResults.filter((result) => result.status === "fail"); const passed = payloadResults.filter((result) => result.status === "pass"); @@ -267,19 +536,25 @@ async function settlePlan( return service.reject({ planId, reason: `${failed.length}/${payloadResults.length} validation payloads failed.`, + authorityProvenance, }); } - if (passed.length > 0) { + if (passed.length > 0 && confirmationAuthorized) { return service.confirm({ planId, confirmationKind: passed.some(hasExploitabilityProof) ? "exploitability-verified" : "evidence-reviewed", + authorityProvenance, }); } return service.markInconclusive({ planId, - reason: "Validation ran but produced no passing payloads.", + reason: + passed.length > 0 + ? "Validation produced passing payloads but the selected authority did not confirm them." + : "Validation ran but produced no passing payloads.", + authorityProvenance, }); } diff --git a/src/server/validation-plans/index.ts b/src/server/validation-plans/index.ts index eb65159c5..ecc38907d 100644 --- a/src/server/validation-plans/index.ts +++ b/src/server/validation-plans/index.ts @@ -22,6 +22,7 @@ export { export { parseValidationAuthorityMode, projectValidationAuthorityProvenance, + projectValidationOutcomeProvenance, readValidationAuthoritySelection, resolveValidationAuthoritySelection, VALIDATION_AUTHORITY_MODES, @@ -31,6 +32,11 @@ export { type ValidationAuthorityProvenance, type ValidationAuthorityRecordKind, type ValidationAuthoritySelection, + type ValidationExecutionAuthority, + type ValidationPayloadResultProvenance, + type ValidationTerminalTransitionProvenance, + type ValidationOutcomeProvenance, + type ValidationOutcomeSource, } from "./authority"; export { createValidationPlanExecutor, diff --git a/src/server/validation-plans/repository.ts b/src/server/validation-plans/repository.ts index bbe899d8c..1e4a49e8b 100644 --- a/src/server/validation-plans/repository.ts +++ b/src/server/validation-plans/repository.ts @@ -228,6 +228,9 @@ function cloneResult(result: PayloadExecutionResult): PayloadExecutionResult { ...(result.proof ? { proof: { ...result.proof, rawArtifactIds: [...result.proof.rawArtifactIds] } } : {}), + ...(result.authorityProvenance + ? { authorityProvenance: { ...result.authorityProvenance } } + : {}), }; } diff --git a/src/server/validation-plans/service.ts b/src/server/validation-plans/service.ts index 2b9222151..4365d8317 100644 --- a/src/server/validation-plans/service.ts +++ b/src/server/validation-plans/service.ts @@ -1,4 +1,9 @@ import { createId } from "../../lib/ids"; +import { + readValidationAuthoritySelection, + type ValidationOutcomeProvenance, + type ValidationTerminalTransitionProvenance, +} from "./authority"; import type { AddPayloadInput, AssertSafeToRunInput, @@ -8,6 +13,7 @@ import type { FinalizePlanResult, ListValidationPlansQuery, MarkValidationPlanInconclusiveInput, + PayloadExecutionResult, RecordPayloadResultInput, RejectPlanInput, StartValidationPlanRunInput, @@ -36,7 +42,14 @@ class ValidationPlanServiceImpl implements ValidationPlanService { id: string, input: UpdateValidationPlanInput, ): Promise { - return this.repository.update(id, { ...input, now: input.now ?? new Date().toISOString() }); + const current = await this.repository.get(id); + if (!current) return undefined; + const metadata = preservePinnedAuthority(current, input.metadata); + return this.repository.update(id, { + ...input, + ...(metadata ? { metadata } : {}), + now: input.now ?? new Date().toISOString(), + }); } async getPlan(id: string): Promise { @@ -53,6 +66,7 @@ class ValidationPlanServiceImpl implements ValidationPlanService { async recordPayloadResult(input: RecordPayloadResultInput): Promise { const now = input.now ?? new Date().toISOString(); + const plan = await this.requirePlan(input.planId); const result = { payloadId: input.payloadId, status: input.status, @@ -60,7 +74,9 @@ class ValidationPlanServiceImpl implements ValidationPlanService { evidence: input.evidence, ...(input.error ? { error: input.error } : {}), ...(input.proof ? { proof: input.proof } : {}), + ...(input.authorityProvenance ? { authorityProvenance: input.authorityProvenance } : {}), }; + assertPayloadResultAuthority(plan, result); return this.repository.recordPayloadResult(input.planId, result, now); } @@ -85,6 +101,7 @@ class ValidationPlanServiceImpl implements ValidationPlanService { async confirm(input: ConfirmPlanInput): Promise { const plan = await this.requirePlan(input.planId); + assertTerminalAuthority(plan, input.authorityProvenance); if (plan.payloadResults.length === 0) { throw new Error("validation-plans: cannot confirm with no payload results recorded"); } @@ -115,13 +132,22 @@ class ValidationPlanServiceImpl implements ValidationPlanService { confidence, confirmationKind, confirmedAt: now, + ...(input.authorityProvenance + ? { + metadata: { + ...(plan.metadata ?? {}), + terminalTransition: input.authorityProvenance, + }, + } + : {}), now, })) ?? base ); } async reject(input: RejectPlanInput): Promise { - await this.requirePlan(input.planId); + const plan = await this.requirePlan(input.planId); + assertTerminalAuthority(plan, input.authorityProvenance); const now = input.now ?? new Date().toISOString(); const base = await this.repository.setStatus(input.planId, "rejected", now); if (!base) { @@ -130,13 +156,22 @@ class ValidationPlanServiceImpl implements ValidationPlanService { return ( (await this.repository.update(input.planId, { rejectedReason: input.reason, + ...(input.authorityProvenance + ? { + metadata: { + ...(plan.metadata ?? {}), + terminalTransition: input.authorityProvenance, + }, + } + : {}), now, })) ?? base ); } async markInconclusive(input: MarkValidationPlanInconclusiveInput): Promise { - await this.requirePlan(input.planId); + const plan = await this.requirePlan(input.planId); + assertTerminalAuthority(plan, input.authorityProvenance); const now = input.now ?? new Date().toISOString(); const base = await this.repository.setStatus(input.planId, "inconclusive", now); if (!base) { @@ -145,6 +180,14 @@ class ValidationPlanServiceImpl implements ValidationPlanService { return ( (await this.repository.update(input.planId, { rejectedReason: input.reason, + ...(input.authorityProvenance + ? { + metadata: { + ...(plan.metadata ?? {}), + terminalTransition: input.authorityProvenance, + }, + } + : {}), now, })) ?? base ); @@ -200,6 +243,101 @@ class ValidationPlanServiceImpl implements ValidationPlanService { } } +function preservePinnedAuthority( + plan: ValidationPlan, + next: Record | undefined, +): Record | undefined { + const pinned = plan.metadata?.validationAuthority; + if (pinned === undefined) return next; + const selection = readValidationAuthoritySelection(pinned); + if (next?.validationAuthority !== undefined) { + const requested = readValidationAuthoritySelection(next.validationAuthority); + if ( + requested.requestedMode !== selection.requestedMode || + requested.effectiveMode !== selection.effectiveMode + ) { + throw new Error( + `validation-plans: authority mode is pinned to ${selection.effectiveMode} and cannot change mid-run`, + ); + } + } + return { ...(next ?? plan.metadata ?? {}), validationAuthority: pinned }; +} + +function assertPayloadResultAuthority(plan: ValidationPlan, result: PayloadExecutionResult) { + const selection = readPinnedSelection(plan); + if (!selection) return; + const provenance = result.authorityProvenance; + assertMatchingProvenance(selection, provenance, "payload-result"); + const mode = selection.effectiveMode; + if ((mode === "strict" || mode === "auto") && provenance?.source === "model-asserted") { + throw new Error(`validation-plans: ${mode} mode rejects model-asserted payload results`); + } + if ((mode === "self" || mode === "yolo") && provenance?.source !== "model-asserted") { + throw new Error(`validation-plans: ${mode} mode requires model-attributed payload results`); + } + if (result.status !== "pass") return; + if ((mode === "strict" || mode === "auto") && result.evidence.length === 0) { + throw new Error(`validation-plans: ${mode} mode requires durable evidence for passing results`); + } + if ( + mode === "auto" && + (!provenance?.toolRunId || !provenance.targetAuthorizationId || !provenance.policyDecisionId) + ) { + throw new Error( + "validation-plans: auto mode requires Tool Run, target authorization, and policy provenance", + ); + } + const payload = plan.payloads.find((candidate) => candidate.id === result.payloadId); + if (mode === "strict" && payload && !payload.isNonDestructive && !provenance?.durableApprovalId) { + throw new Error("validation-plans: strict mode requires exact durable approval provenance"); + } +} + +function assertTerminalAuthority( + plan: ValidationPlan, + provenance: ValidationTerminalTransitionProvenance | undefined, +) { + const selection = readPinnedSelection(plan); + if (!selection) return; + assertMatchingProvenance(selection, provenance, "terminal-transition"); + const modelOwned = provenance?.source === "model-asserted"; + if ((selection.effectiveMode === "strict" || selection.effectiveMode === "auto") && modelOwned) { + throw new Error( + `validation-plans: ${selection.effectiveMode} mode rejects model-asserted terminal transitions`, + ); + } + if ((selection.effectiveMode === "self" || selection.effectiveMode === "yolo") && !modelOwned) { + throw new Error( + `validation-plans: ${selection.effectiveMode} mode requires model-attributed terminal transitions`, + ); + } +} + +function readPinnedSelection(plan: ValidationPlan) { + return plan.metadata?.validationAuthority === undefined + ? undefined + : readValidationAuthoritySelection(plan.metadata.validationAuthority); +} + +function assertMatchingProvenance( + selection: ReturnType, + provenance: ValidationOutcomeProvenance | undefined, + stage: ValidationOutcomeProvenance["stage"], +) { + if (!provenance) { + throw new Error(`validation-plans: pinned authority requires ${stage} provenance`); + } + const recorded = readValidationAuthoritySelection(provenance); + if ( + recorded.requestedMode !== selection.requestedMode || + recorded.effectiveMode !== selection.effectiveMode || + provenance.stage !== stage + ) { + throw new Error("validation-plans: outcome provenance does not match pinned authority"); + } +} + function hasExploitabilityProof(result: { proof?: import("./types").ValidationProof }) { const proof = result.proof; return Boolean( diff --git a/src/server/validation-plans/types.ts b/src/server/validation-plans/types.ts index 066c95b0f..28e7ffab2 100644 --- a/src/server/validation-plans/types.ts +++ b/src/server/validation-plans/types.ts @@ -12,6 +12,11 @@ export type PayloadKind = "http-request" | "command" | "browser-action" | "scrip export type ValidationConfidence = "low" | "medium" | "high"; export type ValidationConfirmationKind = "evidence-reviewed" | "exploitability-verified"; +import type { + ValidationPayloadResultProvenance, + ValidationTerminalTransitionProvenance, +} from "./authority"; + export type ValidationProof = { executionProvenance: "recorded-artifacts" | "isolated-execution" | "authorized-target-execution"; sourceRevision: string; @@ -151,6 +156,7 @@ export type PayloadExecutionResult = { evidence: string[]; error?: string; proof?: ValidationProof; + authorityProvenance?: ValidationPayloadResultProvenance; }; export type RecordPayloadResultInput = PayloadExecutionResult & { @@ -171,12 +177,14 @@ export type ConfirmPlanInput = { planId: string; confirmationKind?: ValidationConfirmationKind; now?: string; + authorityProvenance?: ValidationTerminalTransitionProvenance; }; export type RejectPlanInput = { planId: string; reason: string; now?: string; + authorityProvenance?: ValidationTerminalTransitionProvenance; }; export type CancelPlanInput = { @@ -193,6 +201,7 @@ export type MarkValidationPlanInconclusiveInput = { planId: string; reason: string; now?: string; + authorityProvenance?: ValidationTerminalTransitionProvenance; }; export type StartValidationPlanRunInput = { diff --git a/tests/integration/validation-authority-provenance.test.ts b/tests/integration/validation-authority-provenance.test.ts index a4cf911cb..d421af54b 100644 --- a/tests/integration/validation-authority-provenance.test.ts +++ b/tests/integration/validation-authority-provenance.test.ts @@ -2,70 +2,251 @@ import { describe, expect, it } from "vitest"; import { SECURITY_CAPABILITY_MANIFEST_REVISION } from "../../src/lib/tools/catalog"; import { - projectResearchExecutionProfileToRuntime, - resolveResearchExecutionProfile, + projectResearchExecutionProfileToRuntime, + resolveResearchExecutionProfile, } from "../../src/server/research/execution-profile"; import { - projectValidationAuthorityProvenance, - resolveValidationAuthoritySelection, - VALIDATION_AUTHORITY_RECORD_KINDS, -} from "../../src/server/validation-plans/authority"; + type AddPayloadInput, + createInMemoryValidationPlanRepository, + createValidationPlanExecutor, + createValidationPlanService, + projectValidationAuthorityProvenance, + resolveValidationAuthoritySelection, + type ValidationAuthorityMode, + VALIDATION_AUTHORITY_RECORD_KINDS, +} from "../../src/server/validation-plans"; + +const guardedPayload: AddPayloadInput = { + id: "payload-1", + kind: "http-request", + name: "same guarded validation fixture", + request: { method: "POST", url: "https://target.test/validate" }, + expectedBehavior: "validated marker is returned", + expectedEvidence: ["artifact-validation-1"], + stopConditions: ["scope mismatch"], + cleanup: "none", + isNonDestructive: false, + requiresApproval: true, +}; describe("validation authority provenance", () => { - it("pins requested and effective authority across every durable record projection", () => { - const profile = resolveResearchExecutionProfile( - { - validationAuthorityMode: "self", - modelUri: "llm://lmstudio/qwen/qwen3-30b-a3b?maxTokens=4096", - capabilityManifestRevision: SECURITY_CAPABILITY_MANIFEST_REVISION, - capabilityIds: ["tool:artifactAccessTool"], - skillRegistryRevision: "skill-registry-test-revision", - targetMode: "none", - networkPolicy: "none", - uiCollection: "evidence", - policies: { approval: "durable-intent-v1" }, - }, - { - capabilityManifestRevision: SECURITY_CAPABILITY_MANIFEST_REVISION, - skillRegistryRevision: "skill-registry-test-revision", - }, - ); + it("pins requested and effective authority across every durable record projection", () => { + const profile = resolveResearchExecutionProfile( + { + validationAuthorityMode: "self", + modelUri: "llm://lmstudio/qwen/qwen3-30b-a3b?maxTokens=4096", + capabilityManifestRevision: SECURITY_CAPABILITY_MANIFEST_REVISION, + capabilityIds: ["tool:artifactAccessTool"], + skillRegistryRevision: "skill-registry-test-revision", + targetMode: "none", + networkPolicy: "none", + uiCollection: "evidence", + policies: { approval: "durable-intent-v1" }, + }, + { + capabilityManifestRevision: SECURITY_CAPABILITY_MANIFEST_REVISION, + capabilityIds: ["tool:artifactAccessTool"], + skillRegistryRevision: "skill-registry-test-revision", + selectedSkills: [], + }, + ); + + expect(profile.requested.validationAuthorityMode).toBe("self"); + expect(profile.effective.validationAuthority).toMatchObject({ + requestedMode: "self", + effectiveMode: "self", + }); + expect(projectResearchExecutionProfileToRuntime(profile)).toMatchObject({ + validationAuthorityMode: "self", + validationAuthority: { + requestedMode: "self", + effectiveMode: "self", + recordKind: "research-execution-profile", + }, + }); + + const selection = resolveValidationAuthoritySelection("self"); + const projections = VALIDATION_AUTHORITY_RECORD_KINDS.map((recordKind) => + projectValidationAuthorityProvenance(selection, recordKind), + ); + expect(projections.map((entry) => entry.recordKind)).toEqual(VALIDATION_AUTHORITY_RECORD_KINDS); + expect(projections).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + requestedMode: "self", + effectiveMode: "self", + }), + ]), + ); + + expect(() => resolveValidationAuthoritySelection("manual")).toThrow( + "must be one of strict, auto, self, yolo", + ); + expect(() => resolveValidationAuthoritySelection("autonomous")).toThrow( + "must be one of strict, auto, self, yolo", + ); + }); + + it.each([ + ["strict", "executor-observed", true], + ["auto", "executor-observed", false], + ["self", "model-asserted", false], + ["yolo", "model-asserted", false], + ] as const)( + "executes the same fixture under %s with mode-specific authority and durable provenance", + async (mode, source, needsDurableApproval) => { + const service = createValidationPlanService(createInMemoryValidationPlanRepository()); + const plan = await service.createPlan({ + id: `plan-${mode}`, + projectId: "project-1", + targetId: "target-1", + taskId: "task-1", + title: "Matched validation fixture", + expectedEvidence: ["artifact-validation-1"], + payloads: [guardedPayload], + }); + let calls = 0; + const executor = createValidationPlanExecutor({ + service, + ...(mode !== "yolo" + ? { + resolveExecutionAuthority: async ({ payload }: { payload: { id: string } }) => ({ + allowed: true, + payloadId: payload.id, + targetId: "target-1", + taskId: "task-1", + toolRunId: `tool-run-${mode}`, + targetAuthorizationId: `target-auth-${mode}`, + policyDecisionId: `policy-${mode}`, + taskScopeVerified: true, + ...(needsDurableApproval ? { durableApprovalId: "approval-strict" } : {}), + containmentPolicySnapshotId: `containment-${mode}`, + }), + } + : {}), + executeHttpRequest: async ({ payload }) => { + calls += 1; + return { + payloadId: payload.id, + status: "pass", + observed: "validated marker returned", + evidence: ["artifact-validation-1"], + }; + }, + }); + + const result = await executor.executePlan({ + planId: plan.id, + validationAuthorityMode: mode, + approvalsGranted: true, + ...(mode === "self" ? { modelDecision: { proceed: true, confirm: true } } : {}), + }); + const persisted = await service.getPlan(plan.id); + + expect(calls).toBe(1); + expect(result).toMatchObject({ + status: "confirmed", + validationAuthority: { requestedMode: mode, effectiveMode: mode }, + payloadResults: [ + { + status: "pass", + authorityProvenance: { + requestedMode: mode, + effectiveMode: mode, + stage: "payload-result", + source, + }, + }, + ], + terminalTransition: { + requestedMode: mode, + effectiveMode: mode, + stage: "terminal-transition", + source, + }, + }); + expect(persisted).toMatchObject({ + metadata: { + validationAuthority: { requestedMode: mode, effectiveMode: mode }, + terminalTransition: { + requestedMode: mode, + effectiveMode: mode, + source, + }, + }, + payloadResults: [ + { + authorityProvenance: { + requestedMode: mode, + effectiveMode: mode, + source, + }, + }, + ], + }); + }, + ); - expect(profile.requested.validationAuthorityMode).toBe("self"); - expect(profile.effective.validationAuthority).toMatchObject({ - requestedMode: "self", - effectiveMode: "self", - }); - expect(projectResearchExecutionProfileToRuntime(profile)).toMatchObject({ - validationAuthorityMode: "self", - validationAuthority: { - requestedMode: "self", - effectiveMode: "self", - recordKind: "research-execution-profile", - }, - }); + it("ignores model approval claims outside yolo and keeps the pinned mode immutable", async () => { + const modes: ValidationAuthorityMode[] = ["strict", "auto", "self"]; + for (const mode of modes) { + const service = createValidationPlanService(createInMemoryValidationPlanRepository()); + const plan = await service.createPlan({ + id: `denied-${mode}`, + projectId: "project-1", + targetId: "target-1", + taskId: "task-1", + title: "Unbacked model authority claim", + expectedEvidence: ["artifact-validation-1"], + payloads: [guardedPayload], + }); + let calls = 0; + const executor = createValidationPlanExecutor({ + service, + executeHttpRequest: async ({ payload }) => { + calls += 1; + return { + payloadId: payload.id, + status: "pass", + observed: "should not run", + evidence: ["artifact-validation-1"], + }; + }, + }); - const selection = resolveValidationAuthoritySelection("self"); - const projections = VALIDATION_AUTHORITY_RECORD_KINDS.map((recordKind) => - projectValidationAuthorityProvenance(selection, recordKind), - ); - expect(projections.map((entry) => entry.recordKind)).toEqual( - VALIDATION_AUTHORITY_RECORD_KINDS, - ); - expect(projections).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - requestedMode: "self", - effectiveMode: "self", - }), - ]), - ); + if (mode === "strict") { + await expect( + executor.executePlan({ + planId: plan.id, + validationAuthorityMode: mode, + approvalsGranted: true, + }), + ).rejects.toThrow("destructive payload requires approval"); + } else { + await expect( + executor.executePlan({ + planId: plan.id, + validationAuthorityMode: mode, + approvalsGranted: true, + ...(mode === "self" ? { modelDecision: { proceed: true, confirm: true } } : {}), + }), + ).resolves.toMatchObject({ + status: "inconclusive", + payloadResults: [{ error: "server-authority-not-resolved" }], + }); + } + expect(calls).toBe(0); - expect(() => resolveValidationAuthoritySelection("manual")).toThrow( - "must be one of strict, auto, self, yolo", - ); - expect(() => resolveValidationAuthoritySelection("autonomous")).toThrow( - "must be one of strict, auto, self, yolo", - ); - }); + const persisted = await service.getPlan(plan.id); + expect(persisted?.metadata?.validationAuthority).toMatchObject({ + requestedMode: mode, + effectiveMode: mode, + }); + if (mode === "strict") { + await expect( + executor.executePlan({ planId: plan.id, validationAuthorityMode: "yolo" }), + ).rejects.toThrow("authority mode is pinned to strict"); + } + } + }); }); diff --git a/tests/integration/validation-plan-browser-script-executor.test.ts b/tests/integration/validation-plan-browser-script-executor.test.ts index 453c2b088..53a949f28 100644 --- a/tests/integration/validation-plan-browser-script-executor.test.ts +++ b/tests/integration/validation-plan-browser-script-executor.test.ts @@ -377,6 +377,7 @@ describe("browser and script validation plan executors", () => { projectId: "project-1", threadId: "thread-1", targetMode: "container", + validationAuthorityMode: "yolo", }), } as never; const created = await validationPlanTool.execute?.( diff --git a/tests/integration/validation-plan-tool.test.ts b/tests/integration/validation-plan-tool.test.ts index d2ca36e25..3ed69de69 100644 --- a/tests/integration/validation-plan-tool.test.ts +++ b/tests/integration/validation-plan-tool.test.ts @@ -40,7 +40,11 @@ describe("validation plan Mastra tool", () => { it("creates, finalizes, records, confirms, and lists reproducible validation plans", async () => { const context = { - requestContext: requestContext({ projectId: "project-1", threadId: "thread-1" }), + requestContext: requestContext({ + projectId: "project-1", + threadId: "thread-1", + validationAuthorityMode: "yolo", + }), } as never; const created = await validationPlanTool.execute?.( @@ -256,7 +260,11 @@ describe("validation plan Mastra tool", () => { it("requires a complete typed proof before marking exploitability verified", async () => { const context = { - requestContext: requestContext({ projectId: "project-proof", threadId: "thread-1" }), + requestContext: requestContext({ + projectId: "project-proof", + threadId: "thread-1", + validationAuthorityMode: "yolo", + }), } as never; const created = await validationPlanTool.execute?.( { From 564b052fd36f7c1b6714c789db6e4c30449f5f76 Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Thu, 27 Aug 2026 11:47:01 -0400 Subject: [PATCH 3/5] Carry validation authority into background work --- .../chat/security-research-runtime-context.ts | 1 + src/server/research/execution-profile.ts | 849 ++++++++---------- .../scheduler-backed-research.test.ts | 60 +- .../validation-authority-provenance.test.ts | 7 + 4 files changed, 432 insertions(+), 485 deletions(-) diff --git a/src/server/chat/security-research-runtime-context.ts b/src/server/chat/security-research-runtime-context.ts index 0152c6405..6c4da4d32 100644 --- a/src/server/chat/security-research-runtime-context.ts +++ b/src/server/chat/security-research-runtime-context.ts @@ -272,6 +272,7 @@ export function buildSecurityResearchStageRequestContext(input: { [keys.skillRegistryRevision, input.executionProfile?.skillRegistryRevision], [keys.selectedSkillRefs, input.executionProfile?.selectedSkills], [keys.runtimeSkillCapabilities, input.executionProfile?.capabilityIds], + [keys.validationAuthorityMode, input.executionProfile?.validationAuthority.effectiveMode], [keys.modelUri, input.executionProfile?.models.coordinator], [keys.coordinatorModelUri, input.executionProfile?.models.coordinator], [keys.modelOverrides, input.executionProfile?.models], diff --git a/src/server/research/execution-profile.ts b/src/server/research/execution-profile.ts index e02232318..e3ec6605a 100644 --- a/src/server/research/execution-profile.ts +++ b/src/server/research/execution-profile.ts @@ -3,558 +3,493 @@ import { createHash } from "node:crypto"; import { normalize, parse } from "llm-strings"; import { - asModelConnectionString, - findModelEntryForRef, - MODEL_OVERRIDE_TARGETS, - type ModelOverrideMap, - type ModelOverrideTarget, - modelProviderFromHost, - readModelOverride, + asModelConnectionString, + findModelEntryForRef, + MODEL_OVERRIDE_TARGETS, + type ModelOverrideMap, + type ModelOverrideTarget, + modelProviderFromHost, + readModelOverride, } from "../../lib/models"; import type { SecurityResearchComponentCollection } from "../../lib/render-json/security-research-catalog"; import { - getMastraModelRuntimeOptions, - resolveSecurityResearchMastraModelUri, + getMastraModelRuntimeOptions, + resolveSecurityResearchMastraModelUri, } from "../../mastra/config/model"; import { withDatabase } from "../db/client"; import type { JsonObject } from "../db/types"; import { - projectValidationAuthorityProvenance, - readValidationAuthoritySelection, - resolveValidationAuthoritySelection, - type ValidationAuthorityMode, - type ValidationAuthorityProvenance, - type ValidationAuthoritySelection, + projectValidationAuthorityProvenance, + readValidationAuthoritySelection, + resolveValidationAuthoritySelection, + type ValidationAuthorityMode, + type ValidationAuthorityProvenance, + type ValidationAuthoritySelection, } from "../validation-plans/authority"; -export const RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION = - "research-execution-profile-v1"; +export const RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION = "research-execution-profile-v1"; export const RESEARCH_EXECUTION_PROFILE_METADATA_KEY = "executionProfile"; export type ResearchExecutionTargetMode = "none" | "container" | "remote"; export type RequestedResearchExecutionProfile = { - validationAuthorityMode?: unknown; - modelUri?: string; - runtimeModelUri?: string; - modelOverrides?: ModelOverrideMap; - contextWindowTokens?: number; - maxOutputTokens?: number; - capabilityManifestRevision: string; - capabilityIds: readonly string[]; - skillRegistryRevision: string; - selectedSkills?: readonly { id: string; revision: string }[]; - targetMode: ResearchExecutionTargetMode; - networkPolicy: string; - maxToolCalls?: number; - maxRuntimeMs?: number; - maxTurns?: number; - maxCostUsd?: number; - uiCollection: SecurityResearchComponentCollection; - policies: Readonly>; + validationAuthorityMode?: unknown; + modelUri?: string; + runtimeModelUri?: string; + modelOverrides?: ModelOverrideMap; + contextWindowTokens?: number; + maxOutputTokens?: number; + capabilityManifestRevision: string; + capabilityIds: readonly string[]; + skillRegistryRevision: string; + selectedSkills?: readonly { id: string; revision: string }[]; + targetMode: ResearchExecutionTargetMode; + networkPolicy: string; + maxToolCalls?: number; + maxRuntimeMs?: number; + maxTurns?: number; + maxCostUsd?: number; + uiCollection: SecurityResearchComponentCollection; + policies: Readonly>; }; export type ResearchExecutionModel = { - modelUri: string; - provider: string; - model: string; - contextWindowTokens: number | null; - maxOutputTokens: number | null; - modelSettings: JsonObject; - providerOptions: JsonObject; + modelUri: string; + provider: string; + model: string; + contextWindowTokens: number | null; + maxOutputTokens: number | null; + modelSettings: JsonObject; + providerOptions: JsonObject; }; export type ResearchExecutionProfile = { - schemaVersion: typeof RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION; - profileId: string; - requested: JsonObject; - effective: { - validationAuthority: ValidationAuthoritySelection; - models: Record; - capabilityManifestRevision: string; - capabilityIds: string[]; - skillRegistryRevision: string; - selectedSkills: { id: string; revision: string }[]; - targetMode: ResearchExecutionTargetMode; - networkPolicy: string; - budgets: { - maxToolCalls: number | null; - maxRuntimeMs: number | null; - maxTurns: number | null; - maxCostUsd: number | null; - }; - uiCollection: SecurityResearchComponentCollection; - policies: Record; - }; + schemaVersion: typeof RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION; + profileId: string; + requested: JsonObject; + effective: { + validationAuthority: ValidationAuthoritySelection; + models: Record; + capabilityManifestRevision: string; + capabilityIds: string[]; + skillRegistryRevision: string; + selectedSkills: { id: string; revision: string }[]; + targetMode: ResearchExecutionTargetMode; + networkPolicy: string; + budgets: { + maxToolCalls: number | null; + maxRuntimeMs: number | null; + maxTurns: number | null; + maxCostUsd: number | null; + }; + uiCollection: SecurityResearchComponentCollection; + policies: Record; + }; }; export type ResearchExecutionProfileRuntimeProjection = { - validationAuthorityMode: ValidationAuthorityMode; - validationAuthority: ValidationAuthorityProvenance; - modelUri: string; - modelOverrides: ModelOverrideMap; - contextWindowTokens?: number; - maxOutputTokens?: number; - enabledToolIds: string[]; - runtimeMaxToolCalls?: number; - autonomyMode?: { - maxRuntimeMs?: number; - maxTurns?: number; - maxCostUsd?: number; - }; - targetMode: ResearchExecutionTargetMode; - networkProfile: string; - renderJsonCollection: SecurityResearchComponentCollection; - capabilityManifestRevision: string; - skillRegistryRevision: string; - researchExecutionProfileId: string; + validationAuthorityMode: ValidationAuthorityMode; + validationAuthority: ValidationAuthorityProvenance; + modelUri: string; + modelOverrides: ModelOverrideMap; + contextWindowTokens?: number; + maxOutputTokens?: number; + enabledToolIds: string[]; + runtimeMaxToolCalls?: number; + autonomyMode?: { + maxRuntimeMs?: number; + maxTurns?: number; + maxCostUsd?: number; + }; + targetMode: ResearchExecutionTargetMode; + networkProfile: string; + renderJsonCollection: SecurityResearchComponentCollection; + capabilityManifestRevision: string; + skillRegistryRevision: string; + researchExecutionProfileId: string; }; export type ResearchExecutionBackgroundProfile = { - profileId: string; - models: Record; - capabilityManifestRevision: string; - capabilityIds: string[]; - skillRegistryRevision: string; - selectedSkills: { id: string; revision: string }[]; - targetMode: ResearchExecutionTargetMode; - networkPolicy: string; + profileId: string; + validationAuthority: ValidationAuthoritySelection; + models: Record; + capabilityManifestRevision: string; + capabilityIds: string[]; + skillRegistryRevision: string; + selectedSkills: { id: string; revision: string }[]; + targetMode: ResearchExecutionTargetMode; + networkPolicy: string; }; export function resolveResearchExecutionProfile( - input: RequestedResearchExecutionProfile, - current: { - capabilityManifestRevision: string; - capabilityIds: readonly string[]; - skillRegistryRevision: string; - selectedSkills: readonly { id: string; revision: string }[]; - }, + input: RequestedResearchExecutionProfile, + current: { + capabilityManifestRevision: string; + capabilityIds: readonly string[]; + skillRegistryRevision: string; + selectedSkills: readonly { id: string; revision: string }[]; + }, ): ResearchExecutionProfile { - requireCurrentRevision( - "capability manifest", - input.capabilityManifestRevision, - current.capabilityManifestRevision, - ); - requireCurrentRevision( - "skill registry", - input.skillRegistryRevision, - current.skillRegistryRevision, - ); - const selectedSkills = canonicalSelectedSkills(input.selectedSkills ?? []); - const currentCapabilities = new Set(uniqueStrings(current.capabilityIds)); - for (const capabilityId of uniqueStrings(input.capabilityIds)) { - if (!currentCapabilities.has(capabilityId)) { - throw new Error( - `Research Execution Profile capability ${capabilityId} is unavailable in manifest ${current.capabilityManifestRevision}.`, - ); - } - } - const currentSkills = new Map( - canonicalSelectedSkills(current.selectedSkills).map((skill) => [ - skill.id, - skill.revision, - ]), - ); - const validationAuthority = resolveValidationAuthoritySelection( - input.validationAuthorityMode, - ); - for (const skill of selectedSkills) { - if (!skill.revision.trim()) { - throw new Error( - `Research Execution Profile skill revision is missing for ${skill.id}.`, - ); - } - const currentRevision = currentSkills.get(skill.id); - if (currentRevision !== skill.revision) { - throw new Error( - `Research Execution Profile skill ${skill.id} revision ${skill.revision} is unavailable; current revision is ${currentRevision ?? "missing"}.`, - ); - } - } - - const requestedModelUri = cleanString(input.modelUri); - const runtimeModelUri = cleanString(input.runtimeModelUri); - const requestedOverrides = canonicalModelOverrides(input.modelOverrides); - const coordinatorRequest = - runtimeModelUri ?? - readModelOverride(requestedOverrides, "coordinator") ?? - requestedModelUri; - const coordinatorUri = - resolveSecurityResearchMastraModelUri(coordinatorRequest); - const models = Object.fromEntries( - MODEL_OVERRIDE_TARGETS.map((target) => { - const targetRequest = runtimeModelUri - ? runtimeModelUri - : target === "coordinator" - ? coordinatorRequest - : (readModelOverride(requestedOverrides, target) ?? coordinatorUri); - return [ - target, - resolveExecutionModel(targetRequest, { - contextWindowTokens: - target === "coordinator" ? input.contextWindowTokens : undefined, - maxOutputTokens: - target === "coordinator" ? input.maxOutputTokens : undefined, - }), - ]; - }), - ) as Record; - - const requested = toJsonObject({ - validationAuthorityMode: validationAuthority.requestedMode, - modelUri: requestedModelUri ?? null, - runtimeModelUri: runtimeModelUri ?? null, - modelOverrides: requestedOverrides, - contextWindowTokens: positiveIntegerOrNull(input.contextWindowTokens), - maxOutputTokens: positiveIntegerOrNull(input.maxOutputTokens), - capabilityManifestRevision: input.capabilityManifestRevision, - capabilityIds: uniqueStrings(input.capabilityIds), - skillRegistryRevision: input.skillRegistryRevision, - selectedSkills, - targetMode: input.targetMode, - networkPolicy: input.networkPolicy, - maxToolCalls: positiveIntegerOrNull(input.maxToolCalls), - maxRuntimeMs: positiveIntegerOrNull(input.maxRuntimeMs), - maxTurns: positiveIntegerOrNull(input.maxTurns), - maxCostUsd: nonNegativeNumberOrNull(input.maxCostUsd), - uiCollection: input.uiCollection, - policies: canonicalStringRecord(input.policies), - }); - const body = { - schemaVersion: RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION, - requested, - effective: { - validationAuthority, - models, - capabilityManifestRevision: current.capabilityManifestRevision, - capabilityIds: uniqueStrings(input.capabilityIds), - skillRegistryRevision: current.skillRegistryRevision, - selectedSkills, - targetMode: input.targetMode, - networkPolicy: input.networkPolicy, - budgets: { - maxToolCalls: positiveIntegerOrNull(input.maxToolCalls), - maxRuntimeMs: positiveIntegerOrNull(input.maxRuntimeMs), - maxTurns: positiveIntegerOrNull(input.maxTurns), - maxCostUsd: nonNegativeNumberOrNull(input.maxCostUsd), - }, - uiCollection: input.uiCollection, - policies: canonicalStringRecord(input.policies), - }, - } as const; - return deepFreeze({ - ...body, - profileId: `research-profile-sha256:${sha256(stableStringify(body))}`, - }); + requireCurrentRevision( + "capability manifest", + input.capabilityManifestRevision, + current.capabilityManifestRevision, + ); + requireCurrentRevision( + "skill registry", + input.skillRegistryRevision, + current.skillRegistryRevision, + ); + const selectedSkills = canonicalSelectedSkills(input.selectedSkills ?? []); + const currentCapabilities = new Set(uniqueStrings(current.capabilityIds)); + for (const capabilityId of uniqueStrings(input.capabilityIds)) { + if (!currentCapabilities.has(capabilityId)) { + throw new Error( + `Research Execution Profile capability ${capabilityId} is unavailable in manifest ${current.capabilityManifestRevision}.`, + ); + } + } + const currentSkills = new Map( + canonicalSelectedSkills(current.selectedSkills).map((skill) => [skill.id, skill.revision]), + ); + const validationAuthority = resolveValidationAuthoritySelection(input.validationAuthorityMode); + for (const skill of selectedSkills) { + if (!skill.revision.trim()) { + throw new Error(`Research Execution Profile skill revision is missing for ${skill.id}.`); + } + const currentRevision = currentSkills.get(skill.id); + if (currentRevision !== skill.revision) { + throw new Error( + `Research Execution Profile skill ${skill.id} revision ${skill.revision} is unavailable; current revision is ${currentRevision ?? "missing"}.`, + ); + } + } + + const requestedModelUri = cleanString(input.modelUri); + const runtimeModelUri = cleanString(input.runtimeModelUri); + const requestedOverrides = canonicalModelOverrides(input.modelOverrides); + const coordinatorRequest = + runtimeModelUri ?? readModelOverride(requestedOverrides, "coordinator") ?? requestedModelUri; + const coordinatorUri = resolveSecurityResearchMastraModelUri(coordinatorRequest); + const models = Object.fromEntries( + MODEL_OVERRIDE_TARGETS.map((target) => { + const targetRequest = runtimeModelUri + ? runtimeModelUri + : target === "coordinator" + ? coordinatorRequest + : (readModelOverride(requestedOverrides, target) ?? coordinatorUri); + return [ + target, + resolveExecutionModel(targetRequest, { + contextWindowTokens: target === "coordinator" ? input.contextWindowTokens : undefined, + maxOutputTokens: target === "coordinator" ? input.maxOutputTokens : undefined, + }), + ]; + }), + ) as Record; + + const requested = toJsonObject({ + validationAuthorityMode: validationAuthority.requestedMode, + modelUri: requestedModelUri ?? null, + runtimeModelUri: runtimeModelUri ?? null, + modelOverrides: requestedOverrides, + contextWindowTokens: positiveIntegerOrNull(input.contextWindowTokens), + maxOutputTokens: positiveIntegerOrNull(input.maxOutputTokens), + capabilityManifestRevision: input.capabilityManifestRevision, + capabilityIds: uniqueStrings(input.capabilityIds), + skillRegistryRevision: input.skillRegistryRevision, + selectedSkills, + targetMode: input.targetMode, + networkPolicy: input.networkPolicy, + maxToolCalls: positiveIntegerOrNull(input.maxToolCalls), + maxRuntimeMs: positiveIntegerOrNull(input.maxRuntimeMs), + maxTurns: positiveIntegerOrNull(input.maxTurns), + maxCostUsd: nonNegativeNumberOrNull(input.maxCostUsd), + uiCollection: input.uiCollection, + policies: canonicalStringRecord(input.policies), + }); + const body = { + schemaVersion: RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION, + requested, + effective: { + validationAuthority, + models, + capabilityManifestRevision: current.capabilityManifestRevision, + capabilityIds: uniqueStrings(input.capabilityIds), + skillRegistryRevision: current.skillRegistryRevision, + selectedSkills, + targetMode: input.targetMode, + networkPolicy: input.networkPolicy, + budgets: { + maxToolCalls: positiveIntegerOrNull(input.maxToolCalls), + maxRuntimeMs: positiveIntegerOrNull(input.maxRuntimeMs), + maxTurns: positiveIntegerOrNull(input.maxTurns), + maxCostUsd: nonNegativeNumberOrNull(input.maxCostUsd), + }, + uiCollection: input.uiCollection, + policies: canonicalStringRecord(input.policies), + }, + } as const; + return deepFreeze({ + ...body, + profileId: `research-profile-sha256:${sha256(stableStringify(body))}`, + }); } /** Projection safe for stage/background inheritance; deliberately excludes approval grants. */ export function projectResearchExecutionProfileToBackground( - profile: ResearchExecutionProfile, + profile: ResearchExecutionProfile, ): ResearchExecutionBackgroundProfile { - assertResearchExecutionProfile(profile); - return { - profileId: profile.profileId, - models: Object.fromEntries( - MODEL_OVERRIDE_TARGETS.map((target) => [ - target, - profile.effective.models[target].modelUri, - ]), - ) as Record, - capabilityManifestRevision: profile.effective.capabilityManifestRevision, - capabilityIds: [...profile.effective.capabilityIds], - skillRegistryRevision: profile.effective.skillRegistryRevision, - selectedSkills: profile.effective.selectedSkills.map((skill) => ({ - ...skill, - })), - targetMode: profile.effective.targetMode, - networkPolicy: profile.effective.networkPolicy, - }; + assertResearchExecutionProfile(profile); + return { + profileId: profile.profileId, + validationAuthority: researchExecutionProfileValidationAuthority(profile), + models: Object.fromEntries( + MODEL_OVERRIDE_TARGETS.map((target) => [target, profile.effective.models[target].modelUri]), + ) as Record, + capabilityManifestRevision: profile.effective.capabilityManifestRevision, + capabilityIds: [...profile.effective.capabilityIds], + skillRegistryRevision: profile.effective.skillRegistryRevision, + selectedSkills: profile.effective.selectedSkills.map((skill) => ({ + ...skill, + })), + targetMode: profile.effective.targetMode, + networkPolicy: profile.effective.networkPolicy, + }; } export async function readLatestResearchExecutionBackgroundProfile(input: { - projectId: string; - threadId: string; + projectId: string; + threadId: string; }) { - return withDatabase(async (db) => { - const result = await db.query<{ metadata: JsonObject }>( - `SELECT metadata FROM research_runs + return withDatabase(async (db) => { + const result = await db.query<{ metadata: JsonObject }>( + `SELECT metadata FROM research_runs WHERE project_id = $1 AND thread_id = $2 AND status IN ('running', 'blocked') ORDER BY updated_at DESC, started_at DESC LIMIT 1`, - [input.projectId, input.threadId], - ); - const stored = result.rows[0]?.metadata?.[ - RESEARCH_EXECUTION_PROFILE_METADATA_KEY - ]; - return stored === undefined - ? undefined - : projectResearchExecutionProfileToBackground( - readResearchExecutionProfile(stored), - ); - }); + [input.projectId, input.threadId], + ); + const stored = result.rows[0]?.metadata?.[RESEARCH_EXECUTION_PROFILE_METADATA_KEY]; + return stored === undefined + ? undefined + : projectResearchExecutionProfileToBackground(readResearchExecutionProfile(stored)); + }); } export function projectResearchExecutionProfileToRuntime( - profile: ResearchExecutionProfile, + profile: ResearchExecutionProfile, ): ResearchExecutionProfileRuntimeProjection { - assertResearchExecutionProfile(profile); - const coordinator = profile.effective.models.coordinator; - const modelOverrides = Object.fromEntries( - MODEL_OVERRIDE_TARGETS.filter((target) => target !== "coordinator").map( - (target) => [target, profile.effective.models[target].modelUri], - ), - ) as ModelOverrideMap; - const budgets = profile.effective.budgets; - const validationAuthority = researchExecutionProfileValidationAuthority(profile); - return { - validationAuthorityMode: validationAuthority.effectiveMode, - validationAuthority: projectValidationAuthorityProvenance( - validationAuthority, - "research-execution-profile", - ), - modelUri: coordinator.modelUri, - modelOverrides, - ...(coordinator.contextWindowTokens - ? { contextWindowTokens: coordinator.contextWindowTokens } - : {}), - ...(coordinator.maxOutputTokens - ? { maxOutputTokens: coordinator.maxOutputTokens } - : {}), - enabledToolIds: [...profile.effective.capabilityIds], - ...(budgets.maxToolCalls - ? { runtimeMaxToolCalls: budgets.maxToolCalls } - : {}), - ...(budgets.maxRuntimeMs || budgets.maxTurns || budgets.maxCostUsd !== null - ? { - autonomyMode: { - ...(budgets.maxRuntimeMs - ? { maxRuntimeMs: budgets.maxRuntimeMs } - : {}), - ...(budgets.maxTurns ? { maxTurns: budgets.maxTurns } : {}), - ...(budgets.maxCostUsd !== null - ? { maxCostUsd: budgets.maxCostUsd } - : {}), - }, - } - : {}), - targetMode: profile.effective.targetMode, - networkProfile: profile.effective.networkPolicy, - renderJsonCollection: profile.effective.uiCollection, - capabilityManifestRevision: profile.effective.capabilityManifestRevision, - skillRegistryRevision: profile.effective.skillRegistryRevision, - researchExecutionProfileId: profile.profileId, - }; + assertResearchExecutionProfile(profile); + const coordinator = profile.effective.models.coordinator; + const modelOverrides = Object.fromEntries( + MODEL_OVERRIDE_TARGETS.filter((target) => target !== "coordinator").map((target) => [ + target, + profile.effective.models[target].modelUri, + ]), + ) as ModelOverrideMap; + const budgets = profile.effective.budgets; + const validationAuthority = researchExecutionProfileValidationAuthority(profile); + return { + validationAuthorityMode: validationAuthority.effectiveMode, + validationAuthority: projectValidationAuthorityProvenance( + validationAuthority, + "research-execution-profile", + ), + modelUri: coordinator.modelUri, + modelOverrides, + ...(coordinator.contextWindowTokens + ? { contextWindowTokens: coordinator.contextWindowTokens } + : {}), + ...(coordinator.maxOutputTokens ? { maxOutputTokens: coordinator.maxOutputTokens } : {}), + enabledToolIds: [...profile.effective.capabilityIds], + ...(budgets.maxToolCalls ? { runtimeMaxToolCalls: budgets.maxToolCalls } : {}), + ...(budgets.maxRuntimeMs || budgets.maxTurns || budgets.maxCostUsd !== null + ? { + autonomyMode: { + ...(budgets.maxRuntimeMs ? { maxRuntimeMs: budgets.maxRuntimeMs } : {}), + ...(budgets.maxTurns ? { maxTurns: budgets.maxTurns } : {}), + ...(budgets.maxCostUsd !== null ? { maxCostUsd: budgets.maxCostUsd } : {}), + }, + } + : {}), + targetMode: profile.effective.targetMode, + networkProfile: profile.effective.networkPolicy, + renderJsonCollection: profile.effective.uiCollection, + capabilityManifestRevision: profile.effective.capabilityManifestRevision, + skillRegistryRevision: profile.effective.skillRegistryRevision, + researchExecutionProfileId: profile.profileId, + }; } export function researchExecutionProfileValidationAuthority( - profile: ResearchExecutionProfile, + profile: ResearchExecutionProfile, ): ValidationAuthoritySelection { - assertResearchExecutionProfile(profile); - return profile.effective.validationAuthority - ? readValidationAuthoritySelection(profile.effective.validationAuthority) - : resolveValidationAuthoritySelection( - profile.requested.validationAuthorityMode, - ); + assertResearchExecutionProfile(profile); + return profile.effective.validationAuthority + ? readValidationAuthoritySelection(profile.effective.validationAuthority) + : resolveValidationAuthoritySelection(profile.requested.validationAuthorityMode); } -export function researchExecutionProfileForensics( - profile: ResearchExecutionProfile, -): JsonObject { - assertResearchExecutionProfile(profile); - return toJsonObject(profile); +export function researchExecutionProfileForensics(profile: ResearchExecutionProfile): JsonObject { + assertResearchExecutionProfile(profile); + return toJsonObject(profile); } -export function readResearchExecutionProfile( - value: unknown, -): ResearchExecutionProfile { - assertResearchExecutionProfile(value); - return deepFreeze(value as ResearchExecutionProfile); +export function readResearchExecutionProfile(value: unknown): ResearchExecutionProfile { + assertResearchExecutionProfile(value); + return deepFreeze(value as ResearchExecutionProfile); } export function assertResearchExecutionProfile( - value: unknown, + value: unknown, ): asserts value is ResearchExecutionProfile { - if (!value || typeof value !== "object" || Array.isArray(value)) { - throw new Error("Research Execution Profile is missing or malformed."); - } - const candidate = value as Partial; - if ( - candidate.schemaVersion !== RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION || - typeof candidate.profileId !== "string" || - !candidate.requested || - !candidate.effective - ) { - throw new Error( - "Research Execution Profile revision is missing or unsupported.", - ); - } - const body = { - schemaVersion: candidate.schemaVersion, - requested: candidate.requested, - effective: candidate.effective, - }; - const expectedId = `research-profile-sha256:${sha256(stableStringify(body))}`; - if (candidate.profileId !== expectedId) { - throw new Error( - "Research Execution Profile content does not match its immutable profile id.", - ); - } - const models = candidate.effective.models; - if ( - !models || - MODEL_OVERRIDE_TARGETS.some((target) => !models[target]?.modelUri) - ) { - throw new Error( - "Research Execution Profile is missing an effective model revision.", - ); - } - requireNonEmpty( - "capability manifest", - candidate.effective.capabilityManifestRevision, - ); - requireNonEmpty("skill registry", candidate.effective.skillRegistryRevision); - if (candidate.effective.validationAuthority !== undefined) { - readValidationAuthoritySelection(candidate.effective.validationAuthority); - } + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Research Execution Profile is missing or malformed."); + } + const candidate = value as Partial; + if ( + candidate.schemaVersion !== RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION || + typeof candidate.profileId !== "string" || + !candidate.requested || + !candidate.effective + ) { + throw new Error("Research Execution Profile revision is missing or unsupported."); + } + const body = { + schemaVersion: candidate.schemaVersion, + requested: candidate.requested, + effective: candidate.effective, + }; + const expectedId = `research-profile-sha256:${sha256(stableStringify(body))}`; + if (candidate.profileId !== expectedId) { + throw new Error("Research Execution Profile content does not match its immutable profile id."); + } + const models = candidate.effective.models; + if (!models || MODEL_OVERRIDE_TARGETS.some((target) => !models[target]?.modelUri)) { + throw new Error("Research Execution Profile is missing an effective model revision."); + } + requireNonEmpty("capability manifest", candidate.effective.capabilityManifestRevision); + requireNonEmpty("skill registry", candidate.effective.skillRegistryRevision); + if (candidate.effective.validationAuthority !== undefined) { + readValidationAuthoritySelection(candidate.effective.validationAuthority); + } } function resolveExecutionModel( - requestedModelUri: string | undefined, - overrides: { contextWindowTokens?: number; maxOutputTokens?: number }, + requestedModelUri: string | undefined, + overrides: { contextWindowTokens?: number; maxOutputTokens?: number }, ): ResearchExecutionModel { - const modelUri = resolveSecurityResearchMastraModelUri(requestedModelUri); - const parsed = parse(asModelConnectionString(modelUri)); - const normalized = normalize(parsed); - const registryEntry = findModelEntryForRef(modelUri); - const runtimeOptions = getMastraModelRuntimeOptions(modelUri); - const provider = - normalized.provider ?? - modelProviderFromHost(parsed.hostAlias) ?? - modelProviderFromHost(parsed.host) ?? - parsed.hostAlias ?? - parsed.host; - const normalizedMaxOutput = positiveIntegerOrNull( - normalized.config.params.max_tokens, - ); - return { - modelUri, - provider, - model: parsed.model, - contextWindowTokens: - positiveIntegerOrNull(overrides.contextWindowTokens) ?? - registryEntry?.contextWindow ?? - null, - maxOutputTokens: - positiveIntegerOrNull(overrides.maxOutputTokens) ?? - normalizedMaxOutput ?? - registryEntry?.maxOutputTokens ?? - null, - modelSettings: toJsonObject(runtimeOptions.modelSettings), - providerOptions: toJsonObject(runtimeOptions.providerOptions), - }; + const modelUri = resolveSecurityResearchMastraModelUri(requestedModelUri); + const parsed = parse(asModelConnectionString(modelUri)); + const normalized = normalize(parsed); + const registryEntry = findModelEntryForRef(modelUri); + const runtimeOptions = getMastraModelRuntimeOptions(modelUri); + const provider = + normalized.provider ?? + modelProviderFromHost(parsed.hostAlias) ?? + modelProviderFromHost(parsed.host) ?? + parsed.hostAlias ?? + parsed.host; + const normalizedMaxOutput = positiveIntegerOrNull(normalized.config.params.max_tokens); + return { + modelUri, + provider, + model: parsed.model, + contextWindowTokens: + positiveIntegerOrNull(overrides.contextWindowTokens) ?? registryEntry?.contextWindow ?? null, + maxOutputTokens: + positiveIntegerOrNull(overrides.maxOutputTokens) ?? + normalizedMaxOutput ?? + registryEntry?.maxOutputTokens ?? + null, + modelSettings: toJsonObject(runtimeOptions.modelSettings), + providerOptions: toJsonObject(runtimeOptions.providerOptions), + }; } -function canonicalModelOverrides( - value: ModelOverrideMap | undefined, -): ModelOverrideMap { - return Object.fromEntries( - MODEL_OVERRIDE_TARGETS.flatMap((target) => { - const modelUri = readModelOverride(value, target); - return modelUri ? [[target, modelUri]] : []; - }), - ); +function canonicalModelOverrides(value: ModelOverrideMap | undefined): ModelOverrideMap { + return Object.fromEntries( + MODEL_OVERRIDE_TARGETS.flatMap((target) => { + const modelUri = readModelOverride(value, target); + return modelUri ? [[target, modelUri]] : []; + }), + ); } -function canonicalSelectedSkills( - value: readonly { id: string; revision: string }[], -) { - return [...value] - .map((skill) => ({ id: skill.id.trim(), revision: skill.revision.trim() })) - .filter((skill) => skill.id) - .sort((left, right) => left.id.localeCompare(right.id)); +function canonicalSelectedSkills(value: readonly { id: string; revision: string }[]) { + return [...value] + .map((skill) => ({ id: skill.id.trim(), revision: skill.revision.trim() })) + .filter((skill) => skill.id) + .sort((left, right) => left.id.localeCompare(right.id)); } function canonicalStringRecord(value: Readonly>) { - return Object.fromEntries( - Object.entries(value) - .map(([key, item]) => [key.trim(), item.trim()] as const) - .filter(([key, item]) => key && item) - .sort(([left], [right]) => left.localeCompare(right)), - ); + return Object.fromEntries( + Object.entries(value) + .map(([key, item]) => [key.trim(), item.trim()] as const) + .filter(([key, item]) => key && item) + .sort(([left], [right]) => left.localeCompare(right)), + ); } function uniqueStrings(values: readonly string[]) { - return [ - ...new Set(values.map((value) => value.trim()).filter(Boolean)), - ].sort(); + return [...new Set(values.map((value) => value.trim()).filter(Boolean))].sort(); } -function requireCurrentRevision( - label: string, - requested: string, - current: string, -) { - requireNonEmpty(label, requested); - requireNonEmpty(label, current); - if (requested !== current) { - throw new Error( - `Research Execution Profile ${label} revision ${requested} is unavailable; current revision is ${current}.`, - ); - } +function requireCurrentRevision(label: string, requested: string, current: string) { + requireNonEmpty(label, requested); + requireNonEmpty(label, current); + if (requested !== current) { + throw new Error( + `Research Execution Profile ${label} revision ${requested} is unavailable; current revision is ${current}.`, + ); + } } -function requireNonEmpty( - label: string, - value: unknown, -): asserts value is string { - if (typeof value !== "string" || !value.trim()) { - throw new Error(`Research Execution Profile ${label} revision is missing.`); - } +function requireNonEmpty(label: string, value: unknown): asserts value is string { + if (typeof value !== "string" || !value.trim()) { + throw new Error(`Research Execution Profile ${label} revision is missing.`); + } } function cleanString(value: unknown) { - return typeof value === "string" && value.trim() ? value.trim() : undefined; + return typeof value === "string" && value.trim() ? value.trim() : undefined; } function positiveIntegerOrNull(value: unknown) { - return typeof value === "number" && Number.isSafeInteger(value) && value > 0 - ? value - : null; + return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : null; } function nonNegativeNumberOrNull(value: unknown) { - return typeof value === "number" && Number.isFinite(value) && value >= 0 - ? value - : null; + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null; } function toJsonObject(value: unknown): JsonObject { - return JSON.parse( - JSON.stringify(value, (_key, item) => (item === undefined ? null : item)), - ) as JsonObject; + return JSON.parse( + JSON.stringify(value, (_key, item) => (item === undefined ? null : item)), + ) as JsonObject; } function stableStringify(value: unknown): string { - if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`; - if (value && typeof value === "object") { - return `{${Object.entries(value as Record) - .sort(([left], [right]) => left.localeCompare(right)) - .map(([key, item]) => `${JSON.stringify(key)}:${stableStringify(item)}`) - .join(",")}}`; - } - return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, item]) => `${JSON.stringify(key)}:${stableStringify(item)}`) + .join(",")}}`; + } + return JSON.stringify(value); } function sha256(value: string) { - return createHash("sha256").update(value).digest("hex"); + return createHash("sha256").update(value).digest("hex"); } function deepFreeze(value: T): T { - if (value && typeof value === "object" && !Object.isFrozen(value)) { - Object.freeze(value); - for (const nested of Object.values(value as Record)) - deepFreeze(nested); - } - return value; + if (value && typeof value === "object" && !Object.isFrozen(value)) { + Object.freeze(value); + for (const nested of Object.values(value as Record)) deepFreeze(nested); + } + return value; } diff --git a/tests/integration/scheduler-backed-research.test.ts b/tests/integration/scheduler-backed-research.test.ts index 6f47c4b32..e2fe4c24c 100644 --- a/tests/integration/scheduler-backed-research.test.ts +++ b/tests/integration/scheduler-backed-research.test.ts @@ -66,31 +66,32 @@ function makeRunner( return { artifactId: `artifact-${input.schedulerTaskId}`, handoff: { - schemaVersion: 1, - projectId: input.projectId, - schedulerTaskId: input.schedulerTaskId, - stage: input.stage, - taskId: input.taskId, - attempt: input.attempt, - outcome: "completed", - summary: typeof input.draft.summary === "string" ? input.draft.summary : input.draft.output, - coverage: [], - negativeCoverage: [], - assertionIds: [], - hypothesisIds: [], - artifactIds: [], - findingIds: [], - completedBranches: [], - openBranches: [], - blockers: [], - proposedTaskIds: [], - approvalNeeds: [], - executionProfile: input.executionProfile, - attribution: { producerAgentId: input.producerAgentId, parentHandoffIds: [] }, - budgetUsage: {}, - resourceClaims: [], - output: input.draft.output, - createdAt: new Date(0).toISOString(), + schemaVersion: 1, + projectId: input.projectId, + schedulerTaskId: input.schedulerTaskId, + stage: input.stage, + taskId: input.taskId, + attempt: input.attempt, + outcome: "completed", + summary: + typeof input.draft.summary === "string" ? input.draft.summary : input.draft.output, + coverage: [], + negativeCoverage: [], + assertionIds: [], + hypothesisIds: [], + artifactIds: [], + findingIds: [], + completedBranches: [], + openBranches: [], + blockers: [], + proposedTaskIds: [], + approvalNeeds: [], + executionProfile: input.executionProfile, + attribution: { producerAgentId: input.producerAgentId, parentHandoffIds: [] }, + budgetUsage: {}, + resourceClaims: [], + output: input.draft.output, + createdAt: new Date(0).toISOString(), }, }; }, @@ -249,6 +250,11 @@ describe("scheduler-backed research runner", () => { ); const profile = { profileId: "research-profile-sha256:pinned", + validationAuthority: { + schemaVersion: "validation-authority-v1" as const, + requestedMode: "strict" as const, + effectiveMode: "strict" as const, + }, models: { coordinator: "llm://lmstudio/coordinator/pinned", rag: "llm://lmstudio/coordinator/pinned", @@ -270,9 +276,7 @@ describe("scheduler-backed research runner", () => { networkPolicy: "none", }; - await runner.enqueueResearchTasks( - enqueueInput("project-1", [makeTask({ id: "t-profile" })]), - ); + await runner.enqueueResearchTasks(enqueueInput("project-1", [makeTask({ id: "t-profile" })])); await runner.runEnqueuedTasks({ projectId: "project-1", owner: "worker-profile", diff --git a/tests/integration/validation-authority-provenance.test.ts b/tests/integration/validation-authority-provenance.test.ts index d421af54b..c43982ea3 100644 --- a/tests/integration/validation-authority-provenance.test.ts +++ b/tests/integration/validation-authority-provenance.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { SECURITY_CAPABILITY_MANIFEST_REVISION } from "../../src/lib/tools/catalog"; import { + projectResearchExecutionProfileToBackground, projectResearchExecutionProfileToRuntime, resolveResearchExecutionProfile, } from "../../src/server/research/execution-profile"; @@ -64,6 +65,12 @@ describe("validation authority provenance", () => { recordKind: "research-execution-profile", }, }); + expect(projectResearchExecutionProfileToBackground(profile)).toMatchObject({ + validationAuthority: { + requestedMode: "self", + effectiveMode: "self", + }, + }); const selection = resolveValidationAuthoritySelection("self"); const projections = VALIDATION_AUTHORITY_RECORD_KINDS.map((recordKind) => From e0768407c7ba65e78794a864a73db7371d41470a Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Thu, 27 Aug 2026 11:47:19 -0400 Subject: [PATCH 4/5] Revert "Carry validation authority into background work" This reverts commit 564b052fd36f7c1b6714c789db6e4c30449f5f76. --- .../chat/security-research-runtime-context.ts | 1 - src/server/research/execution-profile.ts | 849 ++++++++++-------- .../scheduler-backed-research.test.ts | 60 +- .../validation-authority-provenance.test.ts | 7 - 4 files changed, 485 insertions(+), 432 deletions(-) diff --git a/src/server/chat/security-research-runtime-context.ts b/src/server/chat/security-research-runtime-context.ts index 6c4da4d32..0152c6405 100644 --- a/src/server/chat/security-research-runtime-context.ts +++ b/src/server/chat/security-research-runtime-context.ts @@ -272,7 +272,6 @@ export function buildSecurityResearchStageRequestContext(input: { [keys.skillRegistryRevision, input.executionProfile?.skillRegistryRevision], [keys.selectedSkillRefs, input.executionProfile?.selectedSkills], [keys.runtimeSkillCapabilities, input.executionProfile?.capabilityIds], - [keys.validationAuthorityMode, input.executionProfile?.validationAuthority.effectiveMode], [keys.modelUri, input.executionProfile?.models.coordinator], [keys.coordinatorModelUri, input.executionProfile?.models.coordinator], [keys.modelOverrides, input.executionProfile?.models], diff --git a/src/server/research/execution-profile.ts b/src/server/research/execution-profile.ts index e3ec6605a..e02232318 100644 --- a/src/server/research/execution-profile.ts +++ b/src/server/research/execution-profile.ts @@ -3,493 +3,558 @@ import { createHash } from "node:crypto"; import { normalize, parse } from "llm-strings"; import { - asModelConnectionString, - findModelEntryForRef, - MODEL_OVERRIDE_TARGETS, - type ModelOverrideMap, - type ModelOverrideTarget, - modelProviderFromHost, - readModelOverride, + asModelConnectionString, + findModelEntryForRef, + MODEL_OVERRIDE_TARGETS, + type ModelOverrideMap, + type ModelOverrideTarget, + modelProviderFromHost, + readModelOverride, } from "../../lib/models"; import type { SecurityResearchComponentCollection } from "../../lib/render-json/security-research-catalog"; import { - getMastraModelRuntimeOptions, - resolveSecurityResearchMastraModelUri, + getMastraModelRuntimeOptions, + resolveSecurityResearchMastraModelUri, } from "../../mastra/config/model"; import { withDatabase } from "../db/client"; import type { JsonObject } from "../db/types"; import { - projectValidationAuthorityProvenance, - readValidationAuthoritySelection, - resolveValidationAuthoritySelection, - type ValidationAuthorityMode, - type ValidationAuthorityProvenance, - type ValidationAuthoritySelection, + projectValidationAuthorityProvenance, + readValidationAuthoritySelection, + resolveValidationAuthoritySelection, + type ValidationAuthorityMode, + type ValidationAuthorityProvenance, + type ValidationAuthoritySelection, } from "../validation-plans/authority"; -export const RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION = "research-execution-profile-v1"; +export const RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION = + "research-execution-profile-v1"; export const RESEARCH_EXECUTION_PROFILE_METADATA_KEY = "executionProfile"; export type ResearchExecutionTargetMode = "none" | "container" | "remote"; export type RequestedResearchExecutionProfile = { - validationAuthorityMode?: unknown; - modelUri?: string; - runtimeModelUri?: string; - modelOverrides?: ModelOverrideMap; - contextWindowTokens?: number; - maxOutputTokens?: number; - capabilityManifestRevision: string; - capabilityIds: readonly string[]; - skillRegistryRevision: string; - selectedSkills?: readonly { id: string; revision: string }[]; - targetMode: ResearchExecutionTargetMode; - networkPolicy: string; - maxToolCalls?: number; - maxRuntimeMs?: number; - maxTurns?: number; - maxCostUsd?: number; - uiCollection: SecurityResearchComponentCollection; - policies: Readonly>; + validationAuthorityMode?: unknown; + modelUri?: string; + runtimeModelUri?: string; + modelOverrides?: ModelOverrideMap; + contextWindowTokens?: number; + maxOutputTokens?: number; + capabilityManifestRevision: string; + capabilityIds: readonly string[]; + skillRegistryRevision: string; + selectedSkills?: readonly { id: string; revision: string }[]; + targetMode: ResearchExecutionTargetMode; + networkPolicy: string; + maxToolCalls?: number; + maxRuntimeMs?: number; + maxTurns?: number; + maxCostUsd?: number; + uiCollection: SecurityResearchComponentCollection; + policies: Readonly>; }; export type ResearchExecutionModel = { - modelUri: string; - provider: string; - model: string; - contextWindowTokens: number | null; - maxOutputTokens: number | null; - modelSettings: JsonObject; - providerOptions: JsonObject; + modelUri: string; + provider: string; + model: string; + contextWindowTokens: number | null; + maxOutputTokens: number | null; + modelSettings: JsonObject; + providerOptions: JsonObject; }; export type ResearchExecutionProfile = { - schemaVersion: typeof RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION; - profileId: string; - requested: JsonObject; - effective: { - validationAuthority: ValidationAuthoritySelection; - models: Record; - capabilityManifestRevision: string; - capabilityIds: string[]; - skillRegistryRevision: string; - selectedSkills: { id: string; revision: string }[]; - targetMode: ResearchExecutionTargetMode; - networkPolicy: string; - budgets: { - maxToolCalls: number | null; - maxRuntimeMs: number | null; - maxTurns: number | null; - maxCostUsd: number | null; - }; - uiCollection: SecurityResearchComponentCollection; - policies: Record; - }; + schemaVersion: typeof RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION; + profileId: string; + requested: JsonObject; + effective: { + validationAuthority: ValidationAuthoritySelection; + models: Record; + capabilityManifestRevision: string; + capabilityIds: string[]; + skillRegistryRevision: string; + selectedSkills: { id: string; revision: string }[]; + targetMode: ResearchExecutionTargetMode; + networkPolicy: string; + budgets: { + maxToolCalls: number | null; + maxRuntimeMs: number | null; + maxTurns: number | null; + maxCostUsd: number | null; + }; + uiCollection: SecurityResearchComponentCollection; + policies: Record; + }; }; export type ResearchExecutionProfileRuntimeProjection = { - validationAuthorityMode: ValidationAuthorityMode; - validationAuthority: ValidationAuthorityProvenance; - modelUri: string; - modelOverrides: ModelOverrideMap; - contextWindowTokens?: number; - maxOutputTokens?: number; - enabledToolIds: string[]; - runtimeMaxToolCalls?: number; - autonomyMode?: { - maxRuntimeMs?: number; - maxTurns?: number; - maxCostUsd?: number; - }; - targetMode: ResearchExecutionTargetMode; - networkProfile: string; - renderJsonCollection: SecurityResearchComponentCollection; - capabilityManifestRevision: string; - skillRegistryRevision: string; - researchExecutionProfileId: string; + validationAuthorityMode: ValidationAuthorityMode; + validationAuthority: ValidationAuthorityProvenance; + modelUri: string; + modelOverrides: ModelOverrideMap; + contextWindowTokens?: number; + maxOutputTokens?: number; + enabledToolIds: string[]; + runtimeMaxToolCalls?: number; + autonomyMode?: { + maxRuntimeMs?: number; + maxTurns?: number; + maxCostUsd?: number; + }; + targetMode: ResearchExecutionTargetMode; + networkProfile: string; + renderJsonCollection: SecurityResearchComponentCollection; + capabilityManifestRevision: string; + skillRegistryRevision: string; + researchExecutionProfileId: string; }; export type ResearchExecutionBackgroundProfile = { - profileId: string; - validationAuthority: ValidationAuthoritySelection; - models: Record; - capabilityManifestRevision: string; - capabilityIds: string[]; - skillRegistryRevision: string; - selectedSkills: { id: string; revision: string }[]; - targetMode: ResearchExecutionTargetMode; - networkPolicy: string; + profileId: string; + models: Record; + capabilityManifestRevision: string; + capabilityIds: string[]; + skillRegistryRevision: string; + selectedSkills: { id: string; revision: string }[]; + targetMode: ResearchExecutionTargetMode; + networkPolicy: string; }; export function resolveResearchExecutionProfile( - input: RequestedResearchExecutionProfile, - current: { - capabilityManifestRevision: string; - capabilityIds: readonly string[]; - skillRegistryRevision: string; - selectedSkills: readonly { id: string; revision: string }[]; - }, + input: RequestedResearchExecutionProfile, + current: { + capabilityManifestRevision: string; + capabilityIds: readonly string[]; + skillRegistryRevision: string; + selectedSkills: readonly { id: string; revision: string }[]; + }, ): ResearchExecutionProfile { - requireCurrentRevision( - "capability manifest", - input.capabilityManifestRevision, - current.capabilityManifestRevision, - ); - requireCurrentRevision( - "skill registry", - input.skillRegistryRevision, - current.skillRegistryRevision, - ); - const selectedSkills = canonicalSelectedSkills(input.selectedSkills ?? []); - const currentCapabilities = new Set(uniqueStrings(current.capabilityIds)); - for (const capabilityId of uniqueStrings(input.capabilityIds)) { - if (!currentCapabilities.has(capabilityId)) { - throw new Error( - `Research Execution Profile capability ${capabilityId} is unavailable in manifest ${current.capabilityManifestRevision}.`, - ); - } - } - const currentSkills = new Map( - canonicalSelectedSkills(current.selectedSkills).map((skill) => [skill.id, skill.revision]), - ); - const validationAuthority = resolveValidationAuthoritySelection(input.validationAuthorityMode); - for (const skill of selectedSkills) { - if (!skill.revision.trim()) { - throw new Error(`Research Execution Profile skill revision is missing for ${skill.id}.`); - } - const currentRevision = currentSkills.get(skill.id); - if (currentRevision !== skill.revision) { - throw new Error( - `Research Execution Profile skill ${skill.id} revision ${skill.revision} is unavailable; current revision is ${currentRevision ?? "missing"}.`, - ); - } - } - - const requestedModelUri = cleanString(input.modelUri); - const runtimeModelUri = cleanString(input.runtimeModelUri); - const requestedOverrides = canonicalModelOverrides(input.modelOverrides); - const coordinatorRequest = - runtimeModelUri ?? readModelOverride(requestedOverrides, "coordinator") ?? requestedModelUri; - const coordinatorUri = resolveSecurityResearchMastraModelUri(coordinatorRequest); - const models = Object.fromEntries( - MODEL_OVERRIDE_TARGETS.map((target) => { - const targetRequest = runtimeModelUri - ? runtimeModelUri - : target === "coordinator" - ? coordinatorRequest - : (readModelOverride(requestedOverrides, target) ?? coordinatorUri); - return [ - target, - resolveExecutionModel(targetRequest, { - contextWindowTokens: target === "coordinator" ? input.contextWindowTokens : undefined, - maxOutputTokens: target === "coordinator" ? input.maxOutputTokens : undefined, - }), - ]; - }), - ) as Record; - - const requested = toJsonObject({ - validationAuthorityMode: validationAuthority.requestedMode, - modelUri: requestedModelUri ?? null, - runtimeModelUri: runtimeModelUri ?? null, - modelOverrides: requestedOverrides, - contextWindowTokens: positiveIntegerOrNull(input.contextWindowTokens), - maxOutputTokens: positiveIntegerOrNull(input.maxOutputTokens), - capabilityManifestRevision: input.capabilityManifestRevision, - capabilityIds: uniqueStrings(input.capabilityIds), - skillRegistryRevision: input.skillRegistryRevision, - selectedSkills, - targetMode: input.targetMode, - networkPolicy: input.networkPolicy, - maxToolCalls: positiveIntegerOrNull(input.maxToolCalls), - maxRuntimeMs: positiveIntegerOrNull(input.maxRuntimeMs), - maxTurns: positiveIntegerOrNull(input.maxTurns), - maxCostUsd: nonNegativeNumberOrNull(input.maxCostUsd), - uiCollection: input.uiCollection, - policies: canonicalStringRecord(input.policies), - }); - const body = { - schemaVersion: RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION, - requested, - effective: { - validationAuthority, - models, - capabilityManifestRevision: current.capabilityManifestRevision, - capabilityIds: uniqueStrings(input.capabilityIds), - skillRegistryRevision: current.skillRegistryRevision, - selectedSkills, - targetMode: input.targetMode, - networkPolicy: input.networkPolicy, - budgets: { - maxToolCalls: positiveIntegerOrNull(input.maxToolCalls), - maxRuntimeMs: positiveIntegerOrNull(input.maxRuntimeMs), - maxTurns: positiveIntegerOrNull(input.maxTurns), - maxCostUsd: nonNegativeNumberOrNull(input.maxCostUsd), - }, - uiCollection: input.uiCollection, - policies: canonicalStringRecord(input.policies), - }, - } as const; - return deepFreeze({ - ...body, - profileId: `research-profile-sha256:${sha256(stableStringify(body))}`, - }); + requireCurrentRevision( + "capability manifest", + input.capabilityManifestRevision, + current.capabilityManifestRevision, + ); + requireCurrentRevision( + "skill registry", + input.skillRegistryRevision, + current.skillRegistryRevision, + ); + const selectedSkills = canonicalSelectedSkills(input.selectedSkills ?? []); + const currentCapabilities = new Set(uniqueStrings(current.capabilityIds)); + for (const capabilityId of uniqueStrings(input.capabilityIds)) { + if (!currentCapabilities.has(capabilityId)) { + throw new Error( + `Research Execution Profile capability ${capabilityId} is unavailable in manifest ${current.capabilityManifestRevision}.`, + ); + } + } + const currentSkills = new Map( + canonicalSelectedSkills(current.selectedSkills).map((skill) => [ + skill.id, + skill.revision, + ]), + ); + const validationAuthority = resolveValidationAuthoritySelection( + input.validationAuthorityMode, + ); + for (const skill of selectedSkills) { + if (!skill.revision.trim()) { + throw new Error( + `Research Execution Profile skill revision is missing for ${skill.id}.`, + ); + } + const currentRevision = currentSkills.get(skill.id); + if (currentRevision !== skill.revision) { + throw new Error( + `Research Execution Profile skill ${skill.id} revision ${skill.revision} is unavailable; current revision is ${currentRevision ?? "missing"}.`, + ); + } + } + + const requestedModelUri = cleanString(input.modelUri); + const runtimeModelUri = cleanString(input.runtimeModelUri); + const requestedOverrides = canonicalModelOverrides(input.modelOverrides); + const coordinatorRequest = + runtimeModelUri ?? + readModelOverride(requestedOverrides, "coordinator") ?? + requestedModelUri; + const coordinatorUri = + resolveSecurityResearchMastraModelUri(coordinatorRequest); + const models = Object.fromEntries( + MODEL_OVERRIDE_TARGETS.map((target) => { + const targetRequest = runtimeModelUri + ? runtimeModelUri + : target === "coordinator" + ? coordinatorRequest + : (readModelOverride(requestedOverrides, target) ?? coordinatorUri); + return [ + target, + resolveExecutionModel(targetRequest, { + contextWindowTokens: + target === "coordinator" ? input.contextWindowTokens : undefined, + maxOutputTokens: + target === "coordinator" ? input.maxOutputTokens : undefined, + }), + ]; + }), + ) as Record; + + const requested = toJsonObject({ + validationAuthorityMode: validationAuthority.requestedMode, + modelUri: requestedModelUri ?? null, + runtimeModelUri: runtimeModelUri ?? null, + modelOverrides: requestedOverrides, + contextWindowTokens: positiveIntegerOrNull(input.contextWindowTokens), + maxOutputTokens: positiveIntegerOrNull(input.maxOutputTokens), + capabilityManifestRevision: input.capabilityManifestRevision, + capabilityIds: uniqueStrings(input.capabilityIds), + skillRegistryRevision: input.skillRegistryRevision, + selectedSkills, + targetMode: input.targetMode, + networkPolicy: input.networkPolicy, + maxToolCalls: positiveIntegerOrNull(input.maxToolCalls), + maxRuntimeMs: positiveIntegerOrNull(input.maxRuntimeMs), + maxTurns: positiveIntegerOrNull(input.maxTurns), + maxCostUsd: nonNegativeNumberOrNull(input.maxCostUsd), + uiCollection: input.uiCollection, + policies: canonicalStringRecord(input.policies), + }); + const body = { + schemaVersion: RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION, + requested, + effective: { + validationAuthority, + models, + capabilityManifestRevision: current.capabilityManifestRevision, + capabilityIds: uniqueStrings(input.capabilityIds), + skillRegistryRevision: current.skillRegistryRevision, + selectedSkills, + targetMode: input.targetMode, + networkPolicy: input.networkPolicy, + budgets: { + maxToolCalls: positiveIntegerOrNull(input.maxToolCalls), + maxRuntimeMs: positiveIntegerOrNull(input.maxRuntimeMs), + maxTurns: positiveIntegerOrNull(input.maxTurns), + maxCostUsd: nonNegativeNumberOrNull(input.maxCostUsd), + }, + uiCollection: input.uiCollection, + policies: canonicalStringRecord(input.policies), + }, + } as const; + return deepFreeze({ + ...body, + profileId: `research-profile-sha256:${sha256(stableStringify(body))}`, + }); } /** Projection safe for stage/background inheritance; deliberately excludes approval grants. */ export function projectResearchExecutionProfileToBackground( - profile: ResearchExecutionProfile, + profile: ResearchExecutionProfile, ): ResearchExecutionBackgroundProfile { - assertResearchExecutionProfile(profile); - return { - profileId: profile.profileId, - validationAuthority: researchExecutionProfileValidationAuthority(profile), - models: Object.fromEntries( - MODEL_OVERRIDE_TARGETS.map((target) => [target, profile.effective.models[target].modelUri]), - ) as Record, - capabilityManifestRevision: profile.effective.capabilityManifestRevision, - capabilityIds: [...profile.effective.capabilityIds], - skillRegistryRevision: profile.effective.skillRegistryRevision, - selectedSkills: profile.effective.selectedSkills.map((skill) => ({ - ...skill, - })), - targetMode: profile.effective.targetMode, - networkPolicy: profile.effective.networkPolicy, - }; + assertResearchExecutionProfile(profile); + return { + profileId: profile.profileId, + models: Object.fromEntries( + MODEL_OVERRIDE_TARGETS.map((target) => [ + target, + profile.effective.models[target].modelUri, + ]), + ) as Record, + capabilityManifestRevision: profile.effective.capabilityManifestRevision, + capabilityIds: [...profile.effective.capabilityIds], + skillRegistryRevision: profile.effective.skillRegistryRevision, + selectedSkills: profile.effective.selectedSkills.map((skill) => ({ + ...skill, + })), + targetMode: profile.effective.targetMode, + networkPolicy: profile.effective.networkPolicy, + }; } export async function readLatestResearchExecutionBackgroundProfile(input: { - projectId: string; - threadId: string; + projectId: string; + threadId: string; }) { - return withDatabase(async (db) => { - const result = await db.query<{ metadata: JsonObject }>( - `SELECT metadata FROM research_runs + return withDatabase(async (db) => { + const result = await db.query<{ metadata: JsonObject }>( + `SELECT metadata FROM research_runs WHERE project_id = $1 AND thread_id = $2 AND status IN ('running', 'blocked') ORDER BY updated_at DESC, started_at DESC LIMIT 1`, - [input.projectId, input.threadId], - ); - const stored = result.rows[0]?.metadata?.[RESEARCH_EXECUTION_PROFILE_METADATA_KEY]; - return stored === undefined - ? undefined - : projectResearchExecutionProfileToBackground(readResearchExecutionProfile(stored)); - }); + [input.projectId, input.threadId], + ); + const stored = result.rows[0]?.metadata?.[ + RESEARCH_EXECUTION_PROFILE_METADATA_KEY + ]; + return stored === undefined + ? undefined + : projectResearchExecutionProfileToBackground( + readResearchExecutionProfile(stored), + ); + }); } export function projectResearchExecutionProfileToRuntime( - profile: ResearchExecutionProfile, + profile: ResearchExecutionProfile, ): ResearchExecutionProfileRuntimeProjection { - assertResearchExecutionProfile(profile); - const coordinator = profile.effective.models.coordinator; - const modelOverrides = Object.fromEntries( - MODEL_OVERRIDE_TARGETS.filter((target) => target !== "coordinator").map((target) => [ - target, - profile.effective.models[target].modelUri, - ]), - ) as ModelOverrideMap; - const budgets = profile.effective.budgets; - const validationAuthority = researchExecutionProfileValidationAuthority(profile); - return { - validationAuthorityMode: validationAuthority.effectiveMode, - validationAuthority: projectValidationAuthorityProvenance( - validationAuthority, - "research-execution-profile", - ), - modelUri: coordinator.modelUri, - modelOverrides, - ...(coordinator.contextWindowTokens - ? { contextWindowTokens: coordinator.contextWindowTokens } - : {}), - ...(coordinator.maxOutputTokens ? { maxOutputTokens: coordinator.maxOutputTokens } : {}), - enabledToolIds: [...profile.effective.capabilityIds], - ...(budgets.maxToolCalls ? { runtimeMaxToolCalls: budgets.maxToolCalls } : {}), - ...(budgets.maxRuntimeMs || budgets.maxTurns || budgets.maxCostUsd !== null - ? { - autonomyMode: { - ...(budgets.maxRuntimeMs ? { maxRuntimeMs: budgets.maxRuntimeMs } : {}), - ...(budgets.maxTurns ? { maxTurns: budgets.maxTurns } : {}), - ...(budgets.maxCostUsd !== null ? { maxCostUsd: budgets.maxCostUsd } : {}), - }, - } - : {}), - targetMode: profile.effective.targetMode, - networkProfile: profile.effective.networkPolicy, - renderJsonCollection: profile.effective.uiCollection, - capabilityManifestRevision: profile.effective.capabilityManifestRevision, - skillRegistryRevision: profile.effective.skillRegistryRevision, - researchExecutionProfileId: profile.profileId, - }; + assertResearchExecutionProfile(profile); + const coordinator = profile.effective.models.coordinator; + const modelOverrides = Object.fromEntries( + MODEL_OVERRIDE_TARGETS.filter((target) => target !== "coordinator").map( + (target) => [target, profile.effective.models[target].modelUri], + ), + ) as ModelOverrideMap; + const budgets = profile.effective.budgets; + const validationAuthority = researchExecutionProfileValidationAuthority(profile); + return { + validationAuthorityMode: validationAuthority.effectiveMode, + validationAuthority: projectValidationAuthorityProvenance( + validationAuthority, + "research-execution-profile", + ), + modelUri: coordinator.modelUri, + modelOverrides, + ...(coordinator.contextWindowTokens + ? { contextWindowTokens: coordinator.contextWindowTokens } + : {}), + ...(coordinator.maxOutputTokens + ? { maxOutputTokens: coordinator.maxOutputTokens } + : {}), + enabledToolIds: [...profile.effective.capabilityIds], + ...(budgets.maxToolCalls + ? { runtimeMaxToolCalls: budgets.maxToolCalls } + : {}), + ...(budgets.maxRuntimeMs || budgets.maxTurns || budgets.maxCostUsd !== null + ? { + autonomyMode: { + ...(budgets.maxRuntimeMs + ? { maxRuntimeMs: budgets.maxRuntimeMs } + : {}), + ...(budgets.maxTurns ? { maxTurns: budgets.maxTurns } : {}), + ...(budgets.maxCostUsd !== null + ? { maxCostUsd: budgets.maxCostUsd } + : {}), + }, + } + : {}), + targetMode: profile.effective.targetMode, + networkProfile: profile.effective.networkPolicy, + renderJsonCollection: profile.effective.uiCollection, + capabilityManifestRevision: profile.effective.capabilityManifestRevision, + skillRegistryRevision: profile.effective.skillRegistryRevision, + researchExecutionProfileId: profile.profileId, + }; } export function researchExecutionProfileValidationAuthority( - profile: ResearchExecutionProfile, + profile: ResearchExecutionProfile, ): ValidationAuthoritySelection { - assertResearchExecutionProfile(profile); - return profile.effective.validationAuthority - ? readValidationAuthoritySelection(profile.effective.validationAuthority) - : resolveValidationAuthoritySelection(profile.requested.validationAuthorityMode); + assertResearchExecutionProfile(profile); + return profile.effective.validationAuthority + ? readValidationAuthoritySelection(profile.effective.validationAuthority) + : resolveValidationAuthoritySelection( + profile.requested.validationAuthorityMode, + ); } -export function researchExecutionProfileForensics(profile: ResearchExecutionProfile): JsonObject { - assertResearchExecutionProfile(profile); - return toJsonObject(profile); +export function researchExecutionProfileForensics( + profile: ResearchExecutionProfile, +): JsonObject { + assertResearchExecutionProfile(profile); + return toJsonObject(profile); } -export function readResearchExecutionProfile(value: unknown): ResearchExecutionProfile { - assertResearchExecutionProfile(value); - return deepFreeze(value as ResearchExecutionProfile); +export function readResearchExecutionProfile( + value: unknown, +): ResearchExecutionProfile { + assertResearchExecutionProfile(value); + return deepFreeze(value as ResearchExecutionProfile); } export function assertResearchExecutionProfile( - value: unknown, + value: unknown, ): asserts value is ResearchExecutionProfile { - if (!value || typeof value !== "object" || Array.isArray(value)) { - throw new Error("Research Execution Profile is missing or malformed."); - } - const candidate = value as Partial; - if ( - candidate.schemaVersion !== RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION || - typeof candidate.profileId !== "string" || - !candidate.requested || - !candidate.effective - ) { - throw new Error("Research Execution Profile revision is missing or unsupported."); - } - const body = { - schemaVersion: candidate.schemaVersion, - requested: candidate.requested, - effective: candidate.effective, - }; - const expectedId = `research-profile-sha256:${sha256(stableStringify(body))}`; - if (candidate.profileId !== expectedId) { - throw new Error("Research Execution Profile content does not match its immutable profile id."); - } - const models = candidate.effective.models; - if (!models || MODEL_OVERRIDE_TARGETS.some((target) => !models[target]?.modelUri)) { - throw new Error("Research Execution Profile is missing an effective model revision."); - } - requireNonEmpty("capability manifest", candidate.effective.capabilityManifestRevision); - requireNonEmpty("skill registry", candidate.effective.skillRegistryRevision); - if (candidate.effective.validationAuthority !== undefined) { - readValidationAuthoritySelection(candidate.effective.validationAuthority); - } + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Research Execution Profile is missing or malformed."); + } + const candidate = value as Partial; + if ( + candidate.schemaVersion !== RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION || + typeof candidate.profileId !== "string" || + !candidate.requested || + !candidate.effective + ) { + throw new Error( + "Research Execution Profile revision is missing or unsupported.", + ); + } + const body = { + schemaVersion: candidate.schemaVersion, + requested: candidate.requested, + effective: candidate.effective, + }; + const expectedId = `research-profile-sha256:${sha256(stableStringify(body))}`; + if (candidate.profileId !== expectedId) { + throw new Error( + "Research Execution Profile content does not match its immutable profile id.", + ); + } + const models = candidate.effective.models; + if ( + !models || + MODEL_OVERRIDE_TARGETS.some((target) => !models[target]?.modelUri) + ) { + throw new Error( + "Research Execution Profile is missing an effective model revision.", + ); + } + requireNonEmpty( + "capability manifest", + candidate.effective.capabilityManifestRevision, + ); + requireNonEmpty("skill registry", candidate.effective.skillRegistryRevision); + if (candidate.effective.validationAuthority !== undefined) { + readValidationAuthoritySelection(candidate.effective.validationAuthority); + } } function resolveExecutionModel( - requestedModelUri: string | undefined, - overrides: { contextWindowTokens?: number; maxOutputTokens?: number }, + requestedModelUri: string | undefined, + overrides: { contextWindowTokens?: number; maxOutputTokens?: number }, ): ResearchExecutionModel { - const modelUri = resolveSecurityResearchMastraModelUri(requestedModelUri); - const parsed = parse(asModelConnectionString(modelUri)); - const normalized = normalize(parsed); - const registryEntry = findModelEntryForRef(modelUri); - const runtimeOptions = getMastraModelRuntimeOptions(modelUri); - const provider = - normalized.provider ?? - modelProviderFromHost(parsed.hostAlias) ?? - modelProviderFromHost(parsed.host) ?? - parsed.hostAlias ?? - parsed.host; - const normalizedMaxOutput = positiveIntegerOrNull(normalized.config.params.max_tokens); - return { - modelUri, - provider, - model: parsed.model, - contextWindowTokens: - positiveIntegerOrNull(overrides.contextWindowTokens) ?? registryEntry?.contextWindow ?? null, - maxOutputTokens: - positiveIntegerOrNull(overrides.maxOutputTokens) ?? - normalizedMaxOutput ?? - registryEntry?.maxOutputTokens ?? - null, - modelSettings: toJsonObject(runtimeOptions.modelSettings), - providerOptions: toJsonObject(runtimeOptions.providerOptions), - }; + const modelUri = resolveSecurityResearchMastraModelUri(requestedModelUri); + const parsed = parse(asModelConnectionString(modelUri)); + const normalized = normalize(parsed); + const registryEntry = findModelEntryForRef(modelUri); + const runtimeOptions = getMastraModelRuntimeOptions(modelUri); + const provider = + normalized.provider ?? + modelProviderFromHost(parsed.hostAlias) ?? + modelProviderFromHost(parsed.host) ?? + parsed.hostAlias ?? + parsed.host; + const normalizedMaxOutput = positiveIntegerOrNull( + normalized.config.params.max_tokens, + ); + return { + modelUri, + provider, + model: parsed.model, + contextWindowTokens: + positiveIntegerOrNull(overrides.contextWindowTokens) ?? + registryEntry?.contextWindow ?? + null, + maxOutputTokens: + positiveIntegerOrNull(overrides.maxOutputTokens) ?? + normalizedMaxOutput ?? + registryEntry?.maxOutputTokens ?? + null, + modelSettings: toJsonObject(runtimeOptions.modelSettings), + providerOptions: toJsonObject(runtimeOptions.providerOptions), + }; } -function canonicalModelOverrides(value: ModelOverrideMap | undefined): ModelOverrideMap { - return Object.fromEntries( - MODEL_OVERRIDE_TARGETS.flatMap((target) => { - const modelUri = readModelOverride(value, target); - return modelUri ? [[target, modelUri]] : []; - }), - ); +function canonicalModelOverrides( + value: ModelOverrideMap | undefined, +): ModelOverrideMap { + return Object.fromEntries( + MODEL_OVERRIDE_TARGETS.flatMap((target) => { + const modelUri = readModelOverride(value, target); + return modelUri ? [[target, modelUri]] : []; + }), + ); } -function canonicalSelectedSkills(value: readonly { id: string; revision: string }[]) { - return [...value] - .map((skill) => ({ id: skill.id.trim(), revision: skill.revision.trim() })) - .filter((skill) => skill.id) - .sort((left, right) => left.id.localeCompare(right.id)); +function canonicalSelectedSkills( + value: readonly { id: string; revision: string }[], +) { + return [...value] + .map((skill) => ({ id: skill.id.trim(), revision: skill.revision.trim() })) + .filter((skill) => skill.id) + .sort((left, right) => left.id.localeCompare(right.id)); } function canonicalStringRecord(value: Readonly>) { - return Object.fromEntries( - Object.entries(value) - .map(([key, item]) => [key.trim(), item.trim()] as const) - .filter(([key, item]) => key && item) - .sort(([left], [right]) => left.localeCompare(right)), - ); + return Object.fromEntries( + Object.entries(value) + .map(([key, item]) => [key.trim(), item.trim()] as const) + .filter(([key, item]) => key && item) + .sort(([left], [right]) => left.localeCompare(right)), + ); } function uniqueStrings(values: readonly string[]) { - return [...new Set(values.map((value) => value.trim()).filter(Boolean))].sort(); + return [ + ...new Set(values.map((value) => value.trim()).filter(Boolean)), + ].sort(); } -function requireCurrentRevision(label: string, requested: string, current: string) { - requireNonEmpty(label, requested); - requireNonEmpty(label, current); - if (requested !== current) { - throw new Error( - `Research Execution Profile ${label} revision ${requested} is unavailable; current revision is ${current}.`, - ); - } +function requireCurrentRevision( + label: string, + requested: string, + current: string, +) { + requireNonEmpty(label, requested); + requireNonEmpty(label, current); + if (requested !== current) { + throw new Error( + `Research Execution Profile ${label} revision ${requested} is unavailable; current revision is ${current}.`, + ); + } } -function requireNonEmpty(label: string, value: unknown): asserts value is string { - if (typeof value !== "string" || !value.trim()) { - throw new Error(`Research Execution Profile ${label} revision is missing.`); - } +function requireNonEmpty( + label: string, + value: unknown, +): asserts value is string { + if (typeof value !== "string" || !value.trim()) { + throw new Error(`Research Execution Profile ${label} revision is missing.`); + } } function cleanString(value: unknown) { - return typeof value === "string" && value.trim() ? value.trim() : undefined; + return typeof value === "string" && value.trim() ? value.trim() : undefined; } function positiveIntegerOrNull(value: unknown) { - return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : null; + return typeof value === "number" && Number.isSafeInteger(value) && value > 0 + ? value + : null; } function nonNegativeNumberOrNull(value: unknown) { - return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null; + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? value + : null; } function toJsonObject(value: unknown): JsonObject { - return JSON.parse( - JSON.stringify(value, (_key, item) => (item === undefined ? null : item)), - ) as JsonObject; + return JSON.parse( + JSON.stringify(value, (_key, item) => (item === undefined ? null : item)), + ) as JsonObject; } function stableStringify(value: unknown): string { - if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`; - if (value && typeof value === "object") { - return `{${Object.entries(value as Record) - .sort(([left], [right]) => left.localeCompare(right)) - .map(([key, item]) => `${JSON.stringify(key)}:${stableStringify(item)}`) - .join(",")}}`; - } - return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, item]) => `${JSON.stringify(key)}:${stableStringify(item)}`) + .join(",")}}`; + } + return JSON.stringify(value); } function sha256(value: string) { - return createHash("sha256").update(value).digest("hex"); + return createHash("sha256").update(value).digest("hex"); } function deepFreeze(value: T): T { - if (value && typeof value === "object" && !Object.isFrozen(value)) { - Object.freeze(value); - for (const nested of Object.values(value as Record)) deepFreeze(nested); - } - return value; + if (value && typeof value === "object" && !Object.isFrozen(value)) { + Object.freeze(value); + for (const nested of Object.values(value as Record)) + deepFreeze(nested); + } + return value; } diff --git a/tests/integration/scheduler-backed-research.test.ts b/tests/integration/scheduler-backed-research.test.ts index e2fe4c24c..6f47c4b32 100644 --- a/tests/integration/scheduler-backed-research.test.ts +++ b/tests/integration/scheduler-backed-research.test.ts @@ -66,32 +66,31 @@ function makeRunner( return { artifactId: `artifact-${input.schedulerTaskId}`, handoff: { - schemaVersion: 1, - projectId: input.projectId, - schedulerTaskId: input.schedulerTaskId, - stage: input.stage, - taskId: input.taskId, - attempt: input.attempt, - outcome: "completed", - summary: - typeof input.draft.summary === "string" ? input.draft.summary : input.draft.output, - coverage: [], - negativeCoverage: [], - assertionIds: [], - hypothesisIds: [], - artifactIds: [], - findingIds: [], - completedBranches: [], - openBranches: [], - blockers: [], - proposedTaskIds: [], - approvalNeeds: [], - executionProfile: input.executionProfile, - attribution: { producerAgentId: input.producerAgentId, parentHandoffIds: [] }, - budgetUsage: {}, - resourceClaims: [], - output: input.draft.output, - createdAt: new Date(0).toISOString(), + schemaVersion: 1, + projectId: input.projectId, + schedulerTaskId: input.schedulerTaskId, + stage: input.stage, + taskId: input.taskId, + attempt: input.attempt, + outcome: "completed", + summary: typeof input.draft.summary === "string" ? input.draft.summary : input.draft.output, + coverage: [], + negativeCoverage: [], + assertionIds: [], + hypothesisIds: [], + artifactIds: [], + findingIds: [], + completedBranches: [], + openBranches: [], + blockers: [], + proposedTaskIds: [], + approvalNeeds: [], + executionProfile: input.executionProfile, + attribution: { producerAgentId: input.producerAgentId, parentHandoffIds: [] }, + budgetUsage: {}, + resourceClaims: [], + output: input.draft.output, + createdAt: new Date(0).toISOString(), }, }; }, @@ -250,11 +249,6 @@ describe("scheduler-backed research runner", () => { ); const profile = { profileId: "research-profile-sha256:pinned", - validationAuthority: { - schemaVersion: "validation-authority-v1" as const, - requestedMode: "strict" as const, - effectiveMode: "strict" as const, - }, models: { coordinator: "llm://lmstudio/coordinator/pinned", rag: "llm://lmstudio/coordinator/pinned", @@ -276,7 +270,9 @@ describe("scheduler-backed research runner", () => { networkPolicy: "none", }; - await runner.enqueueResearchTasks(enqueueInput("project-1", [makeTask({ id: "t-profile" })])); + await runner.enqueueResearchTasks( + enqueueInput("project-1", [makeTask({ id: "t-profile" })]), + ); await runner.runEnqueuedTasks({ projectId: "project-1", owner: "worker-profile", diff --git a/tests/integration/validation-authority-provenance.test.ts b/tests/integration/validation-authority-provenance.test.ts index c43982ea3..d421af54b 100644 --- a/tests/integration/validation-authority-provenance.test.ts +++ b/tests/integration/validation-authority-provenance.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it } from "vitest"; import { SECURITY_CAPABILITY_MANIFEST_REVISION } from "../../src/lib/tools/catalog"; import { - projectResearchExecutionProfileToBackground, projectResearchExecutionProfileToRuntime, resolveResearchExecutionProfile, } from "../../src/server/research/execution-profile"; @@ -65,12 +64,6 @@ describe("validation authority provenance", () => { recordKind: "research-execution-profile", }, }); - expect(projectResearchExecutionProfileToBackground(profile)).toMatchObject({ - validationAuthority: { - requestedMode: "self", - effectiveMode: "self", - }, - }); const selection = resolveValidationAuthoritySelection("self"); const projections = VALIDATION_AUTHORITY_RECORD_KINDS.map((recordKind) => From d64baff41283b6916386cda305b0173b74d99b1c Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Thu, 27 Aug 2026 11:48:33 -0400 Subject: [PATCH 5/5] Preserve authority mode in background work --- src/server/chat/security-research-runtime-context.ts | 1 + src/server/research/execution-profile.ts | 2 ++ tests/integration/scheduler-backed-research.test.ts | 5 +++++ tests/integration/validation-authority-provenance.test.ts | 7 +++++++ 4 files changed, 15 insertions(+) diff --git a/src/server/chat/security-research-runtime-context.ts b/src/server/chat/security-research-runtime-context.ts index 0152c6405..6c4da4d32 100644 --- a/src/server/chat/security-research-runtime-context.ts +++ b/src/server/chat/security-research-runtime-context.ts @@ -272,6 +272,7 @@ export function buildSecurityResearchStageRequestContext(input: { [keys.skillRegistryRevision, input.executionProfile?.skillRegistryRevision], [keys.selectedSkillRefs, input.executionProfile?.selectedSkills], [keys.runtimeSkillCapabilities, input.executionProfile?.capabilityIds], + [keys.validationAuthorityMode, input.executionProfile?.validationAuthority.effectiveMode], [keys.modelUri, input.executionProfile?.models.coordinator], [keys.coordinatorModelUri, input.executionProfile?.models.coordinator], [keys.modelOverrides, input.executionProfile?.models], diff --git a/src/server/research/execution-profile.ts b/src/server/research/execution-profile.ts index e02232318..5095b378f 100644 --- a/src/server/research/execution-profile.ts +++ b/src/server/research/execution-profile.ts @@ -112,6 +112,7 @@ export type ResearchExecutionProfileRuntimeProjection = { export type ResearchExecutionBackgroundProfile = { profileId: string; + validationAuthority: ValidationAuthoritySelection; models: Record; capabilityManifestRevision: string; capabilityIds: string[]; @@ -255,6 +256,7 @@ export function projectResearchExecutionProfileToBackground( assertResearchExecutionProfile(profile); return { profileId: profile.profileId, + validationAuthority: researchExecutionProfileValidationAuthority(profile), models: Object.fromEntries( MODEL_OVERRIDE_TARGETS.map((target) => [ target, diff --git a/tests/integration/scheduler-backed-research.test.ts b/tests/integration/scheduler-backed-research.test.ts index 6f47c4b32..23da0744e 100644 --- a/tests/integration/scheduler-backed-research.test.ts +++ b/tests/integration/scheduler-backed-research.test.ts @@ -249,6 +249,11 @@ describe("scheduler-backed research runner", () => { ); const profile = { profileId: "research-profile-sha256:pinned", + validationAuthority: { + schemaVersion: "validation-authority-v1" as const, + requestedMode: "strict" as const, + effectiveMode: "strict" as const, + }, models: { coordinator: "llm://lmstudio/coordinator/pinned", rag: "llm://lmstudio/coordinator/pinned", diff --git a/tests/integration/validation-authority-provenance.test.ts b/tests/integration/validation-authority-provenance.test.ts index d421af54b..c43982ea3 100644 --- a/tests/integration/validation-authority-provenance.test.ts +++ b/tests/integration/validation-authority-provenance.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { SECURITY_CAPABILITY_MANIFEST_REVISION } from "../../src/lib/tools/catalog"; import { + projectResearchExecutionProfileToBackground, projectResearchExecutionProfileToRuntime, resolveResearchExecutionProfile, } from "../../src/server/research/execution-profile"; @@ -64,6 +65,12 @@ describe("validation authority provenance", () => { recordKind: "research-execution-profile", }, }); + expect(projectResearchExecutionProfileToBackground(profile)).toMatchObject({ + validationAuthority: { + requestedMode: "self", + effectiveMode: "self", + }, + }); const selection = resolveValidationAuthoritySelection("self"); const projections = VALIDATION_AUTHORITY_RECORD_KINDS.map((recordKind) =>