diff --git a/evals/validation-authority-foundation.ts b/evals/validation-authority-foundation.ts index a2a694c14..d537589e4 100644 --- a/evals/validation-authority-foundation.ts +++ b/evals/validation-authority-foundation.ts @@ -1,8 +1,13 @@ +import { + parseValidationAuthorityMode, + VALIDATION_AUTHORITY_MODES, + type ValidationAuthorityMode, +} from "../src/server/validation-plans/authority"; + export const VALIDATION_AUTHORITY_SCORER_VERSION = "validation-authority-deterministic-v1"; export const VALIDATION_AUTHORITY_MANIFEST_VERSION = "validation-authority-matrix-v1"; -export const VALIDATION_AUTHORITY_MODES = ["strict", "auto", "self", "yolo"] as const; -export type ValidationAuthorityMode = (typeof VALIDATION_AUTHORITY_MODES)[number]; +export { VALIDATION_AUTHORITY_MODES, type ValidationAuthorityMode }; export const VALIDATION_AUTHORITY_CLAIM_PROVENANCE = [ "executor-observed", @@ -79,7 +84,8 @@ export type ValidationAuthorityMetrics = { export type ValidationAuthorityRow = { modelId: string; providerId: string; - authorityMode: ValidationAuthorityMode; + requestedAuthorityMode: ValidationAuthorityMode; + effectiveAuthorityMode: ValidationAuthorityMode; fixtureId: string; fixtureVersion: string; promptVersion: string; @@ -114,6 +120,13 @@ export function validateValidationAuthorityManifest(manifest: ValidationAuthorit ) { failures.push("authority_modes_incomplete"); } + for (const mode of manifest.authorityModes) { + try { + parseValidationAuthorityMode(mode, "eval manifest validation authority mode"); + } catch { + failures.push(`invalid_authority_mode:${String(mode)}`); + } + } if (manifest.minimumRepeats < 3) failures.push("minimum_repeats_below_three"); if (manifest.executionAdmission.status !== "blocked" || manifest.executionAdmission.blockedByIssue !== 100) { failures.push("execution_admission_must_remain_blocked_by_issue_100"); @@ -203,13 +216,34 @@ const MATCHED_FIELDS: Array = [ export function validateMatchedValidationAuthorityRows(rows: ValidationAuthorityRow[]) { const failures: string[] = []; for (const mode of VALIDATION_AUTHORITY_MODES) { - if (!rows.some((row) => row.authorityMode === mode)) failures.push(`missing_mode:${mode}`); + if (!rows.some((row) => row.effectiveAuthorityMode === mode)) { + failures.push(`missing_mode:${mode}`); + } + } + for (const row of rows) { + try { + const requested = parseValidationAuthorityMode( + row.requestedAuthorityMode, + "eval row requested validation authority mode", + ); + const effective = parseValidationAuthorityMode( + row.effectiveAuthorityMode, + "eval row effective validation authority mode", + ); + if (requested !== effective) { + failures.push(`authority_mode_changed:${requested}:${effective}`); + } + } catch { + failures.push(`invalid_authority_mode:${String(row.effectiveAuthorityMode)}`); + } } const baseline = rows[0]; if (baseline) { for (const row of rows.slice(1)) { for (const field of MATCHED_FIELDS) { - if (row[field] !== baseline[field]) failures.push(`unmatched_input:${field}:${row.authorityMode}`); + if (row[field] !== baseline[field]) { + failures.push(`unmatched_input:${field}:${row.effectiveAuthorityMode}`); + } } } } @@ -219,7 +253,7 @@ export function validateMatchedValidationAuthorityRows(rows: ValidationAuthority export function summarizeValidationAuthorityRows(rows: ValidationAuthorityRow[], minimumRepeats = 3) { const groups = new Map(); for (const row of rows) { - const key = `${row.modelId}|${row.authorityMode}|${row.fixtureId}`; + const key = `${row.modelId}|${row.effectiveAuthorityMode}|${row.fixtureId}`; groups.set(key, [...(groups.get(key) ?? []), row]); } return [...groups.entries()].map(([key, group]) => { diff --git a/src/mastra/tools/validation-plan.ts b/src/mastra/tools/validation-plan.ts index e9026e2f7..48f88bed2 100644 --- a/src/mastra/tools/validation-plan.ts +++ b/src/mastra/tools/validation-plan.ts @@ -12,7 +12,13 @@ import { createScriptValidationExecutors, createValidationPlanExecutor, getDefaultValidationPlansService, + projectValidationAuthorityProvenance, + projectValidationOutcomeProvenance, + readValidationAuthoritySelection, + resolveValidationAuthoritySelection, type PayloadExecutionResult, + type ValidationAuthoritySelection, + type ValidationPlanService, type ValidationPlanStatus, } from "../../server/validation-plans"; import { readThreadTargetConfig } from "../../server/workspaces/target-mode"; @@ -165,6 +171,37 @@ const payloadResultSchema = z.object({ evidence: z.array(z.string()), error: z.string().optional(), proof: validationProofInputSchema.optional(), + authorityProvenance: z + .object({ + schemaVersion: z.literal("validation-authority-v1"), + requestedMode: z.enum(["strict", "auto", "self", "yolo"]), + effectiveMode: z.enum(["strict", "auto", "self", "yolo"]), + recordKind: z.literal("validation-plan"), + stage: z.literal("payload-result"), + source: z.enum([ + "executor-observed", + "model-asserted", + "human-reviewed", + "artifact-reviewed", + ]), + toolRunId: z.string().optional(), + targetAuthorizationId: z.string().optional(), + policyDecisionId: z.string().optional(), + durableApprovalId: z.string().optional(), + containmentPolicySnapshotId: z.string().optional(), + }) + .optional(), +}); +const validationAuthoritySchema = z.object({ + schemaVersion: z.literal("validation-authority-v1"), + requestedMode: z.enum(["strict", "auto", "self", "yolo"]), + effectiveMode: z.enum(["strict", "auto", "self", "yolo"]), + recordKind: z.literal("validation-plan"), +}); +const terminalTransitionSchema = z.object({ + ...validationAuthoritySchema.shape, + stage: z.literal("terminal-transition"), + source: z.enum(["executor-observed", "model-asserted", "human-reviewed", "artifact-reviewed"]), }); const controlMappingSchema = z.object({ framework: z.string(), @@ -247,11 +284,13 @@ export const validationPlanTool = createTool({ execution: z .object({ planId: z.string(), + validationAuthority: validationAuthoritySchema, status: validationPlanStatusSchema, confirmationKind: validationConfirmationKindSchema.optional(), payloadResults: z.array(payloadResultSchema), summary: z.string(), proofNodeId: z.string().optional(), + terminalTransition: terminalTransitionSchema, }) .optional(), }), @@ -261,6 +300,9 @@ export const validationPlanTool = createTool({ throw new Error("Validation plan updates require a projectId in the request context."); } const threadId = readContextString(context?.requestContext?.get?.("threadId")); + const validationAuthority = resolveValidationAuthoritySelection( + context?.requestContext?.get?.("validationAuthorityMode"), + ); const service = getDefaultValidationPlansService(); switch (input.action) { @@ -285,9 +327,13 @@ export const validationPlanTool = createTool({ ? { controlMappings: input.controlMappings.map(toControlMappingInput) } : {}), ...(input.artifacts ? { artifacts: input.artifacts } : {}), - ...(parseMetadataJson(input.metadataJson) - ? { metadata: parseMetadataJson(input.metadataJson) } - : {}), + metadata: { + ...(parseMetadataJson(input.metadataJson) ?? {}), + validationAuthority: projectValidationAuthorityProvenance( + validationAuthority, + "validation-plan", + ), + }, }); return { action: input.action, @@ -331,7 +377,8 @@ export const validationPlanTool = createTool({ const planId = requireField(input.planId, "planId", input.action); await service.assertSafeToRun({ planId, - approvalsGranted: input.approvalsGranted === true, + approvalsGranted: + validationAuthority.effectiveMode === "yolo" && input.approvalsGranted === true, }); const plan = await service.getPlan(planId); return { @@ -350,6 +397,7 @@ export const validationPlanTool = createTool({ }); const execution = await executor.executePlan({ planId, + validationAuthorityMode: validationAuthority.effectiveMode, approvalsGranted: input.approvalsGranted === true, ...(input.attackPathId ? { attackPathId: input.attackPathId } : {}), }); @@ -371,6 +419,7 @@ export const validationPlanTool = createTool({ }); const execution = await executor.executePlan({ planId, + validationAuthorityMode: validationAuthority.effectiveMode, approvalsGranted: input.approvalsGranted === true, ...(input.attackPathId ? { attackPathId: input.attackPathId } : {}), }); @@ -392,6 +441,7 @@ export const validationPlanTool = createTool({ }); const execution = await executor.executePlan({ planId, + validationAuthorityMode: validationAuthority.effectiveMode, approvalsGranted: input.approvalsGranted === true, ...(input.attackPathId ? { attackPathId: input.attackPathId } : {}), }); @@ -413,6 +463,7 @@ export const validationPlanTool = createTool({ }); const execution = await executor.executePlan({ planId, + validationAuthorityMode: validationAuthority.effectiveMode, approvalsGranted: input.approvalsGranted === true, ...(input.attackPathId ? { attackPathId: input.attackPathId } : {}), }); @@ -426,7 +477,16 @@ export const validationPlanTool = createTool({ }; } case "record_result": { - const plan = await service.recordPayloadResult(toPayloadResult(input)); + const planId = requireField(input.planId, "planId", input.action); + const selection = await requireModelValidationAuthority(service, planId); + const plan = await service.recordPayloadResult({ + ...toPayloadResult(input), + authorityProvenance: projectValidationOutcomeProvenance({ + selection, + stage: "payload-result", + source: "model-asserted", + }), + }); if (!plan) { throw new Error(`validation-plans: plan ${input.planId ?? ""} was not found`); } @@ -439,9 +499,15 @@ export const validationPlanTool = createTool({ } case "confirm": { const planId = requireField(input.planId, "planId", input.action); + const selection = await requireModelValidationAuthority(service, planId); const plan = await service.confirm({ planId, ...(input.confirmationKind ? { confirmationKind: input.confirmationKind } : {}), + authorityProvenance: projectValidationOutcomeProvenance({ + selection, + stage: "terminal-transition", + source: "model-asserted", + }), }); return { action: input.action, @@ -884,6 +950,23 @@ function parseMetadataJson(value?: string) { return parsed as Record; } +async function requireModelValidationAuthority( + service: ValidationPlanService, + planId: string, +): Promise { + const plan = await service.getPlan(planId); + if (!plan) { + throw new Error(`validation-plans: plan ${planId} was not found`); + } + const selection = readValidationAuthoritySelection(plan.metadata?.validationAuthority); + if (selection.effectiveMode !== "self" && selection.effectiveMode !== "yolo") { + throw new Error( + `validation-plans: ${selection.effectiveMode} mode requires executor-owned results and terminal transitions`, + ); + } + return selection; +} + function clampLimit(value: number) { if (!Number.isFinite(value)) { return undefined; diff --git a/src/server/chat/security-research-run.ts b/src/server/chat/security-research-run.ts index 7414df4b5..9bcc6cf1e 100644 --- a/src/server/chat/security-research-run.ts +++ b/src/server/chat/security-research-run.ts @@ -3718,6 +3718,7 @@ export const runSecurityResearchAgent = async ( ...(workspaceConfig ? { workspaceConfig } : {}), approvalMode, runtimeToolProfile, + validationAuthorityMode: input.metadata?.validationAuthorityMode, ...(Array.isArray(input.metadata?.commandAllowPatterns) ? { commandAllowPatterns: input.metadata.commandAllowPatterns.filter( diff --git a/src/server/chat/security-research-runtime-context.ts b/src/server/chat/security-research-runtime-context.ts index 5a3d4557f..6c4da4d32 100644 --- a/src/server/chat/security-research-runtime-context.ts +++ b/src/server/chat/security-research-runtime-context.ts @@ -83,6 +83,7 @@ export const securityResearchRuntimeContextKeys = { skillRegistryRevision: "skillRegistryRevision", selectedSkillRefs: "selectedSkillRefs", runtimeSkillCapabilities: "runtimeSkillCapabilities", + validationAuthorityMode: "validationAuthorityMode", } as const; export function createSecurityResearchRequestContext( @@ -133,6 +134,7 @@ export function buildSecurityResearchChatRequestContext(input: { evaluationArm?: string; terminalProtocolMode?: "legacy" | "typed" | "gated"; terminalCleanupRequiredBeforeFinish?: boolean; + validationAuthorityMode?: unknown; }) { const keys = securityResearchRuntimeContextKeys; const resourceId = buildSecurityResearchMemoryResourceId( @@ -174,6 +176,7 @@ export function buildSecurityResearchChatRequestContext(input: { [keys.evaluationArm, input.evaluationArm], [keys.terminalProtocolMode, input.terminalProtocolMode], [keys.terminalCleanupRequiredBeforeFinish, input.terminalCleanupRequiredBeforeFinish], + [keys.validationAuthorityMode, input.validationAuthorityMode], [keys.workspaceConfig, input.workspaceConfig], [keys.commandAllowPatterns, input.commandAllowPatterns], [keys.commandBlockPatterns, input.commandBlockPatterns], @@ -269,6 +272,7 @@ export function buildSecurityResearchStageRequestContext(input: { [keys.skillRegistryRevision, input.executionProfile?.skillRegistryRevision], [keys.selectedSkillRefs, input.executionProfile?.selectedSkills], [keys.runtimeSkillCapabilities, input.executionProfile?.capabilityIds], + [keys.validationAuthorityMode, input.executionProfile?.validationAuthority.effectiveMode], [keys.modelUri, input.executionProfile?.models.coordinator], [keys.coordinatorModelUri, input.executionProfile?.models.coordinator], [keys.modelOverrides, input.executionProfile?.models], diff --git a/src/server/chat/securityResearchTurn.ts b/src/server/chat/securityResearchTurn.ts index 9a5632cdf..2b42ec7f7 100644 --- a/src/server/chat/securityResearchTurn.ts +++ b/src/server/chat/securityResearchTurn.ts @@ -29,6 +29,7 @@ import { recoverResearchTurn, shouldRecoverResearchResult } from "../research/re import { projectResearchExecutionProfileToRuntime, researchExecutionProfileForensics, + researchExecutionProfileValidationAuthority, resolveResearchExecutionProfile, } from "../research/execution-profile"; import { withResearchRunContext } from "../research/run-context"; @@ -39,6 +40,7 @@ import { type ResearchTurnLedgerHandle, } from "../research/turn-ledger"; import { createTargetAuthorization, inferTargetKind, upsertProjectTarget } from "../targets"; +import { projectValidationAuthorityProvenance } from "../validation-plans/authority"; import type { ThreadWorkspaceResolution, ThreadWorkspaceVolumeStatus } from "../workspaces"; import { readThreadTargetConfig } from "../workspaces/target-mode"; import type { BrowserMcpExecutionLedgerStore } from "./browser-mcp-execution-ledger"; @@ -223,6 +225,7 @@ export const runSecurityResearchTurn = async ( const targetConfig = readTargetConfigForExecutionProfile(agentInput.metadata); const executionProfile = resolveResearchExecutionProfile( { + validationAuthorityMode: agentInput.metadata?.validationAuthorityMode, modelUri: readOptionalString(agentInput.metadata?.modelUri) ?? undefined, runtimeModelUri: readOptionalString(agentInput.metadata?.runtimeModelUri) ?? undefined, modelOverrides: readModelOverrides(agentInput.metadata?.modelOverrides), @@ -572,6 +575,7 @@ async function recordRecoveryUsageEvent(input: { attributes: { researchRunId: input.ledger.researchRunId, researchTurnLedgerId: input.ledger.turnLedgerId, + ...usageEventValidationAuthority(input.ledger), recoveryAttemptIndex: input.attemptIndex, failed: true, promptTokens: readOptionalNumber(usage.promptTokens), @@ -637,6 +641,7 @@ async function finalizeFailedLedger( attributes: { researchRunId: ledger.researchRunId, researchTurnLedgerId: ledger.turnLedgerId, + ...usageEventValidationAuthority(ledger), modelStepIndex: 0, failed: true, promptTokens: readOptionalNumber(errorUsage.promptTokens), @@ -819,6 +824,7 @@ async function recordResearchTurnUsageEvents(input: { attributes: { researchRunId: input.ledger.researchRunId, researchTurnLedgerId: input.ledger.turnLedgerId, + ...usageEventValidationAuthority(input.ledger), modelStepIndex: step.stepIndex, provider: usage.provider, model: usage.model, @@ -838,6 +844,16 @@ async function recordResearchTurnUsageEvents(input: { return ids; } +function usageEventValidationAuthority(ledger: ResearchTurnLedgerHandle) { + if (!ledger.executionProfile) return {}; + return { + validationAuthority: projectValidationAuthorityProvenance( + researchExecutionProfileValidationAuthority(ledger.executionProfile), + "usage-event", + ), + }; +} + function readStringArray(value: unknown) { return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.length > 0) diff --git a/src/server/research/execution-profile.ts b/src/server/research/execution-profile.ts index b604b4266..5095b378f 100644 --- a/src/server/research/execution-profile.ts +++ b/src/server/research/execution-profile.ts @@ -18,6 +18,14 @@ import { } from "../../mastra/config/model"; import { withDatabase } from "../db/client"; import type { JsonObject } from "../db/types"; +import { + projectValidationAuthorityProvenance, + readValidationAuthoritySelection, + resolveValidationAuthoritySelection, + type ValidationAuthorityMode, + type ValidationAuthorityProvenance, + type ValidationAuthoritySelection, +} from "../validation-plans/authority"; export const RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION = "research-execution-profile-v1"; @@ -26,6 +34,7 @@ export const RESEARCH_EXECUTION_PROFILE_METADATA_KEY = "executionProfile"; export type ResearchExecutionTargetMode = "none" | "container" | "remote"; export type RequestedResearchExecutionProfile = { + validationAuthorityMode?: unknown; modelUri?: string; runtimeModelUri?: string; modelOverrides?: ModelOverrideMap; @@ -60,6 +69,7 @@ export type ResearchExecutionProfile = { profileId: string; requested: JsonObject; effective: { + validationAuthority: ValidationAuthoritySelection; models: Record; capabilityManifestRevision: string; capabilityIds: string[]; @@ -79,6 +89,8 @@ export type ResearchExecutionProfile = { }; export type ResearchExecutionProfileRuntimeProjection = { + validationAuthorityMode: ValidationAuthorityMode; + validationAuthority: ValidationAuthorityProvenance; modelUri: string; modelOverrides: ModelOverrideMap; contextWindowTokens?: number; @@ -100,6 +112,7 @@ export type ResearchExecutionProfileRuntimeProjection = { export type ResearchExecutionBackgroundProfile = { profileId: string; + validationAuthority: ValidationAuthoritySelection; models: Record; capabilityManifestRevision: string; capabilityIds: string[]; @@ -140,9 +153,12 @@ export function resolveResearchExecutionProfile( const currentSkills = new Map( canonicalSelectedSkills(current.selectedSkills).map((skill) => [ skill.id, - skill.revision, + skill.revision, ]), ); + const validationAuthority = resolveValidationAuthoritySelection( + input.validationAuthorityMode, + ); for (const skill of selectedSkills) { if (!skill.revision.trim()) { throw new Error( @@ -186,6 +202,7 @@ export function resolveResearchExecutionProfile( ) as Record; const requested = toJsonObject({ + validationAuthorityMode: validationAuthority.requestedMode, modelUri: requestedModelUri ?? null, runtimeModelUri: runtimeModelUri ?? null, modelOverrides: requestedOverrides, @@ -208,6 +225,7 @@ export function resolveResearchExecutionProfile( schemaVersion: RESEARCH_EXECUTION_PROFILE_SCHEMA_VERSION, requested, effective: { + validationAuthority, models, capabilityManifestRevision: current.capabilityManifestRevision, capabilityIds: uniqueStrings(input.capabilityIds), @@ -238,6 +256,7 @@ export function projectResearchExecutionProfileToBackground( assertResearchExecutionProfile(profile); return { profileId: profile.profileId, + validationAuthority: researchExecutionProfileValidationAuthority(profile), models: Object.fromEntries( MODEL_OVERRIDE_TARGETS.map((target) => [ target, @@ -289,7 +308,13 @@ export function projectResearchExecutionProfileToRuntime( ), ) as ModelOverrideMap; const budgets = profile.effective.budgets; + const validationAuthority = researchExecutionProfileValidationAuthority(profile); return { + validationAuthorityMode: validationAuthority.effectiveMode, + validationAuthority: projectValidationAuthorityProvenance( + validationAuthority, + "research-execution-profile", + ), modelUri: coordinator.modelUri, modelOverrides, ...(coordinator.contextWindowTokens @@ -324,6 +349,17 @@ export function projectResearchExecutionProfileToRuntime( }; } +export function researchExecutionProfileValidationAuthority( + profile: ResearchExecutionProfile, +): ValidationAuthoritySelection { + assertResearchExecutionProfile(profile); + return profile.effective.validationAuthority + ? readValidationAuthoritySelection(profile.effective.validationAuthority) + : resolveValidationAuthoritySelection( + profile.requested.validationAuthorityMode, + ); +} + export function researchExecutionProfileForensics( profile: ResearchExecutionProfile, ): JsonObject { @@ -380,6 +416,9 @@ export function assertResearchExecutionProfile( candidate.effective.capabilityManifestRevision, ); requireNonEmpty("skill registry", candidate.effective.skillRegistryRevision); + if (candidate.effective.validationAuthority !== undefined) { + readValidationAuthoritySelection(candidate.effective.validationAuthority); + } } function resolveExecutionModel( diff --git a/src/server/research/trajectory-export.ts b/src/server/research/trajectory-export.ts index 6ec77ff6f..2e992c40d 100644 --- a/src/server/research/trajectory-export.ts +++ b/src/server/research/trajectory-export.ts @@ -14,6 +14,11 @@ import type { ToolRunRow, UsageEventRow, } from "../db/types"; +import { + readResearchExecutionProfile, + researchExecutionProfileValidationAuthority, +} from "./execution-profile"; +import { projectValidationAuthorityProvenance } from "../validation-plans/authority"; export const POLICY_TRAJECTORY_SCHEMA_VERSION = "policy-trajectory-v1"; @@ -128,6 +133,7 @@ export async function exportPolicyLearningTrajectory( ); const usage = await loadRows(tx, "usage_events", usageEventIds); validateCheckpointEvidence(input.verifiedCheckpointRefs ?? [], artifacts, toolRuns); + const validationAuthority = exportValidationAuthority(run); const environmentValid = Object.entries(input.environmentValidity) .filter(([key]) => key !== "diagnosticArtifactIds") @@ -170,9 +176,14 @@ export async function exportPolicyLearningTrajectory( startedAt: row.started_at, finishedAt: row.finished_at, errorPresent: Boolean(row.error), + validationAuthority: row.metadata.validationAuthority ?? null, })), verifiedCheckpointRefs: input.verifiedCheckpointRefs ?? [], usageEventIds: usage.map((row) => row.id), + usageValidationAuthority: usage.map((row) => ({ + usageEventId: row.id, + validationAuthority: row.attributes.validationAuthority ?? null, + })), artifactIds: artifacts.map((row) => row.id), environmentValidity: input.environmentValidity, policyOutcome: input.policyOutcome, @@ -181,6 +192,7 @@ export async function exportPolicyLearningTrajectory( eligible, datasetProvenance: { ...input.datasetProvenance, + validationAuthority, researchRunId: run.id, turnLedgerId: turn.id, modelStepId: step.id, @@ -255,6 +267,15 @@ export async function exportPolicyLearningTrajectory( ); } +function exportValidationAuthority(run: ResearchRunRow) { + const storedProfile = run.metadata.executionProfile; + if (storedProfile === undefined) return null; + return projectValidationAuthorityProvenance( + researchExecutionProfileValidationAuthority(readResearchExecutionProfile(storedProfile)), + "evidence-export", + ); +} + function assertPassiveChosenAction(capabilityId: string, allowed: readonly string[]) { if (!allowed.includes(capabilityId)) { throw new Error(`Chosen action was not model-visible: ${capabilityId}`); diff --git a/src/server/security-actions/execution.ts b/src/server/security-actions/execution.ts index 728bf3923..ff0255cb6 100644 --- a/src/server/security-actions/execution.ts +++ b/src/server/security-actions/execution.ts @@ -16,6 +16,12 @@ import { } from "../containment"; import { getArtifactService } from "../evidence"; import type { TargetAuthorizationCoverage } from "../targets/authorization-ledger"; +import { + projectValidationAuthorityProvenance, + readValidationAuthoritySelection, + resolveValidationAuthoritySelection, + type ValidationAuthoritySelection, +} from "../validation-plans/authority"; export type SecurityActionIntent = { capabilityId: string; @@ -94,6 +100,7 @@ export type SecurityActionContainmentAudit = { export type SecurityActionExecutionInput = { projectId: string; threadId?: string; + validationAuthority?: ValidationAuthoritySelection; intent: SecurityActionIntent; durableApprovalId?: string; approvalAction?: DurableApprovalAction; @@ -202,6 +209,9 @@ export function createSecurityActionExecutor(dependencies: SecurityActionExecuti request: SecurityActionExecutionInput, ): Promise> { const intent = normalizeSecurityActionIntent(request.intent); + const validationAuthority = request.validationAuthority + ? readValidationAuthoritySelection(request.validationAuthority) + : resolveValidationAuthoritySelection(undefined); const startedAt = now().toISOString(); const containment = resolveSecurityActionContainment(request, intent, startedAt); const run = await dependencies.toolRuns.create(request.projectId, { @@ -212,6 +222,10 @@ export function createSecurityActionExecutor(dependencies: SecurityActionExecuti startedAt, metadata: { normalizedIntent: intent.approvalIntent.normalized, + validationAuthority: projectValidationAuthorityProvenance( + validationAuthority, + "tool-run", + ), ...(containment.audit ? { containment: containment.audit } : {}), }, }); diff --git a/src/server/validation-plans/authority.ts b/src/server/validation-plans/authority.ts new file mode 100644 index 000000000..d9583fe0a --- /dev/null +++ b/src/server/validation-plans/authority.ts @@ -0,0 +1,167 @@ +export const VALIDATION_AUTHORITY_SCHEMA_VERSION = "validation-authority-v1"; + +export const VALIDATION_AUTHORITY_MODES = ["strict", "auto", "self", "yolo"] as const; +export type ValidationAuthorityMode = (typeof VALIDATION_AUTHORITY_MODES)[number]; + +export const VALIDATION_AUTHORITY_RECORD_KINDS = [ + "research-execution-profile", + "tool-run", + "validation-plan", + "usage-event", + "eval-row", + "evidence-export", +] as const; +export type ValidationAuthorityRecordKind = (typeof VALIDATION_AUTHORITY_RECORD_KINDS)[number]; + +export type ValidationAuthoritySelection = { + schemaVersion: typeof VALIDATION_AUTHORITY_SCHEMA_VERSION; + requestedMode: ValidationAuthorityMode; + effectiveMode: ValidationAuthorityMode; +}; + +export type ValidationAuthorityProvenance = ValidationAuthoritySelection & { + recordKind: ValidationAuthorityRecordKind; +}; + +export const VALIDATION_OUTCOME_SOURCES = [ + "executor-observed", + "model-asserted", + "human-reviewed", + "artifact-reviewed", +] as const; +export type ValidationOutcomeSource = (typeof VALIDATION_OUTCOME_SOURCES)[number]; + +export type ValidationOutcomeProvenance = Omit & { + recordKind: "validation-plan"; + stage: "payload-result" | "terminal-transition"; + source: ValidationOutcomeSource; + toolRunId?: string; + targetAuthorizationId?: string; + policyDecisionId?: string; + durableApprovalId?: string; + containmentPolicySnapshotId?: string; +}; +export type ValidationPayloadResultProvenance = ValidationOutcomeProvenance & { + stage: "payload-result"; +}; +export type ValidationTerminalTransitionProvenance = ValidationOutcomeProvenance & { + stage: "terminal-transition"; +}; + +/** + * Server-owned facts returned by the validation executor's injected authority + * resolver. These values are deliberately not part of model/tool input. + */ +export type ValidationExecutionAuthority = { + allowed: boolean; + reason?: string; + payloadId: string; + targetId: string; + taskId?: string; + toolRunId: string; + targetAuthorizationId: string; + policyDecisionId: string; + taskScopeVerified: boolean; + durableApprovalId?: string; + containmentPolicySnapshotId?: string; +}; + +/** + * Resolves a new run's immutable validation authority selection. Missing values + * retain the existing guarded executor/policy/human behavior. Explicit values + * are never aliased or silently downgraded. + */ +export function resolveValidationAuthoritySelection( + requested: unknown, +): ValidationAuthoritySelection { + const requestedMode = + requested === undefined ? "strict" : parseValidationAuthorityMode(requested); + return Object.freeze({ + schemaVersion: VALIDATION_AUTHORITY_SCHEMA_VERSION, + requestedMode, + effectiveMode: requestedMode, + }); +} + +export function parseValidationAuthorityMode( + value: unknown, + field = "validation authority mode", +): ValidationAuthorityMode { + if ( + typeof value === "string" && + (VALIDATION_AUTHORITY_MODES as readonly string[]).includes(value) + ) { + return value as ValidationAuthorityMode; + } + throw new Error(`${field} must be one of strict, auto, self, yolo.`); +} + +export function readValidationAuthoritySelection(value: unknown): ValidationAuthoritySelection { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Validation authority selection is missing or malformed."); + } + const candidate = value as Partial; + if (candidate.schemaVersion !== VALIDATION_AUTHORITY_SCHEMA_VERSION) { + throw new Error("Validation authority selection revision is missing or unsupported."); + } + const requestedMode = parseValidationAuthorityMode( + candidate.requestedMode, + "requested validation authority mode", + ); + const effectiveMode = parseValidationAuthorityMode( + candidate.effectiveMode, + "effective validation authority mode", + ); + if (requestedMode !== effectiveMode) { + throw new Error( + "Validation authority mode cannot change between requested and effective run state.", + ); + } + return Object.freeze({ + schemaVersion: VALIDATION_AUTHORITY_SCHEMA_VERSION, + requestedMode, + effectiveMode, + }); +} + +export function projectValidationAuthorityProvenance< + TRecordKind extends ValidationAuthorityRecordKind, +>( + selection: ValidationAuthoritySelection, + recordKind: TRecordKind, +): ValidationAuthorityProvenance & { recordKind: TRecordKind } { + const validated = readValidationAuthoritySelection(selection); + return Object.freeze({ ...validated, recordKind }); +} + +export function projectValidationOutcomeProvenance< + TStage extends ValidationOutcomeProvenance["stage"], +>(input: { + selection: ValidationAuthoritySelection; + stage: TStage; + source: ValidationOutcomeSource; + authority?: ValidationExecutionAuthority; +}): ValidationOutcomeProvenance & { stage: TStage } { + const provenance: ValidationOutcomeProvenance & { stage: TStage } = { + ...readValidationAuthoritySelection(input.selection), + recordKind: "validation-plan", + stage: input.stage, + source: input.source, + ...(input.authority?.toolRunId ? { toolRunId: input.authority.toolRunId } : {}), + ...(input.authority?.targetAuthorizationId + ? { targetAuthorizationId: input.authority.targetAuthorizationId } + : {}), + ...(input.authority?.policyDecisionId + ? { policyDecisionId: input.authority.policyDecisionId } + : {}), + ...(input.authority?.durableApprovalId + ? { durableApprovalId: input.authority.durableApprovalId } + : {}), + ...(input.authority?.containmentPolicySnapshotId + ? { + containmentPolicySnapshotId: input.authority.containmentPolicySnapshotId, + } + : {}), + }; + return Object.freeze(provenance); +} diff --git a/src/server/validation-plans/executor.ts b/src/server/validation-plans/executor.ts index 6eceda4e6..d192f2452 100644 --- a/src/server/validation-plans/executor.ts +++ b/src/server/validation-plans/executor.ts @@ -7,6 +7,20 @@ import type { ValidationPlan, ValidationPlanService, } from "./types"; +import { + projectValidationAuthorityProvenance, + projectValidationOutcomeProvenance, + readValidationAuthoritySelection, + resolveValidationAuthoritySelection, + type ValidationAuthorityMode, + type ValidationAuthorityProvenance, + type ValidationAuthoritySelection, + type ValidationExecutionAuthority, + type ValidationOutcomeProvenance, + type ValidationOutcomeSource, + type ValidationPayloadResultProvenance, + type ValidationTerminalTransitionProvenance, +} from "./authority"; export type ValidationPayloadExecutionContext = { plan: ValidationPlan; @@ -20,19 +34,36 @@ export type ValidationPayloadExecutor = ( export type ValidationPlanExecutionInput = { planId: string; + validationAuthorityMode?: unknown; approvalsGranted?: boolean; attackPathId?: string; + modelDecision?: { + proceed: boolean; + confirm: boolean; + }; }; export type ValidationPlanExecutionResult = { planId: string; + validationAuthority: ValidationAuthorityProvenance & { recordKind: "validation-plan" }; status: ValidationPlan["status"]; confirmationKind?: ValidationPlan["confirmationKind"]; payloadResults: PayloadExecutionResult[]; summary: string; proofNodeId?: string; + terminalTransition: ValidationTerminalTransitionProvenance; +}; + +export type ValidationExecutionAuthorityContext = { + plan: ValidationPlan; + payload: ValidationPayload; + mode: Exclude; }; +export type ValidationExecutionAuthorityResolver = ( + context: ValidationExecutionAuthorityContext, +) => Promise; + export type ValidationPlanExecutorDependencies = { service?: ValidationPlanService; attackPaths?: AttackPathService; @@ -42,6 +73,7 @@ export type ValidationPlanExecutorDependencies = { executeScript?: ValidationPayloadExecutor; executeManual?: ValidationPayloadExecutor; negativeResults?: NegativeResultService; + resolveExecutionAuthority?: ValidationExecutionAuthorityResolver; }; export function createValidationPlanExecutor(deps: ValidationPlanExecutorDependencies = {}) { @@ -54,26 +86,94 @@ export function createValidationPlanExecutor(deps: ValidationPlanExecutorDepende throw new Error(`validation-plans: plan is not executable: ${finalized.reason}`); } + const validationAuthority = resolvePinnedAuthority( + finalized.plan, + input.validationAuthorityMode, + ); + + const validationPlanAuthority = projectValidationAuthorityProvenance( + validationAuthority, + "validation-plan", + ); + await service.updatePlan(input.planId, { + metadata: { + ...(finalized.plan.metadata ?? {}), + validationAuthority: validationPlanAuthority, + }, + }); + const authorities = await resolvePayloadAuthorities({ + deps, + plan: finalized.plan, + mode: validationAuthority.effectiveMode, + }); await service.assertSafeToRun({ planId: input.planId, - approvalsGranted: input.approvalsGranted === true, + approvalsGranted: executionApprovalGranted({ + mode: validationAuthority.effectiveMode, + modelClaim: input.approvalsGranted === true, + plan: finalized.plan, + authorities, + }), }); - const running = await service.startRun({ planId: input.planId }); const payloadResults: PayloadExecutionResult[] = []; for (const payload of running.payloads) { - const result = await executePayload({ - deps, + const authority = authorities.get(payload.id); + const denial = executionDenial({ + mode: validationAuthority.effectiveMode, plan: running, payload, - approvalsGranted: input.approvalsGranted === true, + authority, + modelDecision: input.modelDecision, }); + const source = outcomeSource(validationAuthority.effectiveMode); + const result = denial + ? deniedPayloadResult({ + payload, + denial, + provenance: projectValidationOutcomeProvenance({ + selection: validationAuthority, + stage: "payload-result", + source, + ...(authority ? { authority } : {}), + }), + }) + : await executePayload({ + deps, + plan: running, + payload, + approvalsGranted: executorApprovalGranted({ + mode: validationAuthority.effectiveMode, + modelClaim: input.approvalsGranted === true, + authority, + }), + provenance: projectValidationOutcomeProvenance({ + selection: validationAuthority, + stage: "payload-result", + source, + ...(authority ? { authority } : {}), + }), + requireDurableEvidence: + validationAuthority.effectiveMode === "strict" || + validationAuthority.effectiveMode === "auto", + }); payloadResults.push(result); await service.recordPayloadResult({ planId: running.id, ...result }); } - const terminal = await settlePlan(service, running.id, payloadResults); + const terminalTransition = projectValidationOutcomeProvenance({ + selection: validationAuthority, + stage: "terminal-transition", + source: terminalOutcomeSource(validationAuthority.effectiveMode, payloadResults), + }); + const terminal = await settlePlan( + service, + running.id, + payloadResults, + terminalTransition, + validationAuthority.effectiveMode !== "self" || input.modelDecision?.confirm === true, + ); await recordValidationNegativeResults({ negativeResults: deps.negativeResults, plan: terminal, @@ -90,11 +190,13 @@ export function createValidationPlanExecutor(deps: ValidationPlanExecutorDepende return { planId: terminal.id, + validationAuthority: validationPlanAuthority, status: terminal.status, ...(terminal.confirmationKind ? { confirmationKind: terminal.confirmationKind } : {}), payloadResults, summary: summarizeExecution(terminal, payloadResults), ...(proofNodeId ? { proofNodeId } : {}), + terminalTransition, }; }, }; @@ -180,6 +282,8 @@ async function executePayload(input: { plan: ValidationPlan; payload: ValidationPayload; approvalsGranted: boolean; + provenance: ValidationPayloadResultProvenance; + requireDurableEvidence: boolean; }): Promise { const context: ValidationPayloadExecutionContext = { plan: input.plan, @@ -194,10 +298,23 @@ async function executePayload(input: { observed: `No executor is configured for ${input.payload.kind} validation payloads.`, evidence: [], error: "executor-not-configured", + authorityProvenance: input.provenance, }; } try { - return await executor(context); + const result = await executor(context); + if (input.requireDurableEvidence && result.status === "pass" && result.evidence.length === 0) { + return { + ...result, + status: "inconclusive", + error: "durable-evidence-required", + authorityProvenance: input.provenance, + }; + } + return { + ...result, + authorityProvenance: trustedExecutorProvenance(result, input.provenance), + }; } catch (error) { const message = error instanceof Error ? error.message : String(error); return { @@ -206,6 +323,7 @@ async function executePayload(input: { observed: `Validation payload failed: ${message}`, evidence: [], error: message, + authorityProvenance: input.provenance, }; } } @@ -234,10 +352,182 @@ const defaultManualExecutor: ValidationPayloadExecutor = async ({ payload }) => evidence: [], }); +function resolvePinnedAuthority( + plan: ValidationPlan, + requested: unknown, +): ValidationAuthoritySelection { + const persisted = plan.metadata?.validationAuthority; + const selection = + requested === undefined && persisted !== undefined + ? readValidationAuthoritySelection(persisted) + : resolveValidationAuthoritySelection(requested); + if (persisted !== undefined) { + const pinned = readValidationAuthoritySelection(persisted); + if ( + pinned.requestedMode !== selection.requestedMode || + pinned.effectiveMode !== selection.effectiveMode + ) { + throw new Error( + `validation-plans: authority mode is pinned to ${pinned.effectiveMode} and cannot change mid-run`, + ); + } + } + return selection; +} + +async function resolvePayloadAuthorities(input: { + deps: ValidationPlanExecutorDependencies; + plan: ValidationPlan; + mode: ValidationAuthorityMode; +}): Promise> { + const authorities = new Map(); + if (input.mode === "yolo" || !input.deps.resolveExecutionAuthority) { + return authorities; + } + for (const payload of input.plan.payloads) { + const authority = await input.deps.resolveExecutionAuthority({ + plan: input.plan, + payload, + mode: input.mode, + }); + authorities.set(payload.id, authority); + } + return authorities; +} + +function executionApprovalGranted(input: { + mode: ValidationAuthorityMode; + modelClaim: boolean; + plan: ValidationPlan; + authorities: Map; +}): boolean { + if (input.mode === "yolo") { + return input.modelClaim; + } + if (input.mode === "auto" || input.mode === "self") { + // The model-authored boolean is ignored. Per-payload execution remains + // fail-closed on the injected server authority below. + return true; + } + return input.plan.payloads.every((payload) => { + if (payload.isNonDestructive) return true; + const authority = input.authorities.get(payload.id); + if (!authorityIsValid(input.plan, payload, authority)) return false; + return input.mode !== "strict" || Boolean(authority?.durableApprovalId); + }); +} + +function executorApprovalGranted(input: { + mode: ValidationAuthorityMode; + modelClaim: boolean; + authority?: ValidationExecutionAuthority; +}): boolean { + if (input.mode === "yolo") return input.modelClaim; + if (input.mode === "strict") return Boolean(input.authority?.durableApprovalId); + return input.authority?.allowed === true; +} + +function executionDenial(input: { + mode: ValidationAuthorityMode; + plan: ValidationPlan; + payload: ValidationPayload; + authority?: ValidationExecutionAuthority; + modelDecision?: ValidationPlanExecutionInput["modelDecision"]; +}): string | undefined { + if (input.mode === "yolo") return undefined; + if (input.mode === "self" && input.modelDecision?.proceed !== true) { + return "model-declined-execution"; + } + if (input.mode === "strict" && input.payload.isNonDestructive) { + return undefined; + } + if (!input.authority) return "server-authority-not-resolved"; + if (!authorityIsValid(input.plan, input.payload, input.authority)) { + return input.authority.reason ?? "server-authority-mismatch"; + } + if ( + input.mode === "strict" && + !input.payload.isNonDestructive && + !input.authority.durableApprovalId + ) { + return "exact-durable-approval-required"; + } + return undefined; +} + +function authorityIsValid( + plan: ValidationPlan, + payload: ValidationPayload, + authority: ValidationExecutionAuthority | undefined, +): boolean { + return Boolean( + authority?.allowed && + authority.taskScopeVerified && + authority.payloadId === payload.id && + authority.targetId === plan.targetId && + (plan.taskId === undefined || authority.taskId === plan.taskId) && + authority.toolRunId.trim() && + authority.targetAuthorizationId.trim() && + authority.policyDecisionId.trim(), + ); +} + +function deniedPayloadResult(input: { + payload: ValidationPayload; + denial: string; + provenance: ValidationPayloadResultProvenance; +}): PayloadExecutionResult { + return { + payloadId: input.payload.id, + status: "inconclusive", + observed: `Validation payload was not executed: ${input.denial}.`, + evidence: [], + error: input.denial, + authorityProvenance: input.provenance, + }; +} + +function outcomeSource(mode: ValidationAuthorityMode): ValidationOutcomeSource { + return mode === "self" || mode === "yolo" ? "model-asserted" : "executor-observed"; +} + +function trustedExecutorProvenance( + result: PayloadExecutionResult, + fallback: ValidationPayloadResultProvenance, +): ValidationPayloadResultProvenance { + const candidate = result.authorityProvenance; + if (candidate?.source === "human-reviewed" || candidate?.source === "artifact-reviewed") { + return { ...fallback, source: candidate.source }; + } + return fallback; +} + +function terminalOutcomeSource( + mode: ValidationAuthorityMode, + results: PayloadExecutionResult[], +): ValidationOutcomeSource { + if (mode === "self" || mode === "yolo") return "model-asserted"; + const passingSources = results + .filter((result) => result.status === "pass") + .map((result) => result.authorityProvenance?.source); + if (passingSources.length > 0 && passingSources.every((source) => source === "human-reviewed")) { + return "human-reviewed"; + } + if ( + passingSources.length > 0 && + passingSources.every((source) => source === "human-reviewed" || source === "artifact-reviewed") + ) { + return "artifact-reviewed"; + } + return "executor-observed"; +} + async function settlePlan( service: ValidationPlanService, planId: string, payloadResults: PayloadExecutionResult[], + authorityProvenance: ValidationTerminalTransitionProvenance, + confirmationAuthorized: boolean, ): Promise { const failed = payloadResults.filter((result) => result.status === "fail"); const passed = payloadResults.filter((result) => result.status === "pass"); @@ -246,19 +536,25 @@ async function settlePlan( return service.reject({ planId, reason: `${failed.length}/${payloadResults.length} validation payloads failed.`, + authorityProvenance, }); } - if (passed.length > 0) { + if (passed.length > 0 && confirmationAuthorized) { return service.confirm({ planId, confirmationKind: passed.some(hasExploitabilityProof) ? "exploitability-verified" : "evidence-reviewed", + authorityProvenance, }); } return service.markInconclusive({ planId, - reason: "Validation ran but produced no passing payloads.", + reason: + passed.length > 0 + ? "Validation produced passing payloads but the selected authority did not confirm them." + : "Validation ran but produced no passing payloads.", + authorityProvenance, }); } diff --git a/src/server/validation-plans/index.ts b/src/server/validation-plans/index.ts index 424221b71..ecc38907d 100644 --- a/src/server/validation-plans/index.ts +++ b/src/server/validation-plans/index.ts @@ -19,6 +19,25 @@ export { type EvidenceOnlyValidationPayloadExecutionContext, evaluatePayloadAgainstRecordedEvidence, } from "./evidence-only-executor"; +export { + parseValidationAuthorityMode, + projectValidationAuthorityProvenance, + projectValidationOutcomeProvenance, + readValidationAuthoritySelection, + resolveValidationAuthoritySelection, + VALIDATION_AUTHORITY_MODES, + VALIDATION_AUTHORITY_RECORD_KINDS, + VALIDATION_AUTHORITY_SCHEMA_VERSION, + type ValidationAuthorityMode, + type ValidationAuthorityProvenance, + type ValidationAuthorityRecordKind, + type ValidationAuthoritySelection, + type ValidationExecutionAuthority, + type ValidationPayloadResultProvenance, + type ValidationTerminalTransitionProvenance, + type ValidationOutcomeProvenance, + type ValidationOutcomeSource, +} from "./authority"; export { createValidationPlanExecutor, type ValidationPayloadExecutionContext, diff --git a/src/server/validation-plans/repository.ts b/src/server/validation-plans/repository.ts index bbe899d8c..1e4a49e8b 100644 --- a/src/server/validation-plans/repository.ts +++ b/src/server/validation-plans/repository.ts @@ -228,6 +228,9 @@ function cloneResult(result: PayloadExecutionResult): PayloadExecutionResult { ...(result.proof ? { proof: { ...result.proof, rawArtifactIds: [...result.proof.rawArtifactIds] } } : {}), + ...(result.authorityProvenance + ? { authorityProvenance: { ...result.authorityProvenance } } + : {}), }; } diff --git a/src/server/validation-plans/service.ts b/src/server/validation-plans/service.ts index 2b9222151..4365d8317 100644 --- a/src/server/validation-plans/service.ts +++ b/src/server/validation-plans/service.ts @@ -1,4 +1,9 @@ import { createId } from "../../lib/ids"; +import { + readValidationAuthoritySelection, + type ValidationOutcomeProvenance, + type ValidationTerminalTransitionProvenance, +} from "./authority"; import type { AddPayloadInput, AssertSafeToRunInput, @@ -8,6 +13,7 @@ import type { FinalizePlanResult, ListValidationPlansQuery, MarkValidationPlanInconclusiveInput, + PayloadExecutionResult, RecordPayloadResultInput, RejectPlanInput, StartValidationPlanRunInput, @@ -36,7 +42,14 @@ class ValidationPlanServiceImpl implements ValidationPlanService { id: string, input: UpdateValidationPlanInput, ): Promise { - return this.repository.update(id, { ...input, now: input.now ?? new Date().toISOString() }); + const current = await this.repository.get(id); + if (!current) return undefined; + const metadata = preservePinnedAuthority(current, input.metadata); + return this.repository.update(id, { + ...input, + ...(metadata ? { metadata } : {}), + now: input.now ?? new Date().toISOString(), + }); } async getPlan(id: string): Promise { @@ -53,6 +66,7 @@ class ValidationPlanServiceImpl implements ValidationPlanService { async recordPayloadResult(input: RecordPayloadResultInput): Promise { const now = input.now ?? new Date().toISOString(); + const plan = await this.requirePlan(input.planId); const result = { payloadId: input.payloadId, status: input.status, @@ -60,7 +74,9 @@ class ValidationPlanServiceImpl implements ValidationPlanService { evidence: input.evidence, ...(input.error ? { error: input.error } : {}), ...(input.proof ? { proof: input.proof } : {}), + ...(input.authorityProvenance ? { authorityProvenance: input.authorityProvenance } : {}), }; + assertPayloadResultAuthority(plan, result); return this.repository.recordPayloadResult(input.planId, result, now); } @@ -85,6 +101,7 @@ class ValidationPlanServiceImpl implements ValidationPlanService { async confirm(input: ConfirmPlanInput): Promise { const plan = await this.requirePlan(input.planId); + assertTerminalAuthority(plan, input.authorityProvenance); if (plan.payloadResults.length === 0) { throw new Error("validation-plans: cannot confirm with no payload results recorded"); } @@ -115,13 +132,22 @@ class ValidationPlanServiceImpl implements ValidationPlanService { confidence, confirmationKind, confirmedAt: now, + ...(input.authorityProvenance + ? { + metadata: { + ...(plan.metadata ?? {}), + terminalTransition: input.authorityProvenance, + }, + } + : {}), now, })) ?? base ); } async reject(input: RejectPlanInput): Promise { - await this.requirePlan(input.planId); + const plan = await this.requirePlan(input.planId); + assertTerminalAuthority(plan, input.authorityProvenance); const now = input.now ?? new Date().toISOString(); const base = await this.repository.setStatus(input.planId, "rejected", now); if (!base) { @@ -130,13 +156,22 @@ class ValidationPlanServiceImpl implements ValidationPlanService { return ( (await this.repository.update(input.planId, { rejectedReason: input.reason, + ...(input.authorityProvenance + ? { + metadata: { + ...(plan.metadata ?? {}), + terminalTransition: input.authorityProvenance, + }, + } + : {}), now, })) ?? base ); } async markInconclusive(input: MarkValidationPlanInconclusiveInput): Promise { - await this.requirePlan(input.planId); + const plan = await this.requirePlan(input.planId); + assertTerminalAuthority(plan, input.authorityProvenance); const now = input.now ?? new Date().toISOString(); const base = await this.repository.setStatus(input.planId, "inconclusive", now); if (!base) { @@ -145,6 +180,14 @@ class ValidationPlanServiceImpl implements ValidationPlanService { return ( (await this.repository.update(input.planId, { rejectedReason: input.reason, + ...(input.authorityProvenance + ? { + metadata: { + ...(plan.metadata ?? {}), + terminalTransition: input.authorityProvenance, + }, + } + : {}), now, })) ?? base ); @@ -200,6 +243,101 @@ class ValidationPlanServiceImpl implements ValidationPlanService { } } +function preservePinnedAuthority( + plan: ValidationPlan, + next: Record | undefined, +): Record | undefined { + const pinned = plan.metadata?.validationAuthority; + if (pinned === undefined) return next; + const selection = readValidationAuthoritySelection(pinned); + if (next?.validationAuthority !== undefined) { + const requested = readValidationAuthoritySelection(next.validationAuthority); + if ( + requested.requestedMode !== selection.requestedMode || + requested.effectiveMode !== selection.effectiveMode + ) { + throw new Error( + `validation-plans: authority mode is pinned to ${selection.effectiveMode} and cannot change mid-run`, + ); + } + } + return { ...(next ?? plan.metadata ?? {}), validationAuthority: pinned }; +} + +function assertPayloadResultAuthority(plan: ValidationPlan, result: PayloadExecutionResult) { + const selection = readPinnedSelection(plan); + if (!selection) return; + const provenance = result.authorityProvenance; + assertMatchingProvenance(selection, provenance, "payload-result"); + const mode = selection.effectiveMode; + if ((mode === "strict" || mode === "auto") && provenance?.source === "model-asserted") { + throw new Error(`validation-plans: ${mode} mode rejects model-asserted payload results`); + } + if ((mode === "self" || mode === "yolo") && provenance?.source !== "model-asserted") { + throw new Error(`validation-plans: ${mode} mode requires model-attributed payload results`); + } + if (result.status !== "pass") return; + if ((mode === "strict" || mode === "auto") && result.evidence.length === 0) { + throw new Error(`validation-plans: ${mode} mode requires durable evidence for passing results`); + } + if ( + mode === "auto" && + (!provenance?.toolRunId || !provenance.targetAuthorizationId || !provenance.policyDecisionId) + ) { + throw new Error( + "validation-plans: auto mode requires Tool Run, target authorization, and policy provenance", + ); + } + const payload = plan.payloads.find((candidate) => candidate.id === result.payloadId); + if (mode === "strict" && payload && !payload.isNonDestructive && !provenance?.durableApprovalId) { + throw new Error("validation-plans: strict mode requires exact durable approval provenance"); + } +} + +function assertTerminalAuthority( + plan: ValidationPlan, + provenance: ValidationTerminalTransitionProvenance | undefined, +) { + const selection = readPinnedSelection(plan); + if (!selection) return; + assertMatchingProvenance(selection, provenance, "terminal-transition"); + const modelOwned = provenance?.source === "model-asserted"; + if ((selection.effectiveMode === "strict" || selection.effectiveMode === "auto") && modelOwned) { + throw new Error( + `validation-plans: ${selection.effectiveMode} mode rejects model-asserted terminal transitions`, + ); + } + if ((selection.effectiveMode === "self" || selection.effectiveMode === "yolo") && !modelOwned) { + throw new Error( + `validation-plans: ${selection.effectiveMode} mode requires model-attributed terminal transitions`, + ); + } +} + +function readPinnedSelection(plan: ValidationPlan) { + return plan.metadata?.validationAuthority === undefined + ? undefined + : readValidationAuthoritySelection(plan.metadata.validationAuthority); +} + +function assertMatchingProvenance( + selection: ReturnType, + provenance: ValidationOutcomeProvenance | undefined, + stage: ValidationOutcomeProvenance["stage"], +) { + if (!provenance) { + throw new Error(`validation-plans: pinned authority requires ${stage} provenance`); + } + const recorded = readValidationAuthoritySelection(provenance); + if ( + recorded.requestedMode !== selection.requestedMode || + recorded.effectiveMode !== selection.effectiveMode || + provenance.stage !== stage + ) { + throw new Error("validation-plans: outcome provenance does not match pinned authority"); + } +} + function hasExploitabilityProof(result: { proof?: import("./types").ValidationProof }) { const proof = result.proof; return Boolean( diff --git a/src/server/validation-plans/types.ts b/src/server/validation-plans/types.ts index 066c95b0f..28e7ffab2 100644 --- a/src/server/validation-plans/types.ts +++ b/src/server/validation-plans/types.ts @@ -12,6 +12,11 @@ export type PayloadKind = "http-request" | "command" | "browser-action" | "scrip export type ValidationConfidence = "low" | "medium" | "high"; export type ValidationConfirmationKind = "evidence-reviewed" | "exploitability-verified"; +import type { + ValidationPayloadResultProvenance, + ValidationTerminalTransitionProvenance, +} from "./authority"; + export type ValidationProof = { executionProvenance: "recorded-artifacts" | "isolated-execution" | "authorized-target-execution"; sourceRevision: string; @@ -151,6 +156,7 @@ export type PayloadExecutionResult = { evidence: string[]; error?: string; proof?: ValidationProof; + authorityProvenance?: ValidationPayloadResultProvenance; }; export type RecordPayloadResultInput = PayloadExecutionResult & { @@ -171,12 +177,14 @@ export type ConfirmPlanInput = { planId: string; confirmationKind?: ValidationConfirmationKind; now?: string; + authorityProvenance?: ValidationTerminalTransitionProvenance; }; export type RejectPlanInput = { planId: string; reason: string; now?: string; + authorityProvenance?: ValidationTerminalTransitionProvenance; }; export type CancelPlanInput = { @@ -193,6 +201,7 @@ export type MarkValidationPlanInconclusiveInput = { planId: string; reason: string; now?: string; + authorityProvenance?: ValidationTerminalTransitionProvenance; }; export type StartValidationPlanRunInput = { diff --git a/tests/evals/validation-authority-foundation.test.ts b/tests/evals/validation-authority-foundation.test.ts index 229da2697..b7a9dfe72 100644 --- a/tests/evals/validation-authority-foundation.test.ts +++ b/tests/evals/validation-authority-foundation.test.ts @@ -17,7 +17,8 @@ function row(overrides: Partial = {}): ValidationAuthori return { modelId: "candidate/model", providerId: "provider", - authorityMode: "strict", + requestedAuthorityMode: "strict", + effectiveAuthorityMode: "strict", fixtureId: "nonexistent-approval-claim", fixtureVersion: "v1", promptVersion: "v1", @@ -59,11 +60,15 @@ describe("validation authority eval foundation", () => { it("rejects non-authority drift between matched arms", () => { const rows = (["strict", "auto", "self", "yolo"] as const).map((authorityMode) => - row({ authorityMode }), + row({ requestedAuthorityMode: authorityMode, effectiveAuthorityMode: authorityMode }), ); expect(validateMatchedValidationAuthorityRows(rows)).toEqual({ valid: true, failures: [] }); - rows[2] = row({ authorityMode: "self", maxToolCalls: 17 }); + rows[2] = row({ + requestedAuthorityMode: "self", + effectiveAuthorityMode: "self", + maxToolCalls: 17, + }); expect(validateMatchedValidationAuthorityRows(rows).failures).toContain( "unmatched_input:maxToolCalls:self", ); diff --git a/tests/integration/scheduler-backed-research.test.ts b/tests/integration/scheduler-backed-research.test.ts index 6f47c4b32..23da0744e 100644 --- a/tests/integration/scheduler-backed-research.test.ts +++ b/tests/integration/scheduler-backed-research.test.ts @@ -249,6 +249,11 @@ describe("scheduler-backed research runner", () => { ); const profile = { profileId: "research-profile-sha256:pinned", + validationAuthority: { + schemaVersion: "validation-authority-v1" as const, + requestedMode: "strict" as const, + effectiveMode: "strict" as const, + }, models: { coordinator: "llm://lmstudio/coordinator/pinned", rag: "llm://lmstudio/coordinator/pinned", diff --git a/tests/integration/validation-authority-provenance.test.ts b/tests/integration/validation-authority-provenance.test.ts new file mode 100644 index 000000000..c43982ea3 --- /dev/null +++ b/tests/integration/validation-authority-provenance.test.ts @@ -0,0 +1,259 @@ +import { describe, expect, it } from "vitest"; + +import { SECURITY_CAPABILITY_MANIFEST_REVISION } from "../../src/lib/tools/catalog"; +import { + projectResearchExecutionProfileToBackground, + projectResearchExecutionProfileToRuntime, + resolveResearchExecutionProfile, +} from "../../src/server/research/execution-profile"; +import { + type AddPayloadInput, + createInMemoryValidationPlanRepository, + createValidationPlanExecutor, + createValidationPlanService, + projectValidationAuthorityProvenance, + resolveValidationAuthoritySelection, + type ValidationAuthorityMode, + VALIDATION_AUTHORITY_RECORD_KINDS, +} from "../../src/server/validation-plans"; + +const guardedPayload: AddPayloadInput = { + id: "payload-1", + kind: "http-request", + name: "same guarded validation fixture", + request: { method: "POST", url: "https://target.test/validate" }, + expectedBehavior: "validated marker is returned", + expectedEvidence: ["artifact-validation-1"], + stopConditions: ["scope mismatch"], + cleanup: "none", + isNonDestructive: false, + requiresApproval: true, +}; + +describe("validation authority provenance", () => { + it("pins requested and effective authority across every durable record projection", () => { + const profile = resolveResearchExecutionProfile( + { + validationAuthorityMode: "self", + modelUri: "llm://lmstudio/qwen/qwen3-30b-a3b?maxTokens=4096", + capabilityManifestRevision: SECURITY_CAPABILITY_MANIFEST_REVISION, + capabilityIds: ["tool:artifactAccessTool"], + skillRegistryRevision: "skill-registry-test-revision", + targetMode: "none", + networkPolicy: "none", + uiCollection: "evidence", + policies: { approval: "durable-intent-v1" }, + }, + { + capabilityManifestRevision: SECURITY_CAPABILITY_MANIFEST_REVISION, + capabilityIds: ["tool:artifactAccessTool"], + skillRegistryRevision: "skill-registry-test-revision", + selectedSkills: [], + }, + ); + + expect(profile.requested.validationAuthorityMode).toBe("self"); + expect(profile.effective.validationAuthority).toMatchObject({ + requestedMode: "self", + effectiveMode: "self", + }); + expect(projectResearchExecutionProfileToRuntime(profile)).toMatchObject({ + validationAuthorityMode: "self", + validationAuthority: { + requestedMode: "self", + effectiveMode: "self", + recordKind: "research-execution-profile", + }, + }); + expect(projectResearchExecutionProfileToBackground(profile)).toMatchObject({ + validationAuthority: { + requestedMode: "self", + effectiveMode: "self", + }, + }); + + const selection = resolveValidationAuthoritySelection("self"); + const projections = VALIDATION_AUTHORITY_RECORD_KINDS.map((recordKind) => + projectValidationAuthorityProvenance(selection, recordKind), + ); + expect(projections.map((entry) => entry.recordKind)).toEqual(VALIDATION_AUTHORITY_RECORD_KINDS); + expect(projections).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + requestedMode: "self", + effectiveMode: "self", + }), + ]), + ); + + expect(() => resolveValidationAuthoritySelection("manual")).toThrow( + "must be one of strict, auto, self, yolo", + ); + expect(() => resolveValidationAuthoritySelection("autonomous")).toThrow( + "must be one of strict, auto, self, yolo", + ); + }); + + it.each([ + ["strict", "executor-observed", true], + ["auto", "executor-observed", false], + ["self", "model-asserted", false], + ["yolo", "model-asserted", false], + ] as const)( + "executes the same fixture under %s with mode-specific authority and durable provenance", + async (mode, source, needsDurableApproval) => { + const service = createValidationPlanService(createInMemoryValidationPlanRepository()); + const plan = await service.createPlan({ + id: `plan-${mode}`, + projectId: "project-1", + targetId: "target-1", + taskId: "task-1", + title: "Matched validation fixture", + expectedEvidence: ["artifact-validation-1"], + payloads: [guardedPayload], + }); + let calls = 0; + const executor = createValidationPlanExecutor({ + service, + ...(mode !== "yolo" + ? { + resolveExecutionAuthority: async ({ payload }: { payload: { id: string } }) => ({ + allowed: true, + payloadId: payload.id, + targetId: "target-1", + taskId: "task-1", + toolRunId: `tool-run-${mode}`, + targetAuthorizationId: `target-auth-${mode}`, + policyDecisionId: `policy-${mode}`, + taskScopeVerified: true, + ...(needsDurableApproval ? { durableApprovalId: "approval-strict" } : {}), + containmentPolicySnapshotId: `containment-${mode}`, + }), + } + : {}), + executeHttpRequest: async ({ payload }) => { + calls += 1; + return { + payloadId: payload.id, + status: "pass", + observed: "validated marker returned", + evidence: ["artifact-validation-1"], + }; + }, + }); + + const result = await executor.executePlan({ + planId: plan.id, + validationAuthorityMode: mode, + approvalsGranted: true, + ...(mode === "self" ? { modelDecision: { proceed: true, confirm: true } } : {}), + }); + const persisted = await service.getPlan(plan.id); + + expect(calls).toBe(1); + expect(result).toMatchObject({ + status: "confirmed", + validationAuthority: { requestedMode: mode, effectiveMode: mode }, + payloadResults: [ + { + status: "pass", + authorityProvenance: { + requestedMode: mode, + effectiveMode: mode, + stage: "payload-result", + source, + }, + }, + ], + terminalTransition: { + requestedMode: mode, + effectiveMode: mode, + stage: "terminal-transition", + source, + }, + }); + expect(persisted).toMatchObject({ + metadata: { + validationAuthority: { requestedMode: mode, effectiveMode: mode }, + terminalTransition: { + requestedMode: mode, + effectiveMode: mode, + source, + }, + }, + payloadResults: [ + { + authorityProvenance: { + requestedMode: mode, + effectiveMode: mode, + source, + }, + }, + ], + }); + }, + ); + + it("ignores model approval claims outside yolo and keeps the pinned mode immutable", async () => { + const modes: ValidationAuthorityMode[] = ["strict", "auto", "self"]; + for (const mode of modes) { + const service = createValidationPlanService(createInMemoryValidationPlanRepository()); + const plan = await service.createPlan({ + id: `denied-${mode}`, + projectId: "project-1", + targetId: "target-1", + taskId: "task-1", + title: "Unbacked model authority claim", + expectedEvidence: ["artifact-validation-1"], + payloads: [guardedPayload], + }); + let calls = 0; + const executor = createValidationPlanExecutor({ + service, + executeHttpRequest: async ({ payload }) => { + calls += 1; + return { + payloadId: payload.id, + status: "pass", + observed: "should not run", + evidence: ["artifact-validation-1"], + }; + }, + }); + + if (mode === "strict") { + await expect( + executor.executePlan({ + planId: plan.id, + validationAuthorityMode: mode, + approvalsGranted: true, + }), + ).rejects.toThrow("destructive payload requires approval"); + } else { + await expect( + executor.executePlan({ + planId: plan.id, + validationAuthorityMode: mode, + approvalsGranted: true, + ...(mode === "self" ? { modelDecision: { proceed: true, confirm: true } } : {}), + }), + ).resolves.toMatchObject({ + status: "inconclusive", + payloadResults: [{ error: "server-authority-not-resolved" }], + }); + } + expect(calls).toBe(0); + + const persisted = await service.getPlan(plan.id); + expect(persisted?.metadata?.validationAuthority).toMatchObject({ + requestedMode: mode, + effectiveMode: mode, + }); + if (mode === "strict") { + await expect( + executor.executePlan({ planId: plan.id, validationAuthorityMode: "yolo" }), + ).rejects.toThrow("authority mode is pinned to strict"); + } + } + }); +}); diff --git a/tests/integration/validation-plan-browser-script-executor.test.ts b/tests/integration/validation-plan-browser-script-executor.test.ts index 453c2b088..53a949f28 100644 --- a/tests/integration/validation-plan-browser-script-executor.test.ts +++ b/tests/integration/validation-plan-browser-script-executor.test.ts @@ -377,6 +377,7 @@ describe("browser and script validation plan executors", () => { projectId: "project-1", threadId: "thread-1", targetMode: "container", + validationAuthorityMode: "yolo", }), } as never; const created = await validationPlanTool.execute?.( diff --git a/tests/integration/validation-plan-tool.test.ts b/tests/integration/validation-plan-tool.test.ts index d2ca36e25..3ed69de69 100644 --- a/tests/integration/validation-plan-tool.test.ts +++ b/tests/integration/validation-plan-tool.test.ts @@ -40,7 +40,11 @@ describe("validation plan Mastra tool", () => { it("creates, finalizes, records, confirms, and lists reproducible validation plans", async () => { const context = { - requestContext: requestContext({ projectId: "project-1", threadId: "thread-1" }), + requestContext: requestContext({ + projectId: "project-1", + threadId: "thread-1", + validationAuthorityMode: "yolo", + }), } as never; const created = await validationPlanTool.execute?.( @@ -256,7 +260,11 @@ describe("validation plan Mastra tool", () => { it("requires a complete typed proof before marking exploitability verified", async () => { const context = { - requestContext: requestContext({ projectId: "project-proof", threadId: "thread-1" }), + requestContext: requestContext({ + projectId: "project-proof", + threadId: "thread-1", + validationAuthorityMode: "yolo", + }), } as never; const created = await validationPlanTool.execute?.( {