#100: Pin validation-authority provenance

evals/validation-authority-foundation.ts

@@ -1,8 +1,13 @@
+import {
+  parseValidationAuthorityMode,
+  VALIDATION_AUTHORITY_MODES,
+  type ValidationAuthorityMode,
+} from "../src/server/validation-plans/authority";
+
 export const VALIDATION_AUTHORITY_SCORER_VERSION = "validation-authority-deterministic-v1";
 export const VALIDATION_AUTHORITY_MANIFEST_VERSION = "validation-authority-matrix-v1";
 
-export const VALIDATION_AUTHORITY_MODES = ["strict", "auto", "self", "yolo"] as const;
-export type ValidationAuthorityMode = (typeof VALIDATION_AUTHORITY_MODES)[number];
+export { VALIDATION_AUTHORITY_MODES, type ValidationAuthorityMode };
 
 export const VALIDATION_AUTHORITY_CLAIM_PROVENANCE = [
   "executor-observed",
@@ -79,7 +84,8 @@ export type ValidationAuthorityMetrics = {
 export type ValidationAuthorityRow = {
   modelId: string;
   providerId: string;
-  authorityMode: ValidationAuthorityMode;
+  requestedAuthorityMode: ValidationAuthorityMode;
+  effectiveAuthorityMode: ValidationAuthorityMode;
   fixtureId: string;
   fixtureVersion: string;
   promptVersion: string;
@@ -114,6 +120,13 @@ export function validateValidationAuthorityManifest(manifest: ValidationAuthorit
   ) {
     failures.push("authority_modes_incomplete");
   }
+  for (const mode of manifest.authorityModes) {
+    try {
+      parseValidationAuthorityMode(mode, "eval manifest validation authority mode");
+    } catch {
+      failures.push(`invalid_authority_mode:${String(mode)}`);
+    }
+  }
   if (manifest.minimumRepeats < 3) failures.push("minimum_repeats_below_three");
   if (manifest.executionAdmission.status !== "blocked" || manifest.executionAdmission.blockedByIssue !== 100) {
     failures.push("execution_admission_must_remain_blocked_by_issue_100");
@@ -203,13 +216,34 @@ const MATCHED_FIELDS: Array<keyof ValidationAuthorityRow> = [
 export function validateMatchedValidationAuthorityRows(rows: ValidationAuthorityRow[]) {
   const failures: string[] = [];
   for (const mode of VALIDATION_AUTHORITY_MODES) {
-    if (!rows.some((row) => row.authorityMode === mode)) failures.push(`missing_mode:${mode}`);
+    if (!rows.some((row) => row.effectiveAuthorityMode === mode)) {
+      failures.push(`missing_mode:${mode}`);
+    }
+  }
+  for (const row of rows) {
+    try {
+      const requested = parseValidationAuthorityMode(
+        row.requestedAuthorityMode,
+        "eval row requested validation authority mode",
+      );
+      const effective = parseValidationAuthorityMode(
+        row.effectiveAuthorityMode,
+        "eval row effective validation authority mode",
+      );
+      if (requested !== effective) {
+        failures.push(`authority_mode_changed:${requested}:${effective}`);
+      }
+    } catch {
+      failures.push(`invalid_authority_mode:${String(row.effectiveAuthorityMode)}`);
+    }
   }
   const baseline = rows[0];
   if (baseline) {
     for (const row of rows.slice(1)) {
       for (const field of MATCHED_FIELDS) {
-        if (row[field] !== baseline[field]) failures.push(`unmatched_input:${field}:${row.authorityMode}`);
+        if (row[field] !== baseline[field]) {
+          failures.push(`unmatched_input:${field}:${row.effectiveAuthorityMode}`);
+        }
       }
     }
   }
@@ -219,7 +253,7 @@ export function validateMatchedValidationAuthorityRows(rows: ValidationAuthority
 export function summarizeValidationAuthorityRows(rows: ValidationAuthorityRow[], minimumRepeats = 3) {
   const groups = new Map<string, ValidationAuthorityRow[]>();
   for (const row of rows) {
-    const key = `${row.modelId}|${row.authorityMode}|${row.fixtureId}`;
+    const key = `${row.modelId}|${row.effectiveAuthorityMode}|${row.fixtureId}`;
     groups.set(key, [...(groups.get(key) ?? []), row]);
   }
   return [...groups.entries()].map(([key, group]) => {

src/mastra/tools/validation-plan.ts

@@ -12,7 +12,13 @@ import {
   createScriptValidationExecutors,
   createValidationPlanExecutor,
   getDefaultValidationPlansService,
+  projectValidationAuthorityProvenance,
+  projectValidationOutcomeProvenance,
+  readValidationAuthoritySelection,
+  resolveValidationAuthoritySelection,
   type PayloadExecutionResult,
+  type ValidationAuthoritySelection,
+  type ValidationPlanService,
   type ValidationPlanStatus,
 } from "../../server/validation-plans";
 import { readThreadTargetConfig } from "../../server/workspaces/target-mode";
@@ -165,6 +171,37 @@ const payloadResultSchema = z.object({
   evidence: z.array(z.string()),
   error: z.string().optional(),
   proof: validationProofInputSchema.optional(),
+  authorityProvenance: z
+    .object({
+      schemaVersion: z.literal("validation-authority-v1"),
+      requestedMode: z.enum(["strict", "auto", "self", "yolo"]),
+      effectiveMode: z.enum(["strict", "auto", "self", "yolo"]),
+      recordKind: z.literal("validation-plan"),
+      stage: z.literal("payload-result"),
+      source: z.enum([
+        "executor-observed",
+        "model-asserted",
+        "human-reviewed",
+        "artifact-reviewed",
+      ]),
+      toolRunId: z.string().optional(),
+      targetAuthorizationId: z.string().optional(),
+      policyDecisionId: z.string().optional(),
+      durableApprovalId: z.string().optional(),
+      containmentPolicySnapshotId: z.string().optional(),
+    })
+    .optional(),
+});
+const validationAuthoritySchema = z.object({
+  schemaVersion: z.literal("validation-authority-v1"),
+  requestedMode: z.enum(["strict", "auto", "self", "yolo"]),
+  effectiveMode: z.enum(["strict", "auto", "self", "yolo"]),
+  recordKind: z.literal("validation-plan"),
+});
+const terminalTransitionSchema = z.object({
+  ...validationAuthoritySchema.shape,
+  stage: z.literal("terminal-transition"),
+  source: z.enum(["executor-observed", "model-asserted", "human-reviewed", "artifact-reviewed"]),
 });
 const controlMappingSchema = z.object({
   framework: z.string(),
@@ -247,11 +284,13 @@ export const validationPlanTool = createTool({
     execution: z
       .object({
         planId: z.string(),
+        validationAuthority: validationAuthoritySchema,
         status: validationPlanStatusSchema,
         confirmationKind: validationConfirmationKindSchema.optional(),
         payloadResults: z.array(payloadResultSchema),
         summary: z.string(),
         proofNodeId: z.string().optional(),
+        terminalTransition: terminalTransitionSchema,
       })
       .optional(),
   }),
@@ -261,6 +300,9 @@ export const validationPlanTool = createTool({
       throw new Error("Validation plan updates require a projectId in the request context.");
     }
     const threadId = readContextString(context?.requestContext?.get?.("threadId"));
+    const validationAuthority = resolveValidationAuthoritySelection(
+      context?.requestContext?.get?.("validationAuthorityMode"),
+    );
     const service = getDefaultValidationPlansService();
 
     switch (input.action) {
@@ -285,9 +327,13 @@ export const validationPlanTool = createTool({
             ? { controlMappings: input.controlMappings.map(toControlMappingInput) }
             : {}),
           ...(input.artifacts ? { artifacts: input.artifacts } : {}),
-          ...(parseMetadataJson(input.metadataJson)
-            ? { metadata: parseMetadataJson(input.metadataJson) }
-            : {}),
+          metadata: {
+            ...(parseMetadataJson(input.metadataJson) ?? {}),
+            validationAuthority: projectValidationAuthorityProvenance(
+              validationAuthority,
+              "validation-plan",
+            ),
+          },
         });
         return {
           action: input.action,
@@ -331,7 +377,8 @@ export const validationPlanTool = createTool({
         const planId = requireField(input.planId, "planId", input.action);
         await service.assertSafeToRun({
           planId,
-          approvalsGranted: input.approvalsGranted === true,
+          approvalsGranted:
+            validationAuthority.effectiveMode === "yolo" && input.approvalsGranted === true,
         });
         const plan = await service.getPlan(planId);
         return {
@@ -350,6 +397,7 @@ export const validationPlanTool = createTool({
         });
         const execution = await executor.executePlan({
           planId,
+          validationAuthorityMode: validationAuthority.effectiveMode,
           approvalsGranted: input.approvalsGranted === true,
           ...(input.attackPathId ? { attackPathId: input.attackPathId } : {}),
         });
@@ -371,6 +419,7 @@ export const validationPlanTool = createTool({
         });
         const execution = await executor.executePlan({
           planId,
+          validationAuthorityMode: validationAuthority.effectiveMode,
           approvalsGranted: input.approvalsGranted === true,
           ...(input.attackPathId ? { attackPathId: input.attackPathId } : {}),
         });
@@ -392,6 +441,7 @@ export const validationPlanTool = createTool({
         });
         const execution = await executor.executePlan({
           planId,
+          validationAuthorityMode: validationAuthority.effectiveMode,
           approvalsGranted: input.approvalsGranted === true,
           ...(input.attackPathId ? { attackPathId: input.attackPathId } : {}),
         });
@@ -413,6 +463,7 @@ export const validationPlanTool = createTool({
         });
         const execution = await executor.executePlan({
           planId,
+          validationAuthorityMode: validationAuthority.effectiveMode,
           approvalsGranted: input.approvalsGranted === true,
           ...(input.attackPathId ? { attackPathId: input.attackPathId } : {}),
         });
@@ -426,7 +477,16 @@ export const validationPlanTool = createTool({
         };
       }
       case "record_result": {
-        const plan = await service.recordPayloadResult(toPayloadResult(input));
+        const planId = requireField(input.planId, "planId", input.action);
+        const selection = await requireModelValidationAuthority(service, planId);
+        const plan = await service.recordPayloadResult({
+          ...toPayloadResult(input),
+          authorityProvenance: projectValidationOutcomeProvenance({
+            selection,
+            stage: "payload-result",
+            source: "model-asserted",
+          }),
+        });
         if (!plan) {
           throw new Error(`validation-plans: plan ${input.planId ?? ""} was not found`);
         }
@@ -439,9 +499,15 @@ export const validationPlanTool = createTool({
       }
       case "confirm": {
         const planId = requireField(input.planId, "planId", input.action);
+        const selection = await requireModelValidationAuthority(service, planId);
         const plan = await service.confirm({
           planId,
           ...(input.confirmationKind ? { confirmationKind: input.confirmationKind } : {}),
+          authorityProvenance: projectValidationOutcomeProvenance({
+            selection,
+            stage: "terminal-transition",
+            source: "model-asserted",
+          }),
         });
         return {
           action: input.action,
@@ -884,6 +950,23 @@ function parseMetadataJson(value?: string) {
   return parsed as Record<string, unknown>;
 }
 
+async function requireModelValidationAuthority(
+  service: ValidationPlanService,
+  planId: string,
+): Promise<ValidationAuthoritySelection> {
+  const plan = await service.getPlan(planId);
+  if (!plan) {
+    throw new Error(`validation-plans: plan ${planId} was not found`);
+  }
+  const selection = readValidationAuthoritySelection(plan.metadata?.validationAuthority);
+  if (selection.effectiveMode !== "self" && selection.effectiveMode !== "yolo") {
+    throw new Error(
+      `validation-plans: ${selection.effectiveMode} mode requires executor-owned results and terminal transitions`,
+    );
+  }
+  return selection;
+}
+
 function clampLimit(value: number) {
   if (!Number.isFinite(value)) {
     return undefined;

src/server/chat/security-research-run.ts

@@ -3718,6 +3718,7 @@ export const runSecurityResearchAgent = async (
     ...(workspaceConfig ? { workspaceConfig } : {}),
     approvalMode,
     runtimeToolProfile,
+    validationAuthorityMode: input.metadata?.validationAuthorityMode,
     ...(Array.isArray(input.metadata?.commandAllowPatterns)
       ? {
           commandAllowPatterns: input.metadata.commandAllowPatterns.filter(

src/server/chat/security-research-runtime-context.ts

@@ -83,6 +83,7 @@ export const securityResearchRuntimeContextKeys = {
   skillRegistryRevision: "skillRegistryRevision",
   selectedSkillRefs: "selectedSkillRefs",
   runtimeSkillCapabilities: "runtimeSkillCapabilities",
+  validationAuthorityMode: "validationAuthorityMode",
 } as const;
 
 export function createSecurityResearchRequestContext(
@@ -133,6 +134,7 @@ export function buildSecurityResearchChatRequestContext(input: {
   evaluationArm?: string;
   terminalProtocolMode?: "legacy" | "typed" | "gated";
   terminalCleanupRequiredBeforeFinish?: boolean;
+  validationAuthorityMode?: unknown;
 }) {
   const keys = securityResearchRuntimeContextKeys;
   const resourceId = buildSecurityResearchMemoryResourceId(
@@ -174,6 +176,7 @@ export function buildSecurityResearchChatRequestContext(input: {
     [keys.evaluationArm, input.evaluationArm],
     [keys.terminalProtocolMode, input.terminalProtocolMode],
     [keys.terminalCleanupRequiredBeforeFinish, input.terminalCleanupRequiredBeforeFinish],
+    [keys.validationAuthorityMode, input.validationAuthorityMode],
     [keys.workspaceConfig, input.workspaceConfig],
     [keys.commandAllowPatterns, input.commandAllowPatterns],
     [keys.commandBlockPatterns, input.commandBlockPatterns],
@@ -269,6 +272,7 @@ export function buildSecurityResearchStageRequestContext(input: {
     [keys.skillRegistryRevision, input.executionProfile?.skillRegistryRevision],
     [keys.selectedSkillRefs, input.executionProfile?.selectedSkills],
     [keys.runtimeSkillCapabilities, input.executionProfile?.capabilityIds],
+    [keys.validationAuthorityMode, input.executionProfile?.validationAuthority.effectiveMode],
     [keys.modelUri, input.executionProfile?.models.coordinator],
     [keys.coordinatorModelUri, input.executionProfile?.models.coordinator],
     [keys.modelOverrides, input.executionProfile?.models],

src/server/chat/securityResearchTurn.ts

@@ -29,6 +29,7 @@ import { recoverResearchTurn, shouldRecoverResearchResult } from "../research/re
 import {
   projectResearchExecutionProfileToRuntime,
   researchExecutionProfileForensics,
+  researchExecutionProfileValidationAuthority,
   resolveResearchExecutionProfile,
 } from "../research/execution-profile";
 import { withResearchRunContext } from "../research/run-context";
@@ -39,6 +40,7 @@ import {
   type ResearchTurnLedgerHandle,
 } from "../research/turn-ledger";
 import { createTargetAuthorization, inferTargetKind, upsertProjectTarget } from "../targets";
+import { projectValidationAuthorityProvenance } from "../validation-plans/authority";
 import type { ThreadWorkspaceResolution, ThreadWorkspaceVolumeStatus } from "../workspaces";
 import { readThreadTargetConfig } from "../workspaces/target-mode";
 import type { BrowserMcpExecutionLedgerStore } from "./browser-mcp-execution-ledger";
@@ -223,6 +225,7 @@ export const runSecurityResearchTurn = async (
   const targetConfig = readTargetConfigForExecutionProfile(agentInput.metadata);
   const executionProfile = resolveResearchExecutionProfile(
     {
+      validationAuthorityMode: agentInput.metadata?.validationAuthorityMode,
       modelUri: readOptionalString(agentInput.metadata?.modelUri) ?? undefined,
       runtimeModelUri: readOptionalString(agentInput.metadata?.runtimeModelUri) ?? undefined,
       modelOverrides: readModelOverrides(agentInput.metadata?.modelOverrides),
@@ -572,6 +575,7 @@ async function recordRecoveryUsageEvent(input: {
     attributes: {
       researchRunId: input.ledger.researchRunId,
       researchTurnLedgerId: input.ledger.turnLedgerId,
+      ...usageEventValidationAuthority(input.ledger),
       recoveryAttemptIndex: input.attemptIndex,
       failed: true,
       promptTokens: readOptionalNumber(usage.promptTokens),
@@ -637,6 +641,7 @@ async function finalizeFailedLedger(
       attributes: {
         researchRunId: ledger.researchRunId,
         researchTurnLedgerId: ledger.turnLedgerId,
+        ...usageEventValidationAuthority(ledger),
         modelStepIndex: 0,
         failed: true,
         promptTokens: readOptionalNumber(errorUsage.promptTokens),
@@ -819,6 +824,7 @@ async function recordResearchTurnUsageEvents(input: {
       attributes: {
         researchRunId: input.ledger.researchRunId,
         researchTurnLedgerId: input.ledger.turnLedgerId,
+        ...usageEventValidationAuthority(input.ledger),
         modelStepIndex: step.stepIndex,
         provider: usage.provider,
         model: usage.model,
@@ -838,6 +844,16 @@ async function recordResearchTurnUsageEvents(input: {
   return ids;
 }
 
+function usageEventValidationAuthority(ledger: ResearchTurnLedgerHandle) {
+  if (!ledger.executionProfile) return {};
+  return {
+    validationAuthority: projectValidationAuthorityProvenance(
+      researchExecutionProfileValidationAuthority(ledger.executionProfile),
+      "usage-event",
+    ),
+  };
+}
+
 function readStringArray(value: unknown) {
   return Array.isArray(value)
     ? value.filter((item): item is string => typeof item === "string" && item.length > 0)