From 24be3c4c41095b4bf22b11e72ec1b257edaae85e Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Wed, 26 Aug 2026 16:40:50 -0400 Subject: [PATCH] Add native harness study admission contract --- evals/native-harness-comparison.ts | 388 ++++++++++++++++++ tests/evals/native-harness-comparison.test.ts | 168 ++++++++ 2 files changed, 556 insertions(+) create mode 100644 evals/native-harness-comparison.ts create mode 100644 tests/evals/native-harness-comparison.test.ts diff --git a/evals/native-harness-comparison.ts b/evals/native-harness-comparison.ts new file mode 100644 index 000000000..27a55a6c0 --- /dev/null +++ b/evals/native-harness-comparison.ts @@ -0,0 +1,388 @@ +export const NATIVE_HARNESS_COMPARISON_VERSION = "native-harness-comparison-v1"; + +export type NativeHarnessArmClass = + | "native-anchor" + | "same-model-bridge" + | "capability-expansion"; + +export type NativeHarnessTerminalClass = + | "not_admitted_config" + | "not_admitted_funding" + | "harness_isolation_failure" + | "target_infrastructure_failure" + | "provider_infrastructure_failure" + | "harness_protocol_failure" + | "model_substitution" + | "safeguard_refusal" + | "approval_denied_or_waiting" + | "budget_or_timeout" + | "terminal_contract_failure" + | "abstained_or_blocked" + | "valid_incorrect" + | "valid_success"; + +export type NativeHarnessComparisonManifest = { + schemaVersion: typeof NATIVE_HARNESS_COMPARISON_VERSION; + arm: { + id: string; + class: NativeHarnessArmClass; + prerequisiteArmIds: string[]; + nativeAnchorId?: string; + }; + study: { + mode: "organic-hunt"; + cohortRevision: string; + promptSha256: string; + terminalSchemaSha256: string; + scorerRevision: string; + }; + harness: { + id: string; + version: string; + revision?: string; + executableSha256: string; + argv: string[]; + configSha256: string; + nativeFeatures: string[]; + }; + model: { + requestedProvider: string; + requestedModel: string; + requestedEffort: string; + observedProvider: string; + observedModel: string; + observedEffort: string; + serviceTier?: string; + fallbackChain: string[]; + }; + target: { + taskId: string; + imageDigest: string; + freshInstanceId: string; + authorizationId: string; + networkProfile: "approved-targets"; + secretSeedSha256: string; + }; + budgets: { + activeWorkingMs: number; + wallMs: number; + candidateCostUsd: number; + actionLimit: number; + nativeTurnOrStepLimit?: number; + contextTokens: number; + maxOutputTokens: number; + }; + provenance: { + providerAccountProject: string; + operatorInterventions: string[]; + approvalDecisions: string[]; + ancillaryModelRoutes: Array<{ + purpose: string; + provider: string; + model: string; + attributable: boolean; + }>; + }; + canary: { + userGlobalConfigurationDisabled: boolean; + exactRouteFunded: boolean; + directProviderRouteObserved: boolean; + requestedEffortObserved: boolean; + structuredTerminalSucceeded: boolean; + terminalFailureDistinguishable: boolean; + usageTokens: number; + usageProvider: string; + usageModel: string; + costSource: "provider" | "catalog" | "harness-estimate" | "unavailable"; + outOfScopeWriteDenied: boolean; + outOfScopeDestinationDenied: boolean; + sandboxAvailable: boolean; + ancillaryRoutesDisabledOrAttributed: boolean; + }; +}; + +export type NativeHarnessAdmissionRecord = { + armId: string; + admitted: boolean; + primaryClass?: Extract< + NativeHarnessTerminalClass, + "not_admitted_config" | "not_admitted_funding" | "harness_isolation_failure" + >; + failures: string[]; + observedRoute?: { + provider: string; + model: string; + effort: string; + }; +}; + +export type NativeHarnessOutcomeVector = { + terminalClass: NativeHarnessTerminalClass; + oracleEligible: boolean; + validatedFindingCount: number; + evidenceArtifactCount: number; + reproducibleImpactCount: number; + activeWorkingMs: number; + wallElapsedMs: number; + approvalWaitMs: number; + candidateCostUsd: number | "unavailable"; + inputTokens: number; + outputTokens: number; + reasoningTokens: number | "unavailable"; + targetActionCount: number; + operatorInterventionCount: number; +}; + +const FORBIDDEN_CLI_FLAGS = new Set([ + "--yolo", + "yolo", + "--dangerously-skip-permissions", + "dangerously-skip-permissions", + "--full-auto", + "full-auto", + "--bypass-permissions", + "bypass-permissions", +]); + +const COHORT_FIELDS = [ + "mode", + "cohortRevision", + "promptSha256", + "terminalSchemaSha256", + "scorerRevision", +] as const; + +const BUDGET_FIELDS = [ + "activeWorkingMs", + "wallMs", + "candidateCostUsd", + "actionLimit", + "contextTokens", + "maxOutputTokens", +] as const; + +export function compareNativeHarnessCohortFreeze( + baseline: NativeHarnessComparisonManifest, + candidate: NativeHarnessComparisonManifest, +) { + const mismatches = [ + ...COHORT_FIELDS.flatMap((field) => + baseline.study[field] === candidate.study[field] + ? [] + : [`study.${field}`], + ), + ...BUDGET_FIELDS.flatMap((field) => + baseline.budgets[field] === candidate.budgets[field] + ? [] + : [`budgets.${field}`], + ), + ...(baseline.target.taskId === candidate.target.taskId + ? [] + : ["target.taskId"]), + ...(baseline.target.imageDigest === candidate.target.imageDigest + ? [] + : ["target.imageDigest"]), + ...(baseline.target.networkProfile === candidate.target.networkProfile + ? [] + : ["target.networkProfile"]), + ]; + return { matched: mismatches.length === 0, mismatches }; +} + +export function admitNativeHarnessManifest( + manifest: NativeHarnessComparisonManifest, + admittedArms: readonly NativeHarnessAdmissionRecord[] = [], +): NativeHarnessAdmissionRecord { + const configFailures = validateConfig(manifest); + const fundingFailures = validateFunding(manifest); + const isolationFailures = validateIsolation(manifest); + const sequenceFailures = validateSequence(manifest, admittedArms); + const failures = [ + ...configFailures, + ...fundingFailures, + ...isolationFailures, + ...sequenceFailures, + ]; + + const primaryClass = isolationFailures.length + ? "harness_isolation_failure" + : fundingFailures.length + ? "not_admitted_funding" + : configFailures.length || sequenceFailures.length + ? "not_admitted_config" + : undefined; + + return { + armId: manifest.arm.id, + admitted: failures.length === 0, + ...(primaryClass ? { primaryClass } : {}), + failures, + ...(failures.length === 0 + ? { + observedRoute: { + provider: manifest.model.observedProvider, + model: manifest.model.observedModel, + effort: manifest.model.observedEffort, + }, + } + : {}), + }; +} + +function validateConfig(manifest: NativeHarnessComparisonManifest) { + const failures: string[] = []; + if (manifest.schemaVersion !== NATIVE_HARNESS_COMPARISON_VERSION) { + failures.push("schema_version_unsupported"); + } + if (manifest.study.mode !== "organic-hunt") + failures.push("study_mode_not_organic_hunt"); + for (const [name, value] of requiredStrings(manifest)) { + if (!value.trim()) failures.push(`${name}_missing`); + } + if ( + !Array.isArray(manifest.harness.argv) || + manifest.harness.argv.length === 0 + ) { + failures.push("harness_argv_missing"); + } + if ( + manifest.harness.argv.some((arg) => + FORBIDDEN_CLI_FLAGS.has(arg.trim().toLowerCase()), + ) + ) { + failures.push("unsafe_cli_permission_flag_forbidden"); + } + if (manifest.model.requestedProvider !== manifest.model.observedProvider) { + failures.push("observed_provider_mismatch"); + } + if (manifest.model.requestedModel !== manifest.model.observedModel) { + failures.push("observed_model_mismatch"); + } + if (manifest.model.requestedEffort !== manifest.model.observedEffort) { + failures.push("observed_effort_mismatch"); + } + if (!manifest.canary.requestedEffortObserved) + failures.push("effort_not_observed_in_metadata"); + if (manifest.model.fallbackChain.length > 0) + failures.push("fallback_chain_not_empty"); + if (!manifest.canary.structuredTerminalSucceeded) + failures.push("terminal_canary_failed"); + if (!manifest.canary.terminalFailureDistinguishable) { + failures.push("terminal_failure_not_distinguishable"); + } + if (!manifest.canary.userGlobalConfigurationDisabled) { + failures.push("user_global_configuration_not_disabled"); + } + if (manifest.budgets.activeWorkingMs <= 0) + failures.push("active_working_budget_invalid"); + if (manifest.budgets.wallMs <= 0) failures.push("wall_budget_invalid"); + if (manifest.budgets.candidateCostUsd <= 0) + failures.push("cost_budget_invalid"); + if (manifest.budgets.actionLimit <= 0) failures.push("action_budget_invalid"); + if (manifest.budgets.contextTokens <= 0) + failures.push("context_budget_invalid"); + if (manifest.budgets.maxOutputTokens <= 0) + failures.push("output_budget_invalid"); + return failures; +} + +function validateFunding(manifest: NativeHarnessComparisonManifest) { + const failures: string[] = []; + if (!manifest.canary.exactRouteFunded) + failures.push("exact_route_not_funded"); + if (!manifest.canary.directProviderRouteObserved) + failures.push("direct_provider_route_not_observed"); + if (manifest.canary.usageTokens <= 0) failures.push("positive_usage_missing"); + if (manifest.canary.usageProvider !== manifest.model.observedProvider) { + failures.push("usage_provider_mismatch"); + } + if (manifest.canary.usageModel !== manifest.model.observedModel) { + failures.push("usage_model_mismatch"); + } + return failures; +} + +function validateIsolation(manifest: NativeHarnessComparisonManifest) { + const failures: string[] = []; + if (!manifest.canary.sandboxAvailable) failures.push("sandbox_unavailable"); + if (!manifest.canary.outOfScopeWriteDenied) + failures.push("out_of_scope_write_not_denied"); + if (!manifest.canary.outOfScopeDestinationDenied) { + failures.push("out_of_scope_destination_not_denied"); + } + if (!manifest.canary.ancillaryRoutesDisabledOrAttributed) { + failures.push("ancillary_routes_not_attributed"); + } + if ( + manifest.provenance.ancillaryModelRoutes.some( + (route) => !route.attributable, + ) + ) { + failures.push("unattributed_ancillary_model_route"); + } + return failures; +} + +function validateSequence( + manifest: NativeHarnessComparisonManifest, + admittedArms: readonly NativeHarnessAdmissionRecord[], +) { + const failures: string[] = []; + const admittedById = new Map( + admittedArms + .filter((record) => record.admitted) + .map((record) => [record.armId, record]), + ); + for (const prerequisite of manifest.arm.prerequisiteArmIds) { + if (!admittedById.has(prerequisite)) + failures.push(`prerequisite_not_admitted:${prerequisite}`); + } + if (manifest.arm.class === "same-model-bridge") { + if (!manifest.arm.nativeAnchorId) { + failures.push("bridge_native_anchor_missing"); + } else { + const anchor = admittedById.get(manifest.arm.nativeAnchorId); + if (!anchor) { + failures.push( + `bridge_native_anchor_not_admitted:${manifest.arm.nativeAnchorId}`, + ); + } else if ( + !anchor.observedRoute || + anchor.observedRoute.provider !== manifest.model.observedProvider || + anchor.observedRoute.model !== manifest.model.observedModel || + anchor.observedRoute.effort !== manifest.model.observedEffort + ) { + failures.push("bridge_observed_route_differs_from_native_anchor"); + } + } + } + return failures; +} + +function requiredStrings( + manifest: NativeHarnessComparisonManifest, +): Array<[string, string]> { + return [ + ["arm_id", manifest.arm.id], + ["cohort_revision", manifest.study.cohortRevision], + ["prompt_sha256", manifest.study.promptSha256], + ["terminal_schema_sha256", manifest.study.terminalSchemaSha256], + ["scorer_revision", manifest.study.scorerRevision], + ["harness_id", manifest.harness.id], + ["harness_version", manifest.harness.version], + ["harness_executable_sha256", manifest.harness.executableSha256], + ["harness_config_sha256", manifest.harness.configSha256], + ["requested_provider", manifest.model.requestedProvider], + ["requested_model", manifest.model.requestedModel], + ["requested_effort", manifest.model.requestedEffort], + ["observed_provider", manifest.model.observedProvider], + ["observed_model", manifest.model.observedModel], + ["observed_effort", manifest.model.observedEffort], + ["target_task_id", manifest.target.taskId], + ["target_image_digest", manifest.target.imageDigest], + ["target_fresh_instance_id", manifest.target.freshInstanceId], + ["target_authorization_id", manifest.target.authorizationId], + ["target_secret_seed_sha256", manifest.target.secretSeedSha256], + ["provider_account_project", manifest.provenance.providerAccountProject], + ]; +} diff --git a/tests/evals/native-harness-comparison.test.ts b/tests/evals/native-harness-comparison.test.ts new file mode 100644 index 000000000..c602664be --- /dev/null +++ b/tests/evals/native-harness-comparison.test.ts @@ -0,0 +1,168 @@ +import { describe, expect, it } from "vitest"; + +import { + admitNativeHarnessManifest, + compareNativeHarnessCohortFreeze, + type NativeHarnessComparisonManifest, +} from "../../evals/native-harness-comparison"; + +function manifest(id = "native-anchor"): NativeHarnessComparisonManifest { + return { + schemaVersion: "native-harness-comparison-v1", + arm: { id, class: "native-anchor", prerequisiteArmIds: [] }, + study: { + mode: "organic-hunt", + cohortRevision: "cohort-revision", + promptSha256: "prompt-sha", + terminalSchemaSha256: "terminal-sha", + scorerRevision: "scorer-revision", + }, + harness: { + id: "native-cli", + version: "1.0.0", + executableSha256: "executable-sha", + argv: ["native-cli", "exec", "--json"], + configSha256: "config-sha", + nativeFeatures: ["shell", "structured-output"], + }, + model: { + requestedProvider: "direct-provider", + requestedModel: "candidate-model", + requestedEffort: "low", + observedProvider: "direct-provider", + observedModel: "candidate-model", + observedEffort: "low", + fallbackChain: [], + }, + target: { + taskId: "public-task", + imageDigest: "image-digest", + freshInstanceId: "fresh-instance", + authorizationId: "authorization-row", + networkProfile: "approved-targets", + secretSeedSha256: "evaluator-secret-digest", + }, + budgets: { + activeWorkingMs: 600_000, + wallMs: 900_000, + candidateCostUsd: 10, + actionLimit: 100, + contextTokens: 65_536, + maxOutputTokens: 16_384, + }, + provenance: { + providerAccountProject: "account-project-hash", + operatorInterventions: [], + approvalDecisions: [], + ancillaryModelRoutes: [], + }, + canary: { + userGlobalConfigurationDisabled: true, + exactRouteFunded: true, + directProviderRouteObserved: true, + requestedEffortObserved: true, + structuredTerminalSucceeded: true, + terminalFailureDistinguishable: true, + usageTokens: 3, + usageProvider: "direct-provider", + usageModel: "candidate-model", + costSource: "unavailable", + outOfScopeWriteDenied: true, + outOfScopeDestinationDenied: true, + sandboxAvailable: true, + ancillaryRoutesDisabledOrAttributed: true, + }, + }; +} + +describe("native harness comparison admission", () => { + it("admits an observed, funded, sandboxed native configuration", () => { + expect(admitNativeHarnessManifest(manifest())).toEqual({ + armId: "native-anchor", + admitted: true, + failures: [], + observedRoute: { + provider: "direct-provider", + model: "candidate-model", + effort: "low", + }, + }); + }); + + it("fails closed on substituted models and unsafe permission flags", () => { + const candidate = manifest(); + candidate.model.observedModel = "fallback-model"; + candidate.canary.usageModel = "fallback-model"; + candidate.harness.argv.push("--dangerously-skip-permissions"); + + expect(admitNativeHarnessManifest(candidate)).toMatchObject({ + admitted: false, + primaryClass: "not_admitted_config", + failures: expect.arrayContaining([ + "observed_model_mismatch", + "unsafe_cli_permission_flag_forbidden", + ]), + }); + }); + + it("keeps funding and confinement failures out of model-quality rows", () => { + const candidate = manifest(); + candidate.canary.exactRouteFunded = false; + candidate.canary.outOfScopeDestinationDenied = false; + + expect(admitNativeHarnessManifest(candidate)).toMatchObject({ + admitted: false, + primaryClass: "harness_isolation_failure", + failures: expect.arrayContaining([ + "exact_route_not_funded", + "out_of_scope_destination_not_denied", + ]), + }); + }); + + it("admits a same-model bridge only after its matching native anchor", () => { + const anchor = admitNativeHarnessManifest(manifest("anchor")); + const bridge = manifest("bridge"); + bridge.arm = { + id: "bridge", + class: "same-model-bridge", + prerequisiteArmIds: ["anchor"], + nativeAnchorId: "anchor", + }; + + expect(admitNativeHarnessManifest(bridge)).toMatchObject({ + admitted: false, + failures: expect.arrayContaining([ + "prerequisite_not_admitted:anchor", + "bridge_native_anchor_not_admitted:anchor", + ]), + }); + expect(admitNativeHarnessManifest(bridge, [anchor])).toMatchObject({ + admitted: true, + }); + + bridge.model.observedModel = "different-model"; + bridge.model.requestedModel = "different-model"; + bridge.canary.usageModel = "different-model"; + expect(admitNativeHarnessManifest(bridge, [anchor]).failures).toContain( + "bridge_observed_route_differs_from_native_anchor", + ); + }); + + it("detects cohort and conjunctive-budget drift without equating fresh instances", () => { + const baseline = manifest(); + const candidate = manifest(); + candidate.target.freshInstanceId = "another-fresh-instance"; + expect(compareNativeHarnessCohortFreeze(baseline, candidate)).toEqual({ + matched: true, + mismatches: [], + }); + + candidate.budgets.actionLimit -= 1; + candidate.study.promptSha256 = "changed-prompt"; + expect(compareNativeHarnessCohortFreeze(baseline, candidate)).toEqual({ + matched: false, + mismatches: ["study.promptSha256", "budgets.actionLimit"], + }); + }); +});