#132: Define harness-study admission and attribution

evals/native-harness-comparison.ts

@@ -0,0 +1,388 @@
+export const NATIVE_HARNESS_COMPARISON_VERSION = "native-harness-comparison-v1";
+
+export type NativeHarnessArmClass =
+	| "native-anchor"
+	| "same-model-bridge"
+	| "capability-expansion";
+
+export type NativeHarnessTerminalClass =
+	| "not_admitted_config"
+	| "not_admitted_funding"
+	| "harness_isolation_failure"
+	| "target_infrastructure_failure"
+	| "provider_infrastructure_failure"
+	| "harness_protocol_failure"
+	| "model_substitution"
+	| "safeguard_refusal"
+	| "approval_denied_or_waiting"
+	| "budget_or_timeout"
+	| "terminal_contract_failure"
+	| "abstained_or_blocked"
+	| "valid_incorrect"
+	| "valid_success";
+
+export type NativeHarnessComparisonManifest = {
+	schemaVersion: typeof NATIVE_HARNESS_COMPARISON_VERSION;
+	arm: {
+		id: string;
+		class: NativeHarnessArmClass;
+		prerequisiteArmIds: string[];
+		nativeAnchorId?: string;
+	};
+	study: {
+		mode: "organic-hunt";
+		cohortRevision: string;
+		promptSha256: string;
+		terminalSchemaSha256: string;
+		scorerRevision: string;
+	};
+	harness: {
+		id: string;
+		version: string;
+		revision?: string;
+		executableSha256: string;
+		argv: string[];
+		configSha256: string;
+		nativeFeatures: string[];
+	};
+	model: {
+		requestedProvider: string;
+		requestedModel: string;
+		requestedEffort: string;
+		observedProvider: string;
+		observedModel: string;
+		observedEffort: string;
+		serviceTier?: string;
+		fallbackChain: string[];
+	};
+	target: {
+		taskId: string;
+		imageDigest: string;
+		freshInstanceId: string;
+		authorizationId: string;
+		networkProfile: "approved-targets";
+		secretSeedSha256: string;
+	};
+	budgets: {
+		activeWorkingMs: number;
+		wallMs: number;
+		candidateCostUsd: number;
+		actionLimit: number;
+		nativeTurnOrStepLimit?: number;
+		contextTokens: number;
+		maxOutputTokens: number;
+	};
+	provenance: {
+		providerAccountProject: string;
+		operatorInterventions: string[];
+		approvalDecisions: string[];
+		ancillaryModelRoutes: Array<{
+			purpose: string;
+			provider: string;
+			model: string;
+			attributable: boolean;
+		}>;
+	};
+	canary: {
+		userGlobalConfigurationDisabled: boolean;
+		exactRouteFunded: boolean;
+		directProviderRouteObserved: boolean;
+		requestedEffortObserved: boolean;
+		structuredTerminalSucceeded: boolean;
+		terminalFailureDistinguishable: boolean;
+		usageTokens: number;
+		usageProvider: string;
+		usageModel: string;
+		costSource: "provider" | "catalog" | "harness-estimate" | "unavailable";
+		outOfScopeWriteDenied: boolean;
+		outOfScopeDestinationDenied: boolean;
+		sandboxAvailable: boolean;
+		ancillaryRoutesDisabledOrAttributed: boolean;
+	};
+};
+
+export type NativeHarnessAdmissionRecord = {
+	armId: string;
+	admitted: boolean;
+	primaryClass?: Extract<
+		NativeHarnessTerminalClass,
+		"not_admitted_config" | "not_admitted_funding" | "harness_isolation_failure"
+	>;
+	failures: string[];
+	observedRoute?: {
+		provider: string;
+		model: string;
+		effort: string;
+	};
+};
+
+export type NativeHarnessOutcomeVector = {
+	terminalClass: NativeHarnessTerminalClass;
+	oracleEligible: boolean;
+	validatedFindingCount: number;
+	evidenceArtifactCount: number;
+	reproducibleImpactCount: number;
+	activeWorkingMs: number;
+	wallElapsedMs: number;
+	approvalWaitMs: number;
+	candidateCostUsd: number | "unavailable";
+	inputTokens: number;
+	outputTokens: number;
+	reasoningTokens: number | "unavailable";
+	targetActionCount: number;
+	operatorInterventionCount: number;
+};
+
+const FORBIDDEN_CLI_FLAGS = new Set([
+	"--yolo",
+	"yolo",
+	"--dangerously-skip-permissions",
+	"dangerously-skip-permissions",
+	"--full-auto",
+	"full-auto",
+	"--bypass-permissions",
+	"bypass-permissions",
+]);
+
+const COHORT_FIELDS = [
+	"mode",
+	"cohortRevision",
+	"promptSha256",
+	"terminalSchemaSha256",
+	"scorerRevision",
+] as const;
+
+const BUDGET_FIELDS = [
+	"activeWorkingMs",
+	"wallMs",
+	"candidateCostUsd",
+	"actionLimit",
+	"contextTokens",
+	"maxOutputTokens",
+] as const;
+
+export function compareNativeHarnessCohortFreeze(
+	baseline: NativeHarnessComparisonManifest,
+	candidate: NativeHarnessComparisonManifest,
+) {
+	const mismatches = [
+		...COHORT_FIELDS.flatMap((field) =>
+			baseline.study[field] === candidate.study[field]
+				? []
+				: [`study.${field}`],
+		),
+		...BUDGET_FIELDS.flatMap((field) =>
+			baseline.budgets[field] === candidate.budgets[field]
+				? []
+				: [`budgets.${field}`],
+		),
+		...(baseline.target.taskId === candidate.target.taskId
+			? []
+			: ["target.taskId"]),
+		...(baseline.target.imageDigest === candidate.target.imageDigest
+			? []
+			: ["target.imageDigest"]),
+		...(baseline.target.networkProfile === candidate.target.networkProfile
+			? []
+			: ["target.networkProfile"]),
+	];
+	return { matched: mismatches.length === 0, mismatches };
+}
+
+export function admitNativeHarnessManifest(
+	manifest: NativeHarnessComparisonManifest,
+	admittedArms: readonly NativeHarnessAdmissionRecord[] = [],
+): NativeHarnessAdmissionRecord {
+	const configFailures = validateConfig(manifest);
+	const fundingFailures = validateFunding(manifest);
+	const isolationFailures = validateIsolation(manifest);
+	const sequenceFailures = validateSequence(manifest, admittedArms);
+	const failures = [
+		...configFailures,
+		...fundingFailures,
+		...isolationFailures,
+		...sequenceFailures,
+	];
+
+	const primaryClass = isolationFailures.length
+		? "harness_isolation_failure"
+		: fundingFailures.length
+			? "not_admitted_funding"
+			: configFailures.length || sequenceFailures.length
+				? "not_admitted_config"
+				: undefined;
+
+	return {
+		armId: manifest.arm.id,
+		admitted: failures.length === 0,
+		...(primaryClass ? { primaryClass } : {}),
+		failures,
+		...(failures.length === 0
+			? {
+					observedRoute: {
+						provider: manifest.model.observedProvider,
+						model: manifest.model.observedModel,
+						effort: manifest.model.observedEffort,
+					},
+				}
+			: {}),
+	};
+}
+
+function validateConfig(manifest: NativeHarnessComparisonManifest) {
+	const failures: string[] = [];
+	if (manifest.schemaVersion !== NATIVE_HARNESS_COMPARISON_VERSION) {
+		failures.push("schema_version_unsupported");
+	}
+	if (manifest.study.mode !== "organic-hunt")
+		failures.push("study_mode_not_organic_hunt");
+	for (const [name, value] of requiredStrings(manifest)) {
+		if (!value.trim()) failures.push(`${name}_missing`);
+	}
+	if (
+		!Array.isArray(manifest.harness.argv) ||
+		manifest.harness.argv.length === 0
+	) {
+		failures.push("harness_argv_missing");
+	}
+	if (
+		manifest.harness.argv.some((arg) =>
+			FORBIDDEN_CLI_FLAGS.has(arg.trim().toLowerCase()),
+		)
+	) {
+		failures.push("unsafe_cli_permission_flag_forbidden");
+	}
+	if (manifest.model.requestedProvider !== manifest.model.observedProvider) {
+		failures.push("observed_provider_mismatch");
+	}
+	if (manifest.model.requestedModel !== manifest.model.observedModel) {
+		failures.push("observed_model_mismatch");
+	}
+	if (manifest.model.requestedEffort !== manifest.model.observedEffort) {
+		failures.push("observed_effort_mismatch");
+	}
+	if (!manifest.canary.requestedEffortObserved)
+		failures.push("effort_not_observed_in_metadata");
+	if (manifest.model.fallbackChain.length > 0)
+		failures.push("fallback_chain_not_empty");
+	if (!manifest.canary.structuredTerminalSucceeded)
+		failures.push("terminal_canary_failed");
+	if (!manifest.canary.terminalFailureDistinguishable) {
+		failures.push("terminal_failure_not_distinguishable");
+	}
+	if (!manifest.canary.userGlobalConfigurationDisabled) {
+		failures.push("user_global_configuration_not_disabled");
+	}
+	if (manifest.budgets.activeWorkingMs <= 0)
+		failures.push("active_working_budget_invalid");
+	if (manifest.budgets.wallMs <= 0) failures.push("wall_budget_invalid");
+	if (manifest.budgets.candidateCostUsd <= 0)
+		failures.push("cost_budget_invalid");
+	if (manifest.budgets.actionLimit <= 0) failures.push("action_budget_invalid");
+	if (manifest.budgets.contextTokens <= 0)
+		failures.push("context_budget_invalid");
+	if (manifest.budgets.maxOutputTokens <= 0)
+		failures.push("output_budget_invalid");
+	return failures;
+}
+
+function validateFunding(manifest: NativeHarnessComparisonManifest) {
+	const failures: string[] = [];
+	if (!manifest.canary.exactRouteFunded)
+		failures.push("exact_route_not_funded");
+	if (!manifest.canary.directProviderRouteObserved)
+		failures.push("direct_provider_route_not_observed");
+	if (manifest.canary.usageTokens <= 0) failures.push("positive_usage_missing");
+	if (manifest.canary.usageProvider !== manifest.model.observedProvider) {
+		failures.push("usage_provider_mismatch");
+	}
+	if (manifest.canary.usageModel !== manifest.model.observedModel) {
+		failures.push("usage_model_mismatch");
+	}
+	return failures;
+}
+
+function validateIsolation(manifest: NativeHarnessComparisonManifest) {
+	const failures: string[] = [];
+	if (!manifest.canary.sandboxAvailable) failures.push("sandbox_unavailable");
+	if (!manifest.canary.outOfScopeWriteDenied)
+		failures.push("out_of_scope_write_not_denied");
+	if (!manifest.canary.outOfScopeDestinationDenied) {
+		failures.push("out_of_scope_destination_not_denied");
+	}
+	if (!manifest.canary.ancillaryRoutesDisabledOrAttributed) {
+		failures.push("ancillary_routes_not_attributed");
+	}
+	if (
+		manifest.provenance.ancillaryModelRoutes.some(
+			(route) => !route.attributable,
+		)
+	) {
+		failures.push("unattributed_ancillary_model_route");
+	}
+	return failures;
+}
+
+function validateSequence(
+	manifest: NativeHarnessComparisonManifest,
+	admittedArms: readonly NativeHarnessAdmissionRecord[],
+) {
+	const failures: string[] = [];
+	const admittedById = new Map(
+		admittedArms
+			.filter((record) => record.admitted)
+			.map((record) => [record.armId, record]),
+	);
+	for (const prerequisite of manifest.arm.prerequisiteArmIds) {
+		if (!admittedById.has(prerequisite))
+			failures.push(`prerequisite_not_admitted:${prerequisite}`);
+	}
+	if (manifest.arm.class === "same-model-bridge") {
+		if (!manifest.arm.nativeAnchorId) {
+			failures.push("bridge_native_anchor_missing");
+		} else {
+			const anchor = admittedById.get(manifest.arm.nativeAnchorId);
+			if (!anchor) {
+				failures.push(
+					`bridge_native_anchor_not_admitted:${manifest.arm.nativeAnchorId}`,
+				);
+			} else if (
+				!anchor.observedRoute ||
+				anchor.observedRoute.provider !== manifest.model.observedProvider ||
+				anchor.observedRoute.model !== manifest.model.observedModel ||
+				anchor.observedRoute.effort !== manifest.model.observedEffort
+			) {
+				failures.push("bridge_observed_route_differs_from_native_anchor");
+			}
+		}
+	}
+	return failures;
+}
+
+function requiredStrings(
+	manifest: NativeHarnessComparisonManifest,
+): Array<[string, string]> {
+	return [
+		["arm_id", manifest.arm.id],
+		["cohort_revision", manifest.study.cohortRevision],
+		["prompt_sha256", manifest.study.promptSha256],
+		["terminal_schema_sha256", manifest.study.terminalSchemaSha256],
+		["scorer_revision", manifest.study.scorerRevision],
+		["harness_id", manifest.harness.id],
+		["harness_version", manifest.harness.version],
+		["harness_executable_sha256", manifest.harness.executableSha256],
+		["harness_config_sha256", manifest.harness.configSha256],
+		["requested_provider", manifest.model.requestedProvider],
+		["requested_model", manifest.model.requestedModel],
+		["requested_effort", manifest.model.requestedEffort],
+		["observed_provider", manifest.model.observedProvider],
+		["observed_model", manifest.model.observedModel],
+		["observed_effort", manifest.model.observedEffort],
+		["target_task_id", manifest.target.taskId],
+		["target_image_digest", manifest.target.imageDigest],
+		["target_fresh_instance_id", manifest.target.freshInstanceId],
+		["target_authorization_id", manifest.target.authorizationId],
+		["target_secret_seed_sha256", manifest.target.secretSeedSha256],
+		["provider_account_project", manifest.provenance.providerAccountProject],
+	];
+}