From 1e7d4224a140ae5f3d2cfb049b0045f266d4d020 Mon Sep 17 00:00:00 2001 From: Luis Manrique Date: Wed, 5 Aug 2026 03:50:58 +0000 Subject: [PATCH] test: add seed43 target admission gates --- src/index.ts | 9 ++ src/seed43-target-admission.ts | 125 +++++++++++++++++++++++++ tests/seed43-target-admission.test.mjs | 70 ++++++++++++++ 3 files changed, 204 insertions(+) create mode 100644 src/seed43-target-admission.ts create mode 100644 tests/seed43-target-admission.test.mjs diff --git a/src/index.ts b/src/index.ts index f75eaaa7..41808083 100644 --- a/src/index.ts +++ b/src/index.ts @@ -115,6 +115,15 @@ export { type Hooks as OutcomeExecutorHooks, type StoredCall, } from "./outcome-executors/index.js"; +export { + SEED43_COLLAPSE_SENTINEL, + admitSeed43Targets, + isCollapseSentinel, + sha256TokenSequence, + type Seed43AdmissionResult, + type Seed43TargetRecord, + type Seed43ToolCall, +} from "./seed43-target-admission.js"; export const repoRoot = installedPackageRoot(); diff --git a/src/seed43-target-admission.ts b/src/seed43-target-admission.ts new file mode 100644 index 00000000..be6d9ed6 --- /dev/null +++ b/src/seed43-target-admission.ts @@ -0,0 +1,125 @@ +import { createHash } from "node:crypto"; + +export const SEED43_COLLAPSE_SENTINEL = Object.freeze([16071, 95597, 11]); + +export type Seed43ToolCall = { + name: string; + arguments: unknown; + effects: readonly string[]; +}; + +export type Seed43TargetRecord = { + rowId: string; + actionRequired: boolean; + servingInputSha256: string; + trainingInputSha256: string; + targetTokens: readonly number[]; + targetText?: string; + toolCalls: readonly Seed43ToolCall[]; + expectedToolCalls: readonly Seed43ToolCall[]; + failureFamily?: string; +}; + +export type Seed43AdmissionResult = { + admitted: boolean; + actionRequiredCount: number; + noActionCount: number; + protectedFailureFamilies: string[]; + rejectionClusters: Record; + rejectedRowIds: string[]; +}; + +function stableJson(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + return `{${Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`) + .join(",")}}`; +} + +function sameToolCall(actual: Seed43ToolCall, expected: Seed43ToolCall): boolean { + return actual.name === expected.name + && stableJson(actual.arguments) === stableJson(expected.arguments) + && stableJson(actual.effects) === stableJson(expected.effects); +} + +function reject( + clusters: Record, + rejectedRowIds: string[], + rowId: string, + cluster: string, +): void { + clusters[cluster] = (clusters[cluster] ?? 0) + 1; + rejectedRowIds.push(rowId); +} + +export function sha256TokenSequence(tokens: readonly number[]): string { + return createHash("sha256").update(JSON.stringify(tokens)).digest("hex"); +} + +export function isCollapseSentinel(tokens: readonly number[]): boolean { + return stableJson(tokens) === stableJson(SEED43_COLLAPSE_SENTINEL); +} + +export function admitSeed43Targets( + records: readonly Seed43TargetRecord[], + protectedFailureFamilies: readonly string[], +): Seed43AdmissionResult { + const rejectionClusters: Record = {}; + const rejectedRowIds: string[] = []; + let actionRequiredCount = 0; + let noActionCount = 0; + + for (const row of records) { + if (row.actionRequired) actionRequiredCount += 1; + else noActionCount += 1; + + if (row.servingInputSha256 !== row.trainingInputSha256) { + reject(rejectionClusters, rejectedRowIds, row.rowId, "input_serialization_mismatch"); + } + if (isCollapseSentinel(row.targetTokens)) { + reject(rejectionClusters, rejectedRowIds, row.rowId, "collapse_sentinel"); + } + + if (row.actionRequired) { + if (row.toolCalls.length === 0) { + reject(rejectionClusters, rejectedRowIds, row.rowId, "action_without_tool_call"); + } else if (row.toolCalls.length !== row.expectedToolCalls.length) { + reject(rejectionClusters, rejectedRowIds, row.rowId, "tool_call_count_mismatch"); + } else if (row.toolCalls.some((call, index) => !sameToolCall(call, row.expectedToolCalls[index]))) { + reject(rejectionClusters, rejectedRowIds, row.rowId, "tool_call_contract_mismatch"); + } + if (row.targetText?.trim() === "NO_ACTION") { + reject(rejectionClusters, rejectedRowIds, row.rowId, "positive_no_action_text"); + } + } else if (row.toolCalls.length > 0) { + reject(rejectionClusters, rejectedRowIds, row.rowId, "no_action_with_tool_call"); + } + } + + if (actionRequiredCount === 0) { + rejectionClusters.missing_action_rows = 1; + } + if (noActionCount > actionRequiredCount) { + rejectionClusters.no_action_dominates = 1; + } + const observedFamilies = new Set( + records + .filter((row) => !row.actionRequired && row.failureFamily) + .map((row) => row.failureFamily as string), + ); + const missingFamilies = protectedFailureFamilies.filter((family) => !observedFamilies.has(family)); + if (missingFamilies.length > 0) { + rejectionClusters.missing_protected_failure_family = missingFamilies.length; + } + + return { + admitted: Object.keys(rejectionClusters).length === 0, + actionRequiredCount, + noActionCount, + protectedFailureFamilies: [...observedFamilies].sort(), + rejectionClusters, + rejectedRowIds, + }; +} diff --git a/tests/seed43-target-admission.test.mjs b/tests/seed43-target-admission.test.mjs new file mode 100644 index 00000000..71fb365d --- /dev/null +++ b/tests/seed43-target-admission.test.mjs @@ -0,0 +1,70 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { + SEED43_COLLAPSE_SENTINEL, + admitSeed43Targets, + isCollapseSentinel, +} from "../dist/seed43-target-admission.js"; + +const hash = "a".repeat(64); + +function action(overrides = {}) { + return { + rowId: "action-1", + actionRequired: true, + servingInputSha256: hash, + trainingInputSha256: hash, + targetTokens: [1, 2, 3], + toolCalls: [{ name: "write", arguments: { id: "x" }, effects: ["updated:x"] }], + expectedToolCalls: [{ name: "write", arguments: { id: "x" }, effects: ["updated:x"] }], + ...overrides, + }; +} + +function noop(overrides = {}) { + return { + rowId: "noop-1", + actionRequired: false, + servingInputSha256: hash, + trainingInputSha256: hash, + targetTokens: [4, 5], + toolCalls: [], + expectedToolCalls: [], + failureFamily: "legitimate_noop", + ...overrides, + }; +} + +describe("seed43 target admission", () => { + it("rejects the seed42 three-token collapse sentinel", () => { + assert.equal(isCollapseSentinel(SEED43_COLLAPSE_SENTINEL), true); + assert.equal(isCollapseSentinel([16071, 95597, 12]), false); + }); + + it("requires action rows to round-trip to the exact tool contract", () => { + const result = admitSeed43Targets([action(), noop()], ["legitimate_noop"]); + assert.equal(result.admitted, true); + }); + + it("rejects text-only positive targets and serialization drift", () => { + const result = admitSeed43Targets([ + action({ targetText: "NO_ACTION", trainingInputSha256: "b".repeat(64) }), + noop(), + ], ["legitimate_noop"]); + assert.equal(result.admitted, false); + assert.equal(result.rejectionClusters.input_serialization_mismatch, 1); + assert.equal(result.rejectionClusters.positive_no_action_text, 1); + }); + + it("rejects action/no-action sign inversions and dominance", () => { + const result = admitSeed43Targets([ + action({ toolCalls: [] }), + noop({ rowId: "noop-2" }), + noop({ rowId: "noop-3" }), + ], ["legitimate_noop"]); + assert.equal(result.admitted, false); + assert.equal(result.rejectionClusters.action_without_tool_call, 1); + assert.equal(result.rejectionClusters.no_action_dominates, 1); + }); +});