diff --git a/test/e2e/foundation.test.ts b/test/e2e/foundation.test.ts index 5f255de6..21531117 100644 --- a/test/e2e/foundation.test.ts +++ b/test/e2e/foundation.test.ts @@ -136,6 +136,11 @@ describe("scenario schema", () => { { name: "x", settings: {}, expect: { exit_code: 0, bogus: true } }, /expect: Unrecognized key: "bogus"$/m, ], + [ + "a tiers list, which no runner ever honored", + { name: "x", settings: {}, tiers: ["live"], expect: { exit_code: 0 } }, + /Unrecognized key: "tiers"$/m, + ], ])("rejects %s", (_label, raw, refusal) => { expect(() => parseScenario(raw, "refused.yml")).toThrow(refusal); }); diff --git a/test/e2e/fuzz.ts b/test/e2e/fuzz.ts index 538cc3cb..22715df1 100644 --- a/test/e2e/fuzz.ts +++ b/test/e2e/fuzz.ts @@ -532,7 +532,6 @@ async function witnessIteration( }; const scenario: Scenario = { name: `fuzz-witness-${key}-${kind}-${mode}-${seed}`, - tiers: ["mock"], settings: settingsDoc, inputs: { mode, on_missing_permission: "warn" }, denial_style: "fine_grained", @@ -560,7 +559,6 @@ async function rejectionIteration(seed: number, spec: RejectionSpec): Promise { const scenario: Scenario = { name: `fuzz-chaos-single-${seed}`, - tiers: ["mock"], // Fully granted by default, so the list read reaches the mock and the corruption fires. settings: { labels: [{ name: "chaos", color: "d73a4a" }] }, inputs: { mode: "apply" }, @@ -689,7 +686,6 @@ async function persistentChaosIteration( ): Promise { const scenario: Scenario = { name: `fuzz-chaos-persist-${seed}`, - tiers: ["mock"], settings: { labels: [{ name: "chaos", color: "d73a4a" }] }, inputs: { mode: "apply" }, denial_style: "fine_grained", @@ -1419,7 +1415,6 @@ async function faultedSectionRun(seed: number, plan: SectionFaultPlan): Promise< }; const scenario: Scenario = { name: `fuzz-fault-${plan.section}-${plan.kind}-x${plan.exhausting ? RETRY_BUDGET : 1}-${seed}`, - tiers: ["mock"], settings, inputs: { mode: plan.mode, on_missing_permission: "warn" }, denial_style: "fine_grained", @@ -1456,7 +1451,6 @@ async function unfaultableSectionRun( } const scenario: Scenario = { name: `fuzz-unfaultable-${section}-${seed}`, - tiers: ["mock"], settings: { [section]: UNFAULTABLE_APPLY_SETTINGS[section] }, inputs: { mode: "apply", on_missing_permission: "warn" }, denial_style: "fine_grained", @@ -1638,7 +1632,6 @@ async function preflightFaultRun(seed: number, surviving: boolean): Promise { } }); + test("no token is satisfied by a rejection that names only other keys", () => { + // A token that is a fragment of the error's own boilerplate (the known-sections list, the directive advice) + // would be found whichever key the run refused, so its case could never fail. + const decoy = validateSettingsDoc( + { zzz_decoy: [], _zzz: "x" }, + "settings.yml", + SectionSelection.ALL, + silentIo(), + ); + const boilerplate = describeProblem(decoy._unsafeUnwrapErr()); + // The control: the decoy prints both boilerplates, so the bare fragments a token must not be are found in it. + expect(["label", "_layerin"].filter((bare) => boilerplate.includes(bare))).toEqual([ + "label", + "_layerin", + ]); + + const satisfied = new Set(); + for (const { name, build } of INVALID_SETTINGS_CASES) { + for (let i = 0; i < 200; i++) { + const { offendingToken } = build(new Rng(i * 13 + 1)); + if (boilerplate.includes(offendingToken)) { + satisfied.add(`${name}: ${offendingToken}`); + } + } + } + expect([...satisfied]).toEqual([]); + }); + test("draws every catalog case over seeds", () => { const drawn = new Set(); for (let i = 0; i < 400; i++) { diff --git a/test/e2e/generators/invalid-settings.ts b/test/e2e/generators/invalid-settings.ts index 03f2045a..26753973 100644 --- a/test/e2e/generators/invalid-settings.ts +++ b/test/e2e/generators/invalid-settings.ts @@ -15,7 +15,8 @@ import { genSettings, SECRET_LIST_SECTIONS } from "./settings.js"; /** * Only violations validateSettingsDoc GENUINELY rejects belong here; values the loose shapes accept by design would * assert failures the contract does not promise, so they stay out. - * offendingToken -> must appear in the rejection error: a section path ("labels[2].name"), an unknown key, or a wording fragment + * offendingToken -> must appear in the rejection error, and in no rejection that names only other keys: a section + * path ("labels[2].name"), a section's own issue ("pages: "), or a wording fragment * stays out -> unknown nested keys under a loose shape, un-modeled enums, arbitrary types on loose keys, * `pages: null` */ @@ -75,6 +76,11 @@ const GITHUB_NAMED_SECTIONS = [ /** A hyphen, a leading digit, and the reserved prefix in both cases; `github_token` folds to GITHUB_TOKEN once uppercased. */ const REFUSED_GITHUB_NAMES = ["my-secret", "2_TOKEN", "GITHUB_TOKEN", "github_token"] as const; +/** The start of a section's own issue, `: `; the bare key would also be found in the known-sections list an unknown-key refusal prints. */ +function sectionToken(key: SectionKey): string { + return `${key}: `; +} + /** Entries come back by reference, so a case's mutation lands inside whichever form was drawn; itemToken spells that form's validator path. */ function validItems( rng: Rng, @@ -94,10 +100,11 @@ export const INVALID_SETTINGS_CASES: ReadonlyArray<{ { name: "unknown-top-level-key", build: (rng) => { + // The refusal lists every known section after the typo, so a bare "label" would be found in "labels" whichever key was refused. const typo = rng.pick(["labelz", "label", "milestone", "repositories", "branch"]); return { doc: { labels: genSettings(rng.fork("labels"), "labels") as Json, [typo]: [] }, - offendingToken: typo, + offendingToken: `unknown top-level section: ${typo} (known:`, }; }, }, @@ -105,10 +112,16 @@ export const INVALID_SETTINGS_CASES: ReadonlyArray<{ name: "unknown-underscore-key", build: (rng) => { // The underscore is the two directives' and nothing else's: a note or a misspelled directive is rejected, never dropped. - const key = rng.pick(["_notes", "_owner", "_layerin", "_undeclared"]); + // The advice after an unknown key quotes both directives, so each token carries the wording around its own key. + const [key, offendingToken] = rng.pick<[string, string]>([ + ["_notes", "unknown underscore key: _notes."], + ["_owner", "unknown underscore key: _owner."], + ["_layerin", "unknown underscore key: _layerin."], + [UNDECLARED_KEY, `${UNDECLARED_KEY} must be one of`], + ]); return { doc: { labels: genSettings(rng.fork("labels"), "labels") as Json, [key]: "x" }, - offendingToken: key, + offendingToken, }; }, }, @@ -116,14 +129,20 @@ export const INVALID_SETTINGS_CASES: ReadonlyArray<{ name: "array-section-wrong-type", build: (rng) => { const key = rng.pick(LIST_SECTIONS); - return { doc: { [key]: rng.pick([{ not: "an array" }, "oops", 7]) }, offendingToken: key }; + return { + doc: { [key]: rng.pick([{ not: "an array" }, "oops", 7]) }, + offendingToken: sectionToken(key), + }; }, }, { name: "record-section-wrong-type", build: (rng) => { const key = rng.pick(RECORD_SECTIONS); - return { doc: { [key]: rng.pick(["oops", 7, [1], null] as const) }, offendingToken: key }; + return { + doc: { [key]: rng.pick(["oops", 7, [1], null] as const) }, + offendingToken: sectionToken(key), + }; }, }, { @@ -161,7 +180,7 @@ export const INVALID_SETTINGS_CASES: ReadonlyArray<{ name: "pages-wrong-type", build: (rng) => ({ doc: { pages: rng.pick(["gh-pages", [1]] as const) }, - offendingToken: "pages", + offendingToken: sectionToken("pages"), }), }, { @@ -169,7 +188,7 @@ export const INVALID_SETTINGS_CASES: ReadonlyArray<{ name: "interaction-limits-wrong-type", build: (rng) => ({ doc: { interaction_limits: rng.pick(["oops", 7, [1]] as const) }, - offendingToken: "interaction_limits", + offendingToken: sectionToken("interaction_limits"), }), }, { diff --git a/test/e2e/generators/merge-scenario.ts b/test/e2e/generators/merge-scenario.ts index be81816f..ffd768ba 100644 --- a/test/e2e/generators/merge-scenario.ts +++ b/test/e2e/generators/merge-scenario.ts @@ -1184,7 +1184,6 @@ export function genMergeScenario( const top = layers[count - 1] as MergeLayer; const scenario: Scenario = { name: `fuzz-merge-${rng.seed}`, - tiers: ["mock"], settings: top.doc, settings_layers: layers.slice(0, -1).map((layer) => layer.doc), inputs: { diff --git a/test/e2e/generators/multi-scenario.ts b/test/e2e/generators/multi-scenario.ts index 84c7a1e8..a62f734f 100644 --- a/test/e2e/generators/multi-scenario.ts +++ b/test/e2e/generators/multi-scenario.ts @@ -485,7 +485,6 @@ export function genMultiScenario( const scenario: Scenario = { name: `fuzz-multi-${rng.seed}`, - tiers: ["mock"], settings: {}, inputs: { mode, diff --git a/test/e2e/generators/oracle.test.ts b/test/e2e/generators/oracle.test.ts index bfd74cbb..dd3dbba0 100644 --- a/test/e2e/generators/oracle.test.ts +++ b/test/e2e/generators/oracle.test.ts @@ -179,7 +179,6 @@ describe("read gating fold", () => { const apply = predictOutcomes({ ...readOnly, mode: "apply" }); expect(apply.preflightAborts).toBe("yes"); expect(apply.fullyGranted).toBe(false); - expect(apply.writeDeniedSections).toEqual(["codespaces_secrets"]); }); test("a mixed-gating section under a read-only grant unions the read and denied arms", () => { @@ -204,7 +203,6 @@ describe("read gating fold", () => { expect(mixed).toEqual(predictSectionAt("labels", noGrant, "plain")); const apply = predictSectionAt("labels", { ...noGrant, mode: "apply" }, "mixed"); expect([...apply.allowed]).toEqual(["failed"]); - expect(apply.mayWrite).toBe(false); }); test("the preflight barrier is certain for a write-gated section and only possible for a mixed one", () => { @@ -289,7 +287,7 @@ describe("predictSection rules", () => { label: string, key: SectionKey, overrides: Partial, - want: { allowed: Outcome[]; mayWrite?: boolean }, + want: { allowed: Outcome[] }, ]; const expectAllowed = ( _label: string, @@ -299,9 +297,6 @@ describe("predictSection rules", () => { ) => { const p = predictSection(key, meta(overrides)); expect([...p.allowed].sort()).toEqual(want.allowed); - if (want.mayWrite !== undefined) { - expect(p.mayWrite).toBe(want.mayWrite); - } }; // labels is a denied-semantics section, pages an absent-semantics one: under fine_grained a none grade on pages @@ -319,10 +314,10 @@ describe("predictSection rules", () => { { allowed: ["clean", "drift"] }, ], [ - "write granted: apply => {applied}, and may write", + "write granted: apply => {applied}", "labels", { mode: "apply", mask: { issues: "write" } }, - { allowed: ["applied"], mayWrite: true }, + { allowed: ["applied"] }, ], [ "none + 403 style: skipped under warn", @@ -390,10 +385,10 @@ describe("predictSection rules", () => { { allowed: ["clean"] }, ], [ - "a matching witness pins apply to exactly applied, with no write", + "a matching witness pins apply to exactly applied", "labels", { mode: "apply", liveKinds: { labels: "matching" } }, - { allowed: ["applied"], mayWrite: false }, + { allowed: ["applied"] }, ], [ "a drift-update witness pins check to exactly drift", @@ -445,7 +440,7 @@ describe("predictSection rules", () => { "read grade + matching witness in apply: applied despite the missing write grant", "labels", { mask: { issues: "read" }, mode: "apply", liveKinds: { labels: "matching" } }, - { allowed: ["applied"], mayWrite: false }, + { allowed: ["applied"] }, ], ])("%s", expectAllowed); @@ -463,7 +458,6 @@ describe("predictSection rules", () => { }), ); expect([...p.allowed]).toEqual(["clean"]); - expect(p.mayWrite).toBe(false); } }); @@ -525,7 +519,6 @@ describe("predictSection rules", () => { grades: [], allowed: new Set(["excluded"]), posture: "denied", - mayWrite: false, }, ], [ @@ -537,7 +530,6 @@ describe("predictSection rules", () => { grades: ["none"], allowed: new Set(["failed"]), posture: "denied", - mayWrite: false, }, ], [ @@ -554,7 +546,6 @@ describe("predictSection rules", () => { grades: [], allowed: new Set(["excluded"]), posture: "absent", - mayWrite: false, }, ], [ @@ -572,7 +563,6 @@ describe("predictSection rules", () => { grades: [], allowed: new Set(["excluded"]), posture: "denied", - mayWrite: false, }, ], ])("%s", (_label, key, overrides, prediction) => { @@ -595,19 +585,16 @@ describe("predictSection rules", () => { grades: [], allowed: new Set(["excluded"]), posture: "denied", - mayWrite: false, }, { key: "pages", grades: ["write"], allowed: new Set(["applied"]), posture: "absent", - mayWrite: true, }, ], allowedExitCodes: new Set([0]), noWritesInCheck: false, - writeDeniedSections: [], fullyGranted: true, preflightAborts: "no", }); @@ -620,19 +607,16 @@ describe("predictSection rules", () => { grades: ["none"], allowed: new Set(["failed"]), posture: "denied", - mayWrite: false, }, { key: "pages", grades: ["write"], allowed: new Set(["applied"]), posture: "absent", - mayWrite: true, }, ], allowedExitCodes: new Set([1]), noWritesInCheck: false, - writeDeniedSections: ["labels"], fullyGranted: false, preflightAborts: "yes", }); @@ -671,7 +655,6 @@ describe("predictSection rules", () => { grades: ["write"], allowed: new Set(["applied"]), posture: "denied", - mayWrite: false, }); const checkP = predictSection( @@ -688,7 +671,6 @@ describe("predictSection rules", () => { grades: ["write"], allowed: new Set(["clean"]), posture: "denied", - mayWrite: false, }); }); }); @@ -712,19 +694,16 @@ describe("predictOutcomes run level", () => { grades: ["write"], allowed: new Set(["applied"]), posture: "denied", - mayWrite: false, }, { key: "labels", grades: ["write"], allowed: new Set(["applied"]), posture: "denied", - mayWrite: true, }, ], allowedExitCodes: new Set([0]), noWritesInCheck: false, - writeDeniedSections: [], fullyGranted: true, preflightAborts: "no", }); @@ -738,19 +717,16 @@ describe("predictOutcomes run level", () => { grades: ["none"], allowed: new Set(["failed"]), posture: "absent", - mayWrite: false, }, { key: "labels", grades: ["write"], allowed: new Set(["applied"]), posture: "denied", - mayWrite: true, }, ], allowedExitCodes: new Set([1]), noWritesInCheck: false, - writeDeniedSections: ["teams"], fullyGranted: false, preflightAborts: "yes", }); diff --git a/test/e2e/generators/oracle.ts b/test/e2e/generators/oracle.ts index 110ecdf8..27709600 100644 --- a/test/e2e/generators/oracle.ts +++ b/test/e2e/generators/oracle.ts @@ -160,14 +160,8 @@ export interface SectionPrediction { /** The outcomes the section is allowed to report; the runner must see one. */ allowed: Set; - - /** False folds the section into writeDeniedSections, whose writes the fuzz asserts never mutate state. */ - mayWrite: boolean; } -/** One section's verdict at one grade: what it may report and whether a write may land. */ -type GradeArm = Pick; - export interface RunPrediction { /** One prediction per declared section, in declaration order. */ sections: SectionPrediction[]; @@ -178,9 +172,6 @@ export interface RunPrediction { /** True in check mode, where any write is a violation. */ noWritesInCheck: boolean; - /** Sections whose denied writes must never mutate state. */ - writeDeniedSections: SectionKey[]; - /** The fixpoint proofs in fuzz.ts run only under this. */ fullyGranted: boolean; @@ -351,28 +342,25 @@ function deniedOutcome(meta: ScenarioMeta, required: boolean): Set { } /** At a write grant only the witness decides: a matching one mirrors every field the handler diffs, so no write is ever attempted; a drift witness makes a check-mode clean a false-negative drift detector. */ -function predictAtWriteGrant(check: boolean, witness: LiveWitnessKind | undefined): GradeArm { +function predictAtWriteGrant(check: boolean, witness: LiveWitnessKind | undefined): Set { if (witness === "matching") { - return { allowed: new Set([check ? "clean" : "applied"]), mayWrite: false }; + return new Set([check ? "clean" : "applied"]); } if (witness !== undefined) { - return { allowed: new Set([check ? "drift" : "applied"]), mayWrite: !check }; + return new Set([check ? "drift" : "applied"]); } - return { - allowed: check ? new Set(["clean", "drift"]) : new Set(["applied"]), - mayWrite: !check, - }; + return check ? new Set(["clean", "drift"]) : new Set(["applied"]); } /** The reads go through in check mode, so the witness alone narrows {clean, drift}. */ -function predictCheckReadThrough(witness: LiveWitnessKind | undefined): GradeArm { +function predictCheckReadThrough(witness: LiveWitnessKind | undefined): Set { if (witness === "matching") { - return { allowed: new Set(["clean"]), mayWrite: false }; + return new Set(["clean"]); } if (witness !== undefined) { - return { allowed: new Set(["drift"]), mayWrite: false }; + return new Set(["drift"]); } - return { allowed: new Set(["clean", "drift"]), mayWrite: false }; + return new Set(["clean", "drift"]); } /** @@ -380,27 +368,24 @@ function predictCheckReadThrough(witness: LiveWitnessKind | undefined): GradeArm * so the missing grant is never exercised; a drift witness forces one, so the section can never be a no-op * "applied" (the mid-apply permission-denied fold in orchestrate.ts). Without a witness, a comparing section may * land applied when the live state already matched, while a no-read section's write is unconditional, so its - * denial is certain and "applied" unreachable; an absent-posture section may attempt one write before the denial. + * denial is certain and "applied" unreachable. */ function predictApplyReadThrough( key: SectionKey, meta: ScenarioMeta, witness: LiveWitnessKind | undefined, - posture: DenialPosture, -): GradeArm { +): Set { const required = meta.requiredSections.includes(key); if (witness === "matching") { - return { allowed: new Set(["applied"]), mayWrite: false }; + return new Set(["applied"]); } if (witness !== undefined) { - return { allowed: deniedOutcome(meta, required), mayWrite: false }; + return deniedOutcome(meta, required); } const canSilentlyApply = !NO_READ_SECTIONS.has(key); - const allowed: Set = - required || meta.policy === "fail" - ? new Set(canSilentlyApply ? ["applied", "failed"] : ["failed"]) - : new Set(canSilentlyApply ? ["applied", "skipped"] : ["skipped"]); - return { allowed, mayWrite: posture === "absent" }; + return required || meta.policy === "fail" + ? new Set(canSilentlyApply ? ["applied", "failed"] : ["failed"]) + : new Set(canSilentlyApply ? ["applied", "skipped"] : ["skipped"]); } /** @@ -412,11 +397,11 @@ function predictAtGrade( meta: ScenarioMeta, grade: MaskGrade, posture: DenialPosture, -): GradeArm { +): Set { const check = meta.mode === "check"; const witness = meta.liveKinds?.[key]; if (check && NO_READ_SECTIONS.has(key)) { - return { allowed: new Set(["clean"]), mayWrite: false }; + return new Set(["clean"]); } if (grade === "write") { @@ -430,7 +415,7 @@ function predictAtGrade( !NO_READ_SECTIONS.has(key) && (meta.denialStyle === 403 || posture === "denied"); if (readsAsDenied) { - return { allowed: deniedOutcome(meta, meta.requiredSections.includes(key)), mayWrite: false }; + return deniedOutcome(meta, meta.requiredSections.includes(key)); } // From here the reads go through and only an apply-mode write can be denied. @@ -439,7 +424,7 @@ function predictAtGrade( if (check) { return predictCheckReadThrough(witness); } - return predictApplyReadThrough(key, meta, witness, posture); + return predictApplyReadThrough(key, meta, witness); } /** @@ -462,7 +447,6 @@ export function predictSectionAt( grades: [], posture: DENIAL_POSTURE[key], allowed: new Set(["excluded"]), - mayWrite: false, }; } // The org probe is declared permission "none", so no mask key gates it and its 404 is met before @@ -473,7 +457,6 @@ export function predictSectionAt( grades: ["write"], posture: DENIAL_POSTURE[key], allowed: new Set([meta.mode === "check" ? "clean" : "applied"]), - mayWrite: false, }; } const grant = sectionGrade(key, meta.mask); @@ -495,8 +478,7 @@ export function predictSectionAt( key, grades: orgDenied ? ["none"] : grades, posture, - allowed: new Set(arms.flatMap((arm) => [...arm.allowed])), - mayWrite: arms.some((arm) => arm.mayWrite), + allowed: new Set(arms.flatMap((arm) => [...arm])), }; } @@ -593,7 +575,6 @@ export function predictOutcomes(meta: ScenarioMeta): RunPrediction { sections, allowedExitCodes: exitCodes, noWritesInCheck: check, - writeDeniedSections: sections.filter((s) => !writeGranted(s) && !s.mayWrite).map((s) => s.key), fullyGranted: sections.every(writeGranted), preflightAborts, }; diff --git a/test/e2e/generators/single-scenario.ts b/test/e2e/generators/single-scenario.ts index 6e4f5a10..e4af06ba 100644 --- a/test/e2e/generators/single-scenario.ts +++ b/test/e2e/generators/single-scenario.ts @@ -196,7 +196,6 @@ export function genScenario( const scenario: Scenario = { name: `fuzz-${rng.seed}`, - tiers: ["mock"], settings, inputs: { mode, diff --git a/test/e2e/runner/apply-idempotence-proof.ts b/test/e2e/runner/apply-idempotence-proof.ts index 725d1a35..84781b64 100644 --- a/test/e2e/runner/apply-idempotence-proof.ts +++ b/test/e2e/runner/apply-idempotence-proof.ts @@ -48,10 +48,14 @@ export function captureRerun( }; } -/** Re-runs go through the runner: `invoke` uses the SAME mock and temp dir as the primary invocation; the kill cap is the runner's constant. */ +/** + * Re-runs go through the runner: `invoke` uses the SAME mock and temp dir as the primary invocation, and the kill cap + * is the runner's constant. So is the verdict on a check that did not report clean, which reads the run's outputs and summary. + */ export interface ChildInvoker { invoke(scenario: Scenario): Promise; killNote(run: Invocation): string; + unverifiedCheckFailures(label: string, check: Invocation, requests: LoggedRequest[]): string[]; } /** The multi settings/permissions maps and the discovery pool are run CONFIG the pipeline never mutates, so they stay out of the snapshot. */ @@ -293,15 +297,15 @@ export async function assertApplyIdempotent( const checkViolationsBefore = handle.violations.length; handle.enterCheckMode(); const check = await child.invoke({ ...rerun, inputs: { ...rerun.inputs, mode: "check" } }); - reruns.push( - captureRerun("apply-idempotence check", check, handle.requests.slice(checkRequestsBefore)), - ); + const checkRequests = handle.requests.slice(checkRequestsBefore); + reruns.push(captureRerun("apply-idempotence check", check, checkRequests)); if (check.exitCode !== 0) { failures.push( `apply-idempotence: the check run after the second apply exited ${check.exitCode}, expected 0${child.killNote(check)}`, ); } - const checkWrites = handle.requests.slice(checkRequestsBefore).filter(isWriteRequest); + failures.push(...child.unverifiedCheckFailures("apply-idempotence", check, checkRequests)); + const checkWrites = checkRequests.filter(isWriteRequest); if (checkWrites.length > 0) { failures.push( `apply-idempotence: the check run wrote ${checkWrites.length} time(s): ${checkWrites.map((r) => renderRequest(r, false)).join(", ")}`, diff --git a/test/e2e/runner/runner.test.ts b/test/e2e/runner/runner.test.ts index cf931910..70a5fa53 100644 --- a/test/e2e/runner/runner.test.ts +++ b/test/e2e/runner/runner.test.ts @@ -26,6 +26,7 @@ import { setReplay, snapshotCheckInputs, stripDebugLines, + unverifiedCheckFailures, writtenSnapshotLeaks, writtenSnapshotPaths, yamlStrings, @@ -182,6 +183,91 @@ describe("roundTripFailures (the snapshot round trip's verdict)", () => { }); }); +describe("unverifiedCheckFailures (a converging check that did not report clean)", () => { + // Under on-missing-permission: warn a check exits 0 with every section skipped, having compared nothing; judged + // on exit code and writes alone, both fixpoint proofs would pass on it. + const repo = "/repos/e2e-owner/e2e-repo"; + const read = (pathname: string, deniedBy?: string): LoggedRequest => ({ + method: "GET", + pathname, + query: "", + status: deniedBy === undefined ? 200 : 403, + ...(deniedBy === undefined ? {} : { deniedBy }), + }); + const check = (result: string, skipped: string, rows: string[], exitCode = 0) => ({ + exitCode, + outputs: { result, "skipped-sections": skipped }, + summary: ["| Section | Status | Detail |", "|---|---|---|", ...rows].join("\n"), + stdout: "", + stderr: "", + killedByHarness: false, + }); + const cleanRow = (key: string): string => `| ${key} | :white_check_mark: clean | - |`; + const skippedRow = (key: string): string => `| ${key} | :fast_forward: skipped | - |`; + const nothingClean = + 'proof: the check\'s result is "partial" and no section reads clean, so it verified nothing'; + const unexplainedSkip = + "proof: the check's result is \"partial\", and it skipped [labels] though the token's mask denied no read there"; + + test.each< + [label: string, run: ReturnType, requests: LoggedRequest[], want: string[]] + >([ + ["a clean check needs no further evidence", check("clean", "", []), [], []], + // A drifted check did compare, and its exit code already fails the proof: "verified nothing" would misstate it. + [ + "a check that exited non-zero is left to its exit code", + check("drift", "", ["| labels | :warning: drift | - |"], 1), + [read(`${repo}/labels`)], + [], + ], + [ + "every section skipped and nothing asked of the mock fails on both counts", + check("partial", "labels", [skippedRow("labels")]), + [], + [unexplainedSkip, nothingClean], + ], + [ + "a skip the mask explains beside a clean section passes", + check("partial", "labels", [skippedRow("labels"), cleanRow("autolinks")]), + [read(`${repo}/labels`, "issues"), read(`${repo}/autolinks`)], + [], + ], + [ + "a skip the mask explains with no section clean verified nothing", + check("partial", "labels", [skippedRow("labels")]), + [read(`${repo}/labels`, "issues")], + [nothingClean], + ], + [ + "a skipped section whose read the mock answered is not explained", + check("partial", "labels", [skippedRow("labels"), cleanRow("autolinks")]), + [read(`${repo}/labels`), read(`${repo}/autolinks`)], + [unexplainedSkip], + ], + [ + "a denied read of another section explains nothing", + check("partial", "labels", [skippedRow("labels"), cleanRow("autolinks")]), + [read(`${repo}/milestones`, "issues"), read(`${repo}/autolinks`)], + [unexplainedSkip], + ], + // A multi-repo summary repeats a key per target; read through the last-row-wins map, the clean row would be lost. + [ + "a key clean on one target and skipped on the next still counts as checked", + check("partial", "labels", [cleanRow("labels"), skippedRow("labels")]), + [read("/repos/e2e-owner/svc-a/labels"), read("/repos/e2e-owner/svc-b/labels", "issues")], + [], + ], + [ + "a fleet result of skipped (a target without a settings file) passes beside a clean section", + check("skipped", "", [cleanRow("labels")]), + [read("/repos/e2e-owner/svc-a/labels")], + [], + ], + ])("%s", (_label, run, requests, want) => { + expect(unverifiedCheckFailures("proof", run, requests)).toEqual(want); + }); +}); + describe("snapshotCheckInputs (the round-trip check's inputs)", () => { // private_report at its default is legal in snapshot mode and outside the // three inputs the check once allowlisted: only derivation by exclusion keeps it. @@ -368,6 +454,33 @@ describe("requestLogFailures (the request-log rules over recorded requests)", () expect(requestLogFailures(exp, recorded)).toEqual(want); }); + // An empty list is an empty subsequence, which every write log contains: read that way, a scenario that says + // "this run writes nothing" would pass on a run that wrote anything. + test.each< + [ + label: string, + exp: Parameters[0], + log: LoggedRequest[], + want: string[], + ] + >([ + [ + "an empty mutations list beside a write fails, naming the write", + { mutations: [] }, + recorded, + ["mutations: [] expects no write, but the run sent 1: PATCH /repos/o/r/issues/7"], + ], + [ + "an empty mutations list beside reads alone passes", + { mutations: [] }, + recorded.slice(0, 1), + [], + ], + ["no mutations key asserts nothing about the writes", {}, recorded, []], + ])("%s", (_label, exp, log, want) => { + expect(requestLogFailures(exp, log)).toEqual(want); + }); + // The `{repo}` placeholder must expand in EVERY request-path list: a list left unexpanded is // always-red under `requests_contain` and always-green under `never`, and no scenario would notice. const onAdminRepo: LoggedRequest[] = [ @@ -576,7 +689,6 @@ describe("markReportTitle (counterfactual disambiguation)", () => { describe("failureArtifacts (a verdict the runner did not reach)", () => { const scenario: Scenario = { name: "fuzz-oracle-42", - tiers: ["mock"], settings: {}, inputs: {}, denial_style: "fine_grained", diff --git a/test/e2e/runner/runner.ts b/test/e2e/runner/runner.ts index 2a54b598..6e691adb 100644 --- a/test/e2e/runner/runner.ts +++ b/test/e2e/runner/runner.ts @@ -23,6 +23,7 @@ import { dirname, join, relative } from "node:path"; import { parse as parseYaml, stringify as stringifyYaml } from "yaml"; import { parseRepoSlug } from "../../../src/discovery/targets.js"; import { type OutputName, redactRanges } from "../../../src/io.js"; +import { tableRows } from "../../docs/markdown.js"; import { ROOT } from "../../root.js"; import { ARTIFACTS_DIR, @@ -32,7 +33,7 @@ import { RUNNER_ROOT_FILES, } from "../constants.js"; import { type LoggedRequest, renderRequest } from "../mock/contract.js"; -import { isWriteRequest } from "../mock/dispatch.js"; +import { isWriteRequest, sectionForRequest } from "../mock/dispatch.js"; import { type MockHandle, type ServerOptions, startMockServer } from "../mock/server.js"; import { sharedValidator } from "../openapi/validate.js"; import { collectYmlFiles, type Expect, type Scenario, settingsYamlFor } from "../scenario.js"; @@ -91,6 +92,9 @@ const COMMAND_ENCODING: ReadonlyArray<[raw: string, encoded: string]> = [ /** The index of the command line inside replayBlockLines' block, the one line setReplay swaps. */ const REPLAY_LINE = 4; +/** The Status cell of a section the run compared and found matching, whatever icon the summary gives it. */ +const CLEAN_STATUS_CELL = /^:[a-z_]+: clean$/; + type ScenarioInputs = NonNullable; type ForwardedInput = Exclude; @@ -243,6 +247,11 @@ export function parseSummaryOutcomes(summary: string): Record { return outcomes; } +/** The `skipped-sections` output as a list; undefined when the run published no such output. */ +function publishedSkippedSections(outputs: Record): string[] | undefined { + return outputs[SKIPPED_SECTIONS_OUTPUT]?.split(",").filter(Boolean); +} + /** A missing or unparseable output yields {} instead of throwing, so the expectation reports the gap as a failure. */ export function parseReposResult(raw: string | undefined): Record { if (!raw) { @@ -436,6 +445,46 @@ export function roundTripFailures( return failures; } +/** + * Exit 0 can mean a check skipped every section under on-missing-permission: warn and compared nothing. One that does + * not report clean passes only on evidence: + * every skipped section -> the mock's permission gate denied one of its reads, which only the token's mask does + * the run -> the Status cell of at least one section row reads clean + * The skipped list is the run's union, so on a multi-repo run any target's denied read explains it. + */ +export function unverifiedCheckFailures( + label: string, + check: Invocation, + requests: LoggedRequest[], +): string[] { + const result = check.outputs.result; + if (check.exitCode !== 0 || result === "clean") { + return []; + } + + const denied = new Set( + requests + .filter((request) => request.deniedBy !== undefined && !isWriteRequest(request)) + .map((request) => sectionForRequest(request.method, request.pathname, request.body)), + ); + const unexplained = (publishedSkippedSections(check.outputs) ?? []).filter( + (key) => !denied.has(key), + ); + + const failures: string[] = []; + if (unexplained.length > 0) { + failures.push( + `${label}: the check's result is "${result}", and it skipped [${unexplained.join(", ")}] though the token's mask denied no read there`, + ); + } + if (!tableRows(check.summary).some(([, status = ""]) => CLEAN_STATUS_CELL.test(status))) { + failures.push( + `${label}: the check's result is "${result}" and no section reads clean, so it verified nothing`, + ); + } + return failures; +} + /** * settings-file is a single-repo input the action rejects beside the multi-repo inputs and in * snapshot mode. A merge lists every layer file below settings.yml, lowest first; a snapshot names @@ -687,6 +736,29 @@ export function forbiddenPresent(patterns: string[], log: string[]): string[] { return patterns.filter((pattern) => log.some((entry) => entry.startsWith(pattern))); } +/** + * The `mutations` rule. An empty list says the run wrote nothing: read as a subsequence it would match every log, so + * only an absent key asserts nothing. + */ +function mutationFailures(expected: readonly string[] | undefined, writes: string[]): string[] { + if (expected === undefined) { + return []; + } + if (expected.length === 0 && writes.length > 0) { + return [ + `mutations: [] expects no write, but the run sent ${writes.length}: ${writes.join(", ")}`, + ]; + } + + const mutations = expandRepoPatterns(expected); + if (isSubsequence(mutations, writes)) { + return []; + } + return [ + `mutations not found as a subsequence:\n want: ${mutations.join(", ")}\n writes: ${writes.join(", ")}`, + ]; +} + /** * The three request-log rules over the recorded requests. A write's query is never part of its identity, so * `mutations` match "METHOD /path"; `never` and `requests_contain` match "METHOD /path?query", so one lookup on a path @@ -696,14 +768,8 @@ export function requestLogFailures( exp: Partial>, requests: LoggedRequest[], ): string[] { - const failures: string[] = []; const writes = requests.filter(isWriteRequest).map((r) => renderRequest(r, false)); - const mutations = expandRepoPatterns(exp.mutations); - if (!isSubsequence(mutations, writes)) { - failures.push( - `mutations not found as a subsequence:\n want: ${mutations.join(", ")}\n writes: ${writes.join(", ")}`, - ); - } + const failures = mutationFailures(exp.mutations, writes); const fullLog = requests.map((r) => renderRequest(r, true)); for (const pattern of forbiddenPresent(expandRepoPatterns(exp.never), fullLog)) { failures.push(`forbidden request present: ${pattern}`); @@ -770,11 +836,11 @@ function skippedSectionsFailures(exp: Expect, first: Invocation): string[] { if (exp.skipped_sections === undefined) { return []; } - const published = first.outputs[SKIPPED_SECTIONS_OUTPUT]; + const published = publishedSkippedSections(first.outputs); if (published === undefined) { return [`the ${SKIPPED_SECTIONS_OUTPUT} output was not published at all`]; } - const live = published.split(",").filter(Boolean).sort(); + const live = published.sort(); const want = [...exp.skipped_sections].sort(); if (JSON.stringify(live) === JSON.stringify(want)) { return []; @@ -877,6 +943,7 @@ async function applyIdempotenceFailures({ scenario, dir, handle, reruns }: Run): const idempotence = await assertApplyIdempotent(scenario, handle, { invoke: (rerun) => invoke(rerun, dir, handle.url), killNote, + unverifiedCheckFailures, }); reruns.push(...idempotence.reruns); return idempotence.failures; @@ -905,6 +972,7 @@ async function convergenceFailures({ scenario, dir, handle, reruns }: Run): Prom `convergence: rerun exited ${converge.exitCode}, expected 0${killNote(converge)}`, ); } + failures.push(...unverifiedCheckFailures("convergence", converge, rerunRequests)); if (newWrites.length > 0) { failures.push( `convergence: rerun wrote ${newWrites.length} time(s): ${newWrites.map((r) => renderRequest(r, false)).join(", ")}`, diff --git a/test/e2e/scenario.ts b/test/e2e/scenario.ts index 3a24094a..e3fca96a 100644 --- a/test/e2e/scenario.ts +++ b/test/e2e/scenario.ts @@ -19,9 +19,6 @@ import { LAYER_FILE_PREFIX, RUNNER_ROOT_FILES } from "./constants.js"; import type { LiveState } from "./mock/state.js"; import { LIVE_STATE_KEYS } from "./mock/state.js"; -/** Only "mock" exists today; "live" is reserved for a future App-token tier scenarios can opt into. */ -const TierSchema = z.enum(["mock", "live"]); - /** Every fine-grained PAT resource plus the organization "members" grant, which teams need and is not a PatResource. */ export const MASK_KEYS = [ "administration", @@ -127,7 +124,8 @@ const ExpectSchema = z /** * Ordered "METHOD /path" prefixes the write log must contain as a subsequence; `{repo}` expands to * the scenario's owner/name. A GraphQL operation is spelled "GRAPHQL ", and a GraphQL READ - * never appears in the write log despite its POST. + * never appears in the write log despite its POST. An empty list means the run sent no write at all; + * only an absent key asserts nothing. */ mutations: z.array(z.string()).optional(), /** @@ -188,6 +186,8 @@ const ExpectSchema = z * converges -> rerun in check mode, expecting exit 0 and zero writes * apply_idempotent -> rerun APPLY and prove it a fixpoint (assertApplyIdempotent), ending in a * converging check, so it subsumes `converges`; apply mode without the issue report channel + * The check must also have verified something: it reports `clean`, or every section it skipped is one the + * token's mask denied a read of and at least one section reads clean. */ fixpoint: z.enum(["converges", "apply_idempotent"]).optional(), /** Multi-repo: the per-target rollup from the `repos-result` output, "owner/name" -> result string. */ @@ -316,7 +316,6 @@ const ScenarioSchema = z .object({ name: z.string(), description: z.string().optional(), - tiers: z.array(TierSchema).default(["mock"]), settings: SettingsSchema.optional(), /** * The EXACT settings.yml text the single-repo run reads (a multi-repo target's is