test(e2e): make an empty mutation list and a convergence re-run assert something

test/e2e/foundation.test.ts

@@ -136,6 +136,11 @@ describe("scenario schema", () => {
       { name: "x", settings: {}, expect: { exit_code: 0, bogus: true } },
       /expect: Unrecognized key: "bogus"$/m,
     ],
+    [
+      "a tiers list, which no runner ever honored",
+      { name: "x", settings: {}, tiers: ["live"], expect: { exit_code: 0 } },
+      /Unrecognized key: "tiers"$/m,
+    ],
   ])("rejects %s", (_label, raw, refusal) => {
     expect(() => parseScenario(raw, "refused.yml")).toThrow(refusal);
   });

test/e2e/fuzz.ts

@@ -532,7 +532,6 @@ async function witnessIteration(
   };
   const scenario: Scenario = {
     name: `fuzz-witness-${key}-${kind}-${mode}-${seed}`,
-    tiers: ["mock"],
     settings: settingsDoc,
     inputs: { mode, on_missing_permission: "warn" },
     denial_style: "fine_grained",
@@ -560,7 +559,6 @@ async function rejectionIteration(seed: number, spec: RejectionSpec): Promise<It
   const rng = new Rng(seed);
   const scenario: Scenario = {
     name: `fuzz-input-${spec.label}-${seed}`,
-    tiers: ["mock"],
     ...(spec.settingsRaw === undefined
       ? { settings: spec.settings }
       : { settings_raw: spec.settingsRaw }),
@@ -650,7 +648,6 @@ async function singleShotChaosIteration(
 ): Promise<IterationResult> {
   const scenario: Scenario = {
     name: `fuzz-chaos-single-${seed}`,
-    tiers: ["mock"],
     // Fully granted by default, so the list read reaches the mock and the corruption fires.
     settings: { labels: [{ name: "chaos", color: "d73a4a" }] },
     inputs: { mode: "apply" },
@@ -689,7 +686,6 @@ async function persistentChaosIteration(
 ): Promise<IterationResult> {
   const scenario: Scenario = {
     name: `fuzz-chaos-persist-${seed}`,
-    tiers: ["mock"],
     settings: { labels: [{ name: "chaos", color: "d73a4a" }] },
     inputs: { mode: "apply" },
     denial_style: "fine_grained",
@@ -1419,7 +1415,6 @@ async function faultedSectionRun(seed: number, plan: SectionFaultPlan): Promise<
   };
   const scenario: Scenario = {
     name: `fuzz-fault-${plan.section}-${plan.kind}-x${plan.exhausting ? RETRY_BUDGET : 1}-${seed}`,
-    tiers: ["mock"],
     settings,
     inputs: { mode: plan.mode, on_missing_permission: "warn" },
     denial_style: "fine_grained",
@@ -1456,7 +1451,6 @@ async function unfaultableSectionRun(
   }
   const scenario: Scenario = {
     name: `fuzz-unfaultable-${section}-${seed}`,
-    tiers: ["mock"],
     settings: { [section]: UNFAULTABLE_APPLY_SETTINGS[section] },
     inputs: { mode: "apply", on_missing_permission: "warn" },
     denial_style: "fine_grained",
@@ -1638,7 +1632,6 @@ async function preflightFaultRun(seed: number, surviving: boolean): Promise<Iter
   const times = surviving ? 2 * RETRY_BUDGET : RETRY_BUDGET;
   const scenario: Scenario = {
     name: `fuzz-preflight-fault-${surviving ? "survives" : "consumed"}-${seed}`,
-    tiers: ["mock"],
     settings,
     inputs: { mode: "apply", on_missing_permission: "fail" },
     denial_style: "fine_grained",

test/e2e/generators/discovery-scenario.ts

@@ -115,7 +115,6 @@ export function genDiscoveryScenario(
   const privateRepos = "redact" as const;
   const scenario: Scenario = {
     name: `fuzz-discovery-${rng.seed}`,
-    tiers: ["mock"],
     settings: {},
     inputs: { mode: "apply", on_missing_permission: "warn", private_repos: privateRepos },
     denial_style: "fine_grained",

test/e2e/generators/generators.test.ts

@@ -663,6 +663,34 @@ describe("genInvalidSettings", () => {
     }
   });
 
+  test("no token is satisfied by a rejection that names only other keys", () => {
+    // A token that is a fragment of the error's own boilerplate (the known-sections list, the directive advice)
+    // would be found whichever key the run refused, so its case could never fail.
+    const decoy = validateSettingsDoc(
+      { zzz_decoy: [], _zzz: "x" },
+      "settings.yml",
+      SectionSelection.ALL,
+      silentIo(),
+    );
+    const boilerplate = describeProblem(decoy._unsafeUnwrapErr());
+    // The control: the decoy prints both boilerplates, so the bare fragments a token must not be are found in it.
+    expect(["label", "_layerin"].filter((bare) => boilerplate.includes(bare))).toEqual([
+      "label",
+      "_layerin",
+    ]);
+
+    const satisfied = new Set<string>();
+    for (const { name, build } of INVALID_SETTINGS_CASES) {
+      for (let i = 0; i < 200; i++) {
+        const { offendingToken } = build(new Rng(i * 13 + 1));
+        if (boilerplate.includes(offendingToken)) {
+          satisfied.add(`${name}: ${offendingToken}`);
+        }
+      }
+    }
+    expect([...satisfied]).toEqual([]);
+  });
+
   test("draws every catalog case over seeds", () => {
     const drawn = new Set<string>();
     for (let i = 0; i < 400; i++) {

test/e2e/generators/invalid-settings.ts

@@ -15,7 +15,8 @@ import { genSettings, SECRET_LIST_SECTIONS } from "./settings.js";
 /**
  * Only violations validateSettingsDoc GENUINELY rejects belong here; values the loose shapes accept by design would
  * assert failures the contract does not promise, so they stay out.
- *   offendingToken  -> must appear in the rejection error: a section path ("labels[2].name"), an unknown key, or a wording fragment
+ *   offendingToken  -> must appear in the rejection error, and in no rejection that names only other keys: a section
+ *                      path ("labels[2].name"), a section's own issue ("pages: "), or a wording fragment
  *   stays out       -> unknown nested keys under a loose shape, un-modeled enums, arbitrary types on loose keys,
  *                      `pages: null`
  */
@@ -75,6 +76,11 @@ const GITHUB_NAMED_SECTIONS = [
 /** A hyphen, a leading digit, and the reserved prefix in both cases; `github_token` folds to GITHUB_TOKEN once uppercased. */
 const REFUSED_GITHUB_NAMES = ["my-secret", "2_TOKEN", "GITHUB_TOKEN", "github_token"] as const;
 
+/** The start of a section's own issue, `<key>: <message>`; the bare key would also be found in the known-sections list an unknown-key refusal prints. */
+function sectionToken(key: SectionKey): string {
+  return `${key}: `;
+}
+
 /** Entries come back by reference, so a case's mutation lands inside whichever form was drawn; itemToken spells that form's validator path. */
 function validItems(
   rng: Rng,
@@ -94,36 +100,49 @@ export const INVALID_SETTINGS_CASES: ReadonlyArray<{
   {
     name: "unknown-top-level-key",
     build: (rng) => {
+      // The refusal lists every known section after the typo, so a bare "label" would be found in "labels" whichever key was refused.
       const typo = rng.pick(["labelz", "label", "milestone", "repositories", "branch"]);
       return {
         doc: { labels: genSettings(rng.fork("labels"), "labels") as Json, [typo]: [] },
-        offendingToken: typo,
+        offendingToken: `unknown top-level section: ${typo} (known:`,
       };
     },
   },
   {
     name: "unknown-underscore-key",
     build: (rng) => {
       // The underscore is the two directives' and nothing else's: a note or a misspelled directive is rejected, never dropped.
-      const key = rng.pick(["_notes", "_owner", "_layerin", "_undeclared"]);
+      // The advice after an unknown key quotes both directives, so each token carries the wording around its own key.
+      const [key, offendingToken] = rng.pick<[string, string]>([
+        ["_notes", "unknown underscore key: _notes."],
+        ["_owner", "unknown underscore key: _owner."],
+        ["_layerin", "unknown underscore key: _layerin."],
+        [UNDECLARED_KEY, `${UNDECLARED_KEY} must be one of`],
+      ]);
       return {
         doc: { labels: genSettings(rng.fork("labels"), "labels") as Json, [key]: "x" },
-        offendingToken: key,
+        offendingToken,
       };
     },
   },
   {
     name: "array-section-wrong-type",
     build: (rng) => {
       const key = rng.pick(LIST_SECTIONS);
-      return { doc: { [key]: rng.pick([{ not: "an array" }, "oops", 7]) }, offendingToken: key };
+      return {
+        doc: { [key]: rng.pick([{ not: "an array" }, "oops", 7]) },
+        offendingToken: sectionToken(key),
+      };
     },
   },
   {
     name: "record-section-wrong-type",
     build: (rng) => {
       const key = rng.pick(RECORD_SECTIONS);
-      return { doc: { [key]: rng.pick(["oops", 7, [1], null] as const) }, offendingToken: key };
+      return {
+        doc: { [key]: rng.pick(["oops", 7, [1], null] as const) },
+        offendingToken: sectionToken(key),
+      };
     },
   },
   {
@@ -161,15 +180,15 @@ export const INVALID_SETTINGS_CASES: ReadonlyArray<{
     name: "pages-wrong-type",
     build: (rng) => ({
       doc: { pages: rng.pick(["gh-pages", [1]] as const) },
-      offendingToken: "pages",
+      offendingToken: sectionToken("pages"),
     }),
   },
   {
     // null is NOT in the pick list: it is a valid declared value (clear).
     name: "interaction-limits-wrong-type",
     build: (rng) => ({
       doc: { interaction_limits: rng.pick(["oops", 7, [1]] as const) },
-      offendingToken: "interaction_limits",
+      offendingToken: sectionToken("interaction_limits"),
     }),
   },
   {

test/e2e/generators/merge-scenario.ts

@@ -1184,7 +1184,6 @@ export function genMergeScenario(
   const top = layers[count - 1] as MergeLayer;
   const scenario: Scenario = {
     name: `fuzz-merge-${rng.seed}`,
-    tiers: ["mock"],
     settings: top.doc,
     settings_layers: layers.slice(0, -1).map((layer) => layer.doc),
     inputs: {

test/e2e/generators/multi-scenario.ts

@@ -485,7 +485,6 @@ export function genMultiScenario(
 
   const scenario: Scenario = {
     name: `fuzz-multi-${rng.seed}`,
-    tiers: ["mock"],
     settings: {},
     inputs: {
       mode,