test(e2e): check GraphQL variables against GitHub's schema in the mock

.github/scripts/gen-gaps-index.ts

@@ -107,7 +107,7 @@ export const UNDOCUMENTED_ROUTES: readonly (SupplementalRoute | SpecOnlyRoute)[]
 /**
  * The SDL GitHub's GraphQL API serves but the pinned @octokit/graphql-schema
  * release lacks, each with the gap file that carries it. Consumed by
- * test/sections/graphql-queries.test.ts, which extends the published schema
+ * test/e2e/mock/graphql-schema.ts, where the published schema is extended
  * with it before validating the declared queries.
  */
 export const UNSHIPPED_GRAPHQL_SDL: readonly UnshippedGraphqlSdl[] = unshippedGraphqlSdl(GAPS);

src/generated/upstream-gaps.ts

@@ -42,7 +42,7 @@ export const UNDOCUMENTED_ROUTES: readonly (SupplementalRoute | SpecOnlyRoute)[]
 /**
  * The SDL GitHub's GraphQL API serves but the pinned @octokit/graphql-schema
  * release lacks, each with the gap file that carries it. Consumed by
- * test/sections/graphql-queries.test.ts, which extends the published schema
+ * test/e2e/mock/graphql-schema.ts, where the published schema is extended
  * with it before validating the declared queries.
  */
 export const UNSHIPPED_GRAPHQL_SDL: readonly UnshippedGraphqlSdl[] = unshippedGraphqlSdl(GAPS);

test/e2e/mock/graphql-pipeline.test.ts

@@ -32,43 +32,57 @@ function scenario(overrides: Partial<Scenario> = {}): Scenario {
 }
 
 // Fixture operations, tagged onto the real repository section so the permission gate resolves its real requirement (administration).
+// Each is a document GitHub's schema accepts, named as its declaration names it: the pipeline refuses any other before a barrier runs.
+
+function repoIdQuery(name: string) {
+  return `query ${name}($owner: String!, $repo: String!) { repository(owner: $owner, name: $repo) { id } }` as const;
+}
 
 const G_READ: TaggedGraphqlOp = {
   name: "RepoToggles",
   kind: "read",
-  query:
-    "query RepoToggles($owner: String!, $repo: String!) { repository(owner: $owner, name: $repo) { id } }",
+  query: repoIdQuery("RepoToggles"),
   outcomes: { ok: "the toggles" },
   section: "repository",
   role: "gToggles",
 };
 const G_READ_TOLERANT: TaggedGraphqlOp = {
   ...G_READ,
   name: "RepoTogglesProbe",
+  query: repoIdQuery("RepoTogglesProbe"),
   role: "gProbe",
   outcomes: { ok: "the toggles", NOT_FOUND: "the feature is off" },
 };
 const G_READ_EXECUTION: TaggedGraphqlOp = {
   ...G_READ,
   name: "RepoNodeId",
+  query: repoIdQuery("RepoNodeId"),
   role: "gNodeId",
   phase: "execution",
 };
 const G_WRITE: TaggedGraphqlOp = {
   name: "UpdateToggles",
   kind: "write",
   query:
-    "mutation UpdateToggles($repositoryId: ID!, $hasWiki: Boolean!) { updateRepository(input: {repositoryId: $repositoryId, hasWiki: $hasWiki}) { clientMutationId } }",
+    "mutation UpdateToggles($repositoryId: ID!, $hasWikiEnabled: Boolean) { updateRepository(input: {repositoryId: $repositoryId, hasWikiEnabled: $hasWikiEnabled}) { clientMutationId } }",
   outcomes: { ok: "toggles updated" },
   section: "repository",
   role: "gUpdate",
 };
+const G_WRITE_NESTED: TaggedGraphqlOp = {
+  ...G_WRITE,
+  name: "UpdateTogglesInput",
+  query:
+    "mutation UpdateTogglesInput($input: UpdateRepositoryInput!) { updateRepository(input: $input) { clientMutationId } }",
+  role: "gUpdateInput",
+};
 
 const OPS: Readonly<Record<string, TaggedGraphqlOp>> = {
   "repository.gToggles": G_READ,
   "repository.gProbe": G_READ_TOLERANT,
   "repository.gNodeId": G_READ_EXECUTION,
   "repository.gUpdate": G_WRITE,
+  "repository.gUpdateInput": G_WRITE_NESTED,
 };
 
 const HANDLERS: Record<string, GraphqlHandler> = {
@@ -80,7 +94,11 @@ const HANDLERS: Record<string, GraphqlHandler> = {
     data: { repository: { id: String(state.repo.node_id) } },
   }),
   "repository.gUpdate": ({ state, variables }) => {
-    state.repo.has_wiki = variables.hasWiki === true;
+    state.repo.has_wiki = variables.hasWikiEnabled === true;
+    return { data: { updateRepository: { clientMutationId: null } } };
+  },
+  "repository.gUpdateInput": ({ state, variables }) => {
+    state.repo.has_wiki = (variables.input as Json).hasWikiEnabled === true;
     return { data: { updateRepository: { clientMutationId: null } } };
   },
 };
@@ -171,7 +189,7 @@ describe("GraphQL dispatch and logging", () => {
     const opts = options(scenario());
     const state = opts.working.mode === "single" ? opts.working.state : undefined;
     const id = mintNodeId("repo", ADMIN_SLUG, "");
-    const result = dispatch(G_WRITE, { repositoryId: id, hasWiki: true }, opts);
+    const result = dispatch(G_WRITE, { repositoryId: id, hasWikiEnabled: true }, opts);
     expect(result.violation).toBeUndefined();
     expect(state?.repo.has_wiki).toBe(true);
     expect(isWriteRequest(result.log)).toBe(true);
@@ -191,12 +209,34 @@ describe("GraphQL dispatch and logging", () => {
       violation: ['node ids of "acme/other"', `serves only "${ADMIN_SLUG}"`],
     },
   ])("a single-mode mutation $name is a violation", ({ repositoryId, violation }) => {
-    const result = dispatch(G_WRITE, { repositoryId, hasWiki: true }, options(scenario()));
+    const result = dispatch(G_WRITE, { repositoryId, hasWikiEnabled: true }, options(scenario()));
     for (const fragment of violation) {
       expect(result.violation).toContain(fragment);
     }
   });
 
+  test("control: the handler receives the variables exactly as sent, an undeclared one included", () => {
+    // The schema check coerces to judge, never to rewrite: the coerced variables would lack `stray`.
+    let received: unknown;
+    const handlers: Record<string, GraphqlHandler> = {
+      ...HANDLERS,
+      "repository.gToggles": ({ state, variables }) => {
+        received = variables;
+        return { data: { repository: { id: String(state.repo.node_id) } } };
+      },
+    };
+    const body = gqlBody(G_READ, { owner: OWNER, repo: REPO, stray: 1 });
+    const result = handleGraphqlRequest(
+      { method: "POST", body },
+      options(scenario()),
+      baseLog(body),
+      OPS,
+      handlers,
+    );
+    expect(result.violation).toBeUndefined();
+    expect(received).toEqual({ owner: OWNER, repo: REPO, stray: 1 });
+  });
+
   test("sectionForRequest attributes a /graphql request through its body", () => {
     // The live registry declares no "RepoToggles", so attribution resolves null and the REST
     // fallback stays intact.
@@ -205,6 +245,34 @@ describe("GraphQL dispatch and logging", () => {
   });
 });
 
+describe("GraphQL schema check", () => {
+  test("a request GitHub's schema refuses is answered ahead of the check-mode, fault, and permission barriers", () => {
+    // Behind any of the three, a check-mode, faulted, or denied request would hide variables GitHub refuses.
+    const opts = options(
+      scenario({ inputs: { mode: "check" }, token_permissions: { administration: "none" } }),
+      { faults: [{ key: "repository.gUpdate", kind: "server_error" }] },
+    );
+    const result = dispatch(
+      G_WRITE,
+      { repositoryId: mintNodeId("repo", ADMIN_SLUG, ""), hasWikiEnabled: "yes" },
+      opts,
+    );
+    expect(result.response).toEqual({
+      status: 200,
+      body: {
+        errors: [
+          expect.objectContaining({
+            message: expect.stringContaining("Boolean cannot represent a non boolean value"),
+          }),
+        ],
+      },
+    });
+    expect(result.violation).toStartWith("GraphQL UpdateToggles is refused by GitHub's schema: ");
+    expect(opts.faultCounts.size).toBe(0);
+    expect(result.log.deniedBy).toBeUndefined();
+  });
+});
+
 describe("GraphQL check-mode barrier", () => {
   test("a write in check mode is a violation, before the permission gate", () => {
     const s = scenario({
@@ -316,7 +384,7 @@ describe("GraphQL denial barrier (shared with REST)", () => {
     const wikiBefore = state.repo.has_wiki;
     const write = dispatch(
       G_WRITE,
-      { repositoryId: mintNodeId("repo", ADMIN_SLUG, ""), hasWiki: !wikiBefore },
+      { repositoryId: mintNodeId("repo", ADMIN_SLUG, ""), hasWikiEnabled: !wikiBefore },
       opts,
     );
     expect(write.violation).toBeUndefined();
@@ -343,7 +411,7 @@ describe("GraphQL denial barrier (shared with REST)", () => {
     const wikiBefore = state.repo.has_wiki;
     const write = dispatch(
       G_WRITE,
-      { repositoryId: mintNodeId("repo", ADMIN_SLUG, ""), hasWiki: !wikiBefore },
+      { repositoryId: mintNodeId("repo", ADMIN_SLUG, ""), hasWikiEnabled: !wikiBefore },
       opts,
     );
     expect(write.violation).toBeUndefined();
@@ -377,8 +445,22 @@ describe("GraphQL multi-repo slug resolution", () => {
     expect(denied.log.deniedBy).toBe("administration");
   });
 
-  test("a read without $owner/$repo variables is a violation", () => {
-    const result = dispatch(G_READ, { owner: "acme" }, multiOptions(scenario()));
+  test("a read addressing no repository is a violation", () => {
+    // A read that omits a $repo its document declares is GitHub's own refusal; this one declares no address at all.
+    const unaddressed: TaggedGraphqlOp = {
+      ...G_READ,
+      name: "ViewerLogin",
+      query: "query ViewerLogin { viewer { login } }",
+      role: "gViewer",
+    };
+    const body = gqlBody(unaddressed, {});
+    const result = handleGraphqlRequest(
+      { method: "POST", body },
+      multiOptions(scenario()),
+      baseLog(body),
+      { ...OPS, "repository.gViewer": unaddressed },
+      HANDLERS,
+    );
     expect(result.violation).toContain("carries no $owner/$repo variables");
   });
 
@@ -388,7 +470,8 @@ describe("GraphQL multi-repo slug resolution", () => {
     // The fixture starts every repo at has_wiki: true, so flipping alpha to
     // false is what proves the write landed on alpha and ONLY alpha.
     const id = mintNodeId("repo", "acme/alpha", "");
-    const result = dispatch(G_WRITE, { input: { repositoryId: id }, hasWiki: false }, opts);
+    const input = { repositoryId: id, hasWikiEnabled: false };
+    const result = dispatch(G_WRITE_NESTED, { input }, opts);
     expect(result.violation).toBeUndefined();
     expect(multi?.repos.get("acme/alpha")?.repo.has_wiki).toBe(false);
     expect(multi?.repos.get("acme/beta")?.repo.has_wiki).toBe(true);

test/e2e/mock/graphql-request.test.ts

@@ -0,0 +1,98 @@
+/**
+ * The pipeline answers a GraphQL request GitHub's schema refuses the way GitHub does, with errors and no data, ahead
+ * of every handler, and fails the run. Without it the mock stores what GitHub would have refused and every scenario
+ * passes on it. Pinned over the wire, since a handler unit test bypasses the pipeline.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { allGraphqlOps } from "../../../src/sections/registry.js";
+import { ADMIN_SLUG, ADMIN_OWNER as OWNER, ADMIN_REPO as REPO } from "../constants.js";
+import { sharedValidator } from "../openapi/validate.js";
+import { mintNodeId } from "./node-id.js";
+import { call, json, mockServerLifecycle, scenario, singleState } from "./server-test-support.js";
+
+const start = mockServerLifecycle();
+const OPS = allGraphqlOps();
+const RULE_ID = mintNodeId("rule", ADMIN_SLUG, "release/*");
+const ENVIRONMENT_ID = mintNodeId("environment", ADMIN_SLUG, "prod");
+
+/** One wildcard rule and one pinned environment, so a handler that ran would have something to change. */
+function seeded() {
+  return scenario({
+    live_state: {
+      branch_protection_rules: [{ pattern: "release/*", isAdminEnforced: false }],
+      environments: { prod: { name: "prod", protection_rules: [] } },
+      pinned_environments: ["prod"],
+    },
+  });
+}
+
+describe("a request GitHub's schema refuses is answered with errors and no data, and fails the run", () => {
+  test.each([
+    {
+      name: "an input field the schema lacks (the REST key on a rule mutation)",
+      op: OPS["branches.updateRule"],
+      variables: { input: { branchProtectionRuleId: RULE_ID, enforce_admins: true } },
+      refusal: 'not to include unknown field "enforce_admins"',
+    },
+    {
+      // Unrefused, the pin handler reads a null as "unpin" and drops the pin.
+      name: "null for a variable the operation declares Boolean!",
+      op: OPS["environments.pin"],
+      variables: { environmentId: ENVIRONMENT_ID, pinned: null },
+      refusal: 'Variable "$pinned" has invalid value: Expected value of non-null type "Boolean!"',
+    },
+    {
+      name: "a document selecting a field the schema lacks",
+      op: {
+        ...OPS["branches.repoLookup"],
+        query:
+          "query BranchProtectionRepository($owner: String!, $repo: String!) { repository(owner: $owner, name: $repo) { id nodeId } }",
+      },
+      variables: { owner: OWNER, repo: REPO },
+      refusal: 'Cannot query field "nodeId" on type "Repository"',
+    },
+    {
+      name: "a document that does not parse",
+      op: { ...OPS["branches.repoLookup"], query: "query BranchProtectionRepository {" },
+      variables: { owner: OWNER, repo: REPO },
+      refusal: "Syntax Error: Expected Name, found <EOF>",
+    },
+  ])("$name stores nothing", async ({ op, variables, refusal }) => {
+    const h = await start(seeded());
+    const before = structuredClone(singleState(h));
+    const res = await call(h, "POST", "/graphql", {
+      body: { query: op.query, operationName: op.name, variables },
+    });
+    expect(res.status).toBe(200);
+    // The whole body: GitHub sends no `data` key beside a request error.
+    expect(await json(res)).toEqual({
+      errors: [expect.objectContaining({ message: expect.stringContaining(refusal) })],
+    });
+    expect(singleState(h)).toEqual(before);
+    expect(h.violations).toEqual([
+      expect.stringContaining(`GraphQL ${op.name} is refused by GitHub's schema: `),
+    ]);
+    expect(h.violations[0]).toContain(refusal);
+    // The violation above is the one report: the contract validator must not add its own for the untyped errors.
+    expect(sharedValidator().validateLog(h.requests)).toEqual([]);
+  });
+});
+
+test("control: a request on the schema is answered by its handler, with data and no violation", async () => {
+  const h = await start(seeded());
+  const op = OPS["environments.pin"];
+  const res = await call(h, "POST", "/graphql", {
+    body: {
+      query: op.query,
+      operationName: op.name,
+      variables: { environmentId: ENVIRONMENT_ID, pinned: false },
+    },
+  });
+  expect(res.status).toBe(200);
+  expect(await json(res)).toEqual({
+    data: { pinEnvironment: { environment: { name: "prod", isPinned: false } } },
+  });
+  expect(singleState(h).pinned_environments).toEqual([]);
+  expect(h.violations).toEqual([]);
+});

test/e2e/mock/graphql-request.ts

@@ -0,0 +1,57 @@
+/**
+ * What GitHub refuses of a GraphQL request before it runs any of it: a document that does not validate against its
+ * schema, and variables that do not coerce to the types the operation declares. A handler handed such a request
+ * would store an input field GitHub's schema does not have, and every scenario would pass on it.
+ */
+
+import {
+  type DocumentNode,
+  GraphQLError,
+  type GraphQLFormattedError,
+  parse,
+  validate,
+  validateExecutionArgs,
+} from "graphql";
+import { sharedGraphqlSchema } from "./graphql-schema.js";
+
+/** The document, or the syntax error graphql-js reports by throwing it. */
+function parsedDocument(query: string): DocumentNode | GraphQLError {
+  try {
+    return parse(query);
+  } catch (error) {
+    if (error instanceof GraphQLError) {
+      return error;
+    }
+    throw error;
+  }
+}
+
+/**
+ * The errors GitHub answers `operationName` of `query` with under `variables`, or null when it accepts the request.
+ * graphql-js words them; api.github.com words the same refusals its own way. Coercion only judges: a caller keeps
+ * handing on the variables as sent.
+ */
+export function graphqlRequestErrors(
+  query: string,
+  operationName: string,
+  variables: Readonly<Record<string, unknown>>,
+): readonly GraphQLFormattedError[] | null {
+  const document = parsedDocument(query);
+  if (document instanceof GraphQLError) {
+    return [document.toJSON()];
+  }
+
+  const schema = sharedGraphqlSchema();
+  const documentErrors = validate(schema, document);
+  if (documentErrors.length > 0) {
+    return documentErrors.map((error) => error.toJSON());
+  }
+
+  const prepared = validateExecutionArgs({
+    schema,
+    document,
+    operationName,
+    variableValues: variables,
+  });
+  return "operation" in prepared ? null : prepared.map((error) => error.toJSON());
+}