diff --git a/.changeset/mcp-result-field-values.md b/.changeset/mcp-result-field-values.md new file mode 100644 index 000000000..e8bdf5241 --- /dev/null +++ b/.changeset/mcp-result-field-values.md @@ -0,0 +1,5 @@ +--- +"@testplanit/mcp-server": patch +--- + +`testplanit_test_run_results_create` now accepts an optional `fieldValues: [{ name, value }]` input to record custom Result Field entries alongside the result. Resolution is by display name (case-insensitive) or system name, scoped to the case's template; unknown names are rejected with the available field list in the error message. This unblocks result submission against templates that mark any Result Field required — previously the server rejected those submissions with `REQUIRED_FIELDS_MISSING` because the tool surface couldn't construct a valid payload. diff --git a/packages/mcp-server/src/tools/runs/results/create.test.ts b/packages/mcp-server/src/tools/runs/results/create.test.ts new file mode 100644 index 000000000..e6c8fb30f --- /dev/null +++ b/packages/mcp-server/src/tools/runs/results/create.test.ts @@ -0,0 +1,284 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; + +// Mock global fetch — submit-result is reached via a raw fetch. +const mockFetch = vi.fn(); +global.fetch = mockFetch as unknown as typeof fetch; + +vi.mock("../../../api.js", () => ({ + zenstack: vi.fn(), +})); + +import { zenstack } from "../../../api.js"; +import { registerRunResultsCreate } from "./create.js"; + +const mockZenstack = vi.mocked(zenstack); + +const mockEnv = { + apiUrl: "https://testplanit.example.com", + apiToken: "tpi_testtoken", +}; + +const deps = { env: mockEnv }; + +const okSubmitResponse = (id: number) => ({ + ok: true, + status: 200, + text: async () => JSON.stringify({ result: { id } }), +}); + +const errorSubmitResponse = (status: number, body: unknown) => ({ + ok: false, + status, + text: async () => (typeof body === "string" ? body : JSON.stringify(body)), +}); + +function makeRawDetail(overrides: Record = {}) { + return { + id: 999, + statusId: 1, + status: { id: 1, name: "Passed" }, + executedBy: { id: "u1", name: "Alice", email: "a@b" }, + editedBy: null, + editedAt: null, + executedAt: "2026-02-01T00:00:00.000Z", + attempt: 1, + elapsed: null, + notes: null, + evidence: null, + testRunCase: { + id: 50, + repositoryCaseId: 100, + repositoryCase: { id: 100, name: "Case 1", source: "MANUAL" }, + testRun: { id: 7, name: "Run A" }, + }, + attachments: [], + issues: [], + resultFieldValues: [], + stepResults: [], + ...overrides, + }; +} + +const RUN_CASE = { + id: 50, + testRunId: 7, + testRun: { projectId: 1 }, + repositoryCase: { templateId: 12 }, +}; + +async function setupClient() { + const server = new McpServer({ name: "test", version: "0.0.0" }); + registerRunResultsCreate(server, deps); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + const client = new Client({ name: "test-client", version: "0.0.0" }); + await server.connect(serverTransport); + await client.connect(clientTransport); + return { client }; +} + +describe("registerRunResultsCreate", () => { + beforeEach(() => { + mockZenstack.mockReset(); + mockFetch.mockReset(); + }); + + it("creates a minimal result (no fieldValues): looks up runCase, status, count, re-fetch", async () => { + // 1. testRunCases.findUnique + mockZenstack.mockResolvedValueOnce(RUN_CASE); + // 2. status.findMany + mockZenstack.mockResolvedValueOnce([{ id: 5 }]); + // 3. testRunResults.count + mockZenstack.mockResolvedValueOnce(2); + // submit-result via fetch + mockFetch.mockResolvedValueOnce(okSubmitResponse(999)); + // 4. testRunResults.findUnique (re-fetch) + mockZenstack.mockResolvedValueOnce(makeRawDetail()); + + const { client } = await setupClient(); + const result = await client.callTool({ + name: "testplanit_test_run_results_create", + arguments: { testRunCaseId: 50, statusName: "Passed" }, + }); + + expect(result.isError).toBeFalsy(); + expect(mockFetch).toHaveBeenCalledTimes(1); + const body = JSON.parse(mockFetch.mock.calls[0]?.[1].body as string); + expect(body).toMatchObject({ + testRunId: 7, + testRunCaseId: 50, + statusId: 5, + attempt: 3, + }); + expect(body.fieldValues).toBeUndefined(); + }); + + it("resolves fieldValues by displayName (case-insensitive) → fieldId; sends them in the submit payload", async () => { + mockZenstack.mockResolvedValueOnce(RUN_CASE); + mockZenstack.mockResolvedValueOnce([{ id: 5 }]); + // templateResultAssignment.findMany + mockZenstack.mockResolvedValueOnce([ + { + resultField: { + id: 11, + displayName: "Defect Severity", + systemName: "defect_severity", + }, + }, + { + resultField: { + id: 12, + displayName: "Reproducibility", + systemName: "reproducibility", + }, + }, + ]); + mockZenstack.mockResolvedValueOnce(0); // count + mockFetch.mockResolvedValueOnce(okSubmitResponse(999)); + mockZenstack.mockResolvedValueOnce(makeRawDetail()); + + const { client } = await setupClient(); + const result = await client.callTool({ + name: "testplanit_test_run_results_create", + arguments: { + testRunCaseId: 50, + statusName: "Failed", + fieldValues: [ + { name: "defect severity", value: "High" }, // case-insensitive displayName + { name: "reproducibility", value: 3 }, // number coerced to string + ], + }, + }); + + expect(result.isError).toBeFalsy(); + const body = JSON.parse(mockFetch.mock.calls[0]?.[1].body as string); + expect(body.fieldValues).toEqual([ + { fieldId: 11, value: "High" }, + { fieldId: 12, value: "3" }, + ]); + }); + + it("resolves fieldValues by systemName when displayName doesn't match", async () => { + mockZenstack.mockResolvedValueOnce(RUN_CASE); + mockZenstack.mockResolvedValueOnce([{ id: 5 }]); + mockZenstack.mockResolvedValueOnce([ + { + resultField: { + id: 11, + displayName: "Defect Severity", + systemName: "severity", + }, + }, + ]); + mockZenstack.mockResolvedValueOnce(0); + mockFetch.mockResolvedValueOnce(okSubmitResponse(999)); + mockZenstack.mockResolvedValueOnce(makeRawDetail()); + + const { client } = await setupClient(); + const result = await client.callTool({ + name: "testplanit_test_run_results_create", + arguments: { + testRunCaseId: 50, + statusName: "Passed", + fieldValues: [{ name: "severity", value: "Low" }], + }, + }); + + expect(result.isError).toBeFalsy(); + const body = JSON.parse(mockFetch.mock.calls[0]?.[1].body as string); + expect(body.fieldValues).toEqual([{ fieldId: 11, value: "Low" }]); + }); + + it("returns isError with the available field list when a fieldValues name doesn't match", async () => { + mockZenstack.mockResolvedValueOnce(RUN_CASE); + mockZenstack.mockResolvedValueOnce([{ id: 5 }]); + mockZenstack.mockResolvedValueOnce([ + { + resultField: { + id: 11, + displayName: "Defect Severity", + systemName: "defect_severity", + }, + }, + ]); + + const { client } = await setupClient(); + const result = await client.callTool({ + name: "testplanit_test_run_results_create", + arguments: { + testRunCaseId: 50, + statusName: "Passed", + fieldValues: [{ name: "Unknown Field", value: "x" }], + }, + }); + + expect(result.isError).toBe(true); + const text = (result.content as Array<{ text: string }>)[0].text; + expect(text).toMatch(/Unknown result field name\(s\): Unknown Field/); + expect(text).toMatch(/Defect Severity/); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it("surfaces REQUIRED_FIELDS_MISSING from the server unchanged (gate intact)", async () => { + mockZenstack.mockResolvedValueOnce(RUN_CASE); + mockZenstack.mockResolvedValueOnce([{ id: 5 }]); + mockZenstack.mockResolvedValueOnce(0); // count (no fieldValues path, no template lookup) + mockFetch.mockResolvedValueOnce( + errorSubmitResponse(400, { + error: "A required result field is missing a value", + code: "REQUIRED_FIELDS_MISSING", + }) + ); + + const { client } = await setupClient(); + const result = await client.callTool({ + name: "testplanit_test_run_results_create", + arguments: { testRunCaseId: 50, statusName: "Passed" }, + }); + + expect(result.isError).toBe(true); + const text = (result.content as Array<{ text: string }>)[0].text; + expect(text).toMatch(/required result field is missing/); + }); + + it("rejects fieldValues when the case has no template (templateId null)", async () => { + mockZenstack.mockResolvedValueOnce({ + ...RUN_CASE, + repositoryCase: { templateId: null }, + }); + mockZenstack.mockResolvedValueOnce([{ id: 5 }]); + + const { client } = await setupClient(); + const result = await client.callTool({ + name: "testplanit_test_run_results_create", + arguments: { + testRunCaseId: 50, + statusName: "Passed", + fieldValues: [{ name: "Severity", value: "High" }], + }, + }); + + expect(result.isError).toBe(true); + const text = (result.content as Array<{ text: string }>)[0].text; + expect(text).toMatch(/no template/i); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it("returns the not-found error when the testRunCase doesn't exist", async () => { + mockZenstack.mockResolvedValueOnce(null); + + const { client } = await setupClient(); + const result = await client.callTool({ + name: "testplanit_test_run_results_create", + arguments: { testRunCaseId: 999, statusName: "Passed" }, + }); + + expect(result.isError).toBe(true); + const text = (result.content as Array<{ text: string }>)[0].text; + expect(text).toMatch(/TestRunCase 999 not found/); + expect(mockFetch).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/mcp-server/src/tools/runs/results/create.ts b/packages/mcp-server/src/tools/runs/results/create.ts index 134202620..241b17264 100644 --- a/packages/mcp-server/src/tools/runs/results/create.ts +++ b/packages/mcp-server/src/tools/runs/results/create.ts @@ -33,18 +33,16 @@ async function submitResult( elapsed: number | null; attempt: number; testRunCaseVersion: number; + fieldValues: Array<{ fieldId: number; value: string }> | undefined; }, - env: EnvConfig, + env: EnvConfig ): Promise<{ result: { id: number } }> { - const response = await fetch( - `${env.apiUrl}/api/test-runs/submit-result`, - { - method: "POST", - headers: bearerHeaders(env), - body: JSON.stringify(payload), - signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), - }, - ); + const response = await fetch(`${env.apiUrl}/api/test-runs/submit-result`, { + method: "POST", + headers: bearerHeaders(env), + body: JSON.stringify(payload), + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + }); const text = await response.text(); if (!response.ok) { let parsedMessage: string | undefined; @@ -62,7 +60,7 @@ async function submitResult( } throw new TestPlanItHttpError( `HTTP ${response.status} from /api/test-runs/submit-result${parsedMessage ? `: ${parsedMessage}` : ""}`, - { statusCode: response.status }, + { statusCode: response.status } ); } return JSON.parse(text) as { result: { id: number } }; @@ -70,13 +68,13 @@ async function submitResult( export function registerRunResultsCreate( server: McpServer, - deps: RunResultsCreateDeps, + deps: RunResultsCreateDeps ): void { server.registerTool( "testplanit_test_run_results_create", { description: - "Submit a test result for a case in a run. Atomically creates the result record and updates the run case's current status. The attempt number is auto-incremented — no need to track it manually. Returns the full denormalized result (same shape as testplanit_test_run_results_get).", + "Submit a test result for a case in a run. Atomically creates the result record and updates the run case's current status. The attempt number is auto-incremented — no need to track it manually. Optional `fieldValues` records custom Result Field entries alongside the result; pass either the field's display name or its system name. The server rejects the submission if the case's template marks any Result Field required and `fieldValues` does not supply each one. Returns the full denormalized result (same shape as testplanit_test_run_results_get).", inputSchema: { testRunCaseId: z .number() @@ -87,7 +85,7 @@ export function registerRunResultsCreate( .string() .min(1) .describe( - "Status name (e.g. 'Passed', 'Failed', 'Blocked'). Must match a status enabled for the project.", + "Status name (e.g. 'Passed', 'Failed', 'Blocked'). Must match a status enabled for the project." ), notes: z .string() @@ -100,15 +98,37 @@ export function registerRunResultsCreate( .nullable() .optional() .describe("Elapsed time in milliseconds, or null."), + fieldValues: z + .array( + z.object({ + name: z + .string() + .min(1) + .describe( + "Display name (case-insensitive) or system name of a Result Field assigned to the case's template." + ), + value: z + .union([z.string(), z.number(), z.boolean()]) + .describe( + "Value to record. Strings/numbers/booleans are coerced to strings and stored as-is — pass the option's name for dropdowns." + ), + }) + ) + .optional() + .describe( + "Custom Result Field values to record alongside the result. Required when the case's template marks any Result Field required." + ), }, }, async (input) => { try { - // Fetch the test run case to get testRunId and projectId. + // Fetch the test run case to get testRunId, projectId, and templateId + // (templateId is needed to resolve fieldValues by name). const runCase = await zenstack<{ id: number; testRunId: number; testRun: { projectId: number }; + repositoryCase: { templateId: number | null }; } | null>( "testRunCases", "findUnique", @@ -118,9 +138,10 @@ export function registerRunResultsCreate( id: true, testRunId: true, testRun: { select: { projectId: true } }, + repositoryCase: { select: { templateId: true } }, } satisfies Prisma.TestRunCasesSelect, }, - deps.env, + deps.env ); if (!runCase) { @@ -149,7 +170,7 @@ export function registerRunResultsCreate( select: { id: true } satisfies Prisma.StatusSelect, take: 1, }, - deps.env, + deps.env ); if (!statuses || statuses.length === 0) { @@ -164,6 +185,97 @@ export function registerRunResultsCreate( }; } + // Resolve fieldValues input — map field names to numeric fieldIds via + // the case's template result-field assignments. The server enforces + // REQUIRED_FIELDS_MISSING; this step just translates names → IDs so the + // agent doesn't need to know the numeric IDs. + let serverFieldValues: + | Array<{ fieldId: number; value: string }> + | undefined; + + if (input.fieldValues && input.fieldValues.length > 0) { + if (runCase.repositoryCase.templateId == null) { + return { + isError: true as const, + content: [ + { + type: "text" as const, + text: "Cannot resolve fieldValues: the case has no template. Remove fieldValues, or assign a template to the case.", + }, + ], + }; + } + + const assignments = await zenstack< + Array<{ + resultField: { + id: number; + displayName: string; + systemName: string; + }; + }> + >( + "templateResultAssignment", + "findMany", + { + where: { + templateId: runCase.repositoryCase.templateId, + resultField: { isEnabled: true, isDeleted: false }, + }, + select: { + resultField: { + select: { + id: true, + displayName: true, + systemName: true, + }, + }, + } satisfies Prisma.TemplateResultAssignmentSelect, + }, + deps.env + ); + + const byDisplay = new Map(); + const bySystem = new Map(); + for (const a of assignments ?? []) { + byDisplay.set( + a.resultField.displayName.toLowerCase(), + a.resultField.id + ); + bySystem.set(a.resultField.systemName, a.resultField.id); + } + + const unresolved: string[] = []; + const resolved: Array<{ fieldId: number; value: string }> = []; + for (const fv of input.fieldValues) { + const id = + byDisplay.get(fv.name.toLowerCase()) ?? bySystem.get(fv.name); + if (id === undefined) { + unresolved.push(fv.name); + continue; + } + resolved.push({ fieldId: id, value: String(fv.value) }); + } + + if (unresolved.length > 0) { + const available = + (assignments ?? []) + .map((a) => a.resultField.displayName) + .join(", ") || "(none assigned to this template)"; + return { + isError: true as const, + content: [ + { + type: "text" as const, + text: `Unknown result field name(s): ${unresolved.join(", ")}. Available fields on this template: ${available}.`, + }, + ], + }; + } + + serverFieldValues = resolved; + } + // Auto-increment attempt: count existing non-deleted results. const existingCount = await zenstack( "testRunResults", @@ -174,7 +286,7 @@ export function registerRunResultsCreate( isDeleted: false, } satisfies Prisma.TestRunResultsWhereInput, }, - deps.env, + deps.env ); const attempt = (existingCount ?? 0) + 1; @@ -187,8 +299,9 @@ export function registerRunResultsCreate( elapsed: input.elapsed ?? null, attempt, testRunCaseVersion: 1, + fieldValues: serverFieldValues, }, - deps.env, + deps.env ); // Re-fetch the created result with the full denormalized shape. @@ -199,7 +312,7 @@ export function registerRunResultsCreate( where: { id: result.id }, include: RUN_RESULT_DETAIL_INCLUDE, }, - deps.env, + deps.env ); if (!raw) { @@ -222,6 +335,6 @@ export function registerRunResultsCreate( } catch (err) { return mapHttpErrorToToolResult(err); } - }, + } ); } diff --git a/packages/mcp-server/src/tools/runs/results/index.test.ts b/packages/mcp-server/src/tools/runs/results/index.test.ts index b8c955218..5f461e9e8 100644 --- a/packages/mcp-server/src/tools/runs/results/index.test.ts +++ b/packages/mcp-server/src/tools/runs/results/index.test.ts @@ -5,7 +5,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { registerRunResults } from "./index.js"; describe("registerRunResults", () => { - it("registers list + get tools for run results", async () => { + it("registers list + get + create tools for run results", async () => { const server = new McpServer({ name: "test", version: "0.0.0" }); registerRunResults(server, { env: { apiUrl: "https://x", apiToken: "tpi_x" }, @@ -20,5 +20,6 @@ describe("registerRunResults", () => { const names = tools.tools.map((t) => t.name); expect(names).toContain("testplanit_test_run_results_list"); expect(names).toContain("testplanit_test_run_results_get"); + expect(names).toContain("testplanit_test_run_results_create"); }); });