From 88e3dd8a5602b88eb26d8d5a5a40e9b6f9d1a22f Mon Sep 17 00:00:00 2001 From: Jonathan Pool Date: Mon, 28 Sep 2026 02:55:03 -0700 Subject: [PATCH 1/5] add test and retest ordering for MCP and API --- api/awaitTest.test.ts | 106 ++++++ api/awaitTest.ts | 97 ++++++ api/orderRetest.test.ts | 179 +++++++++++ api/orderRetest.ts | 119 +++++++ api/orderTest.test.ts | 213 ++++++++++++ api/orderTest.ts | 145 +++++++++ api/requestRetest.ts | 43 +-- api/requestTest.ts | 43 +-- api/routes.test.ts | 7 +- api/routes.ts | 40 ++- api/schemas.ts | 104 +++++- api/util.ts | 59 +++- capability.md | 2 +- env.example | 4 + index.test.ts | 39 +++ index.ts | 40 ++- llms-full.txt | 31 ++ llms.txt | 3 + mcp.test.ts | 72 ++++- mcp.ts | 74 ++++- memory/MEMORY.md | 1 + memory/plan_issue_111_instant_test.md | 184 +++++++++++ openapi.yaml | 447 +++++++++++++++++++++++++- package.json | 2 +- util.test.ts | 212 ++++++++++++ util.ts | 163 ++++++++++ 26 files changed, 2345 insertions(+), 84 deletions(-) create mode 100644 api/awaitTest.test.ts create mode 100644 api/awaitTest.ts create mode 100644 api/orderRetest.test.ts create mode 100644 api/orderRetest.ts create mode 100644 api/orderTest.test.ts create mode 100644 api/orderTest.ts create mode 100644 memory/plan_issue_111_instant_test.md diff --git a/api/awaitTest.test.ts b/api/awaitTest.test.ts new file mode 100644 index 00000000..b8122627 --- /dev/null +++ b/api/awaitTest.test.ts @@ -0,0 +1,106 @@ +/* + awaitTest.test.ts + Tests for api/awaitTest.ts using the fixture corpus. +*/ + +// IMPORTS + +import {test, before, after} from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import {fixtureDBDir} from '../test/dbFixture.ts'; + +for (const key of ['MANAGER_EMAIL', 'ALERT_API_HOST', 'ALERT_API_PATH', 'ALERT_API_KEY', 'ALERT_FROM']) { + process.env[key] = ''; +} +// Keep the poll fast in this file, since these tests do not exercise timing edge cases +// (those live in util.test.ts, closer to awaitJob itself); a short interval just keeps +// this file's own tests quick. +process.env.AWAIT_JOB_POLL_MS = '10'; +process.env.AWAIT_JOB_TIMEOUT_MS = '50'; + +// SETUP AND TEARDOWN + +const savedDBDir = process.env.DB_DIR; + +before(() => { + process.env.DB_DIR = fixtureDBDir; +}); + +import {response} from './awaitTest.ts'; + +after(() => { + delete process.env.AWAIT_JOB_POLL_MS; + delete process.env.AWAIT_JOB_TIMEOUT_MS; + if (savedDBDir !== undefined) { + process.env.DB_DIR = savedDBDir; + } + else { + delete process.env.DB_DIR; + } +}); + +// TESTS + +test('awaitTest rejects a malformed timestamp or job identifier', async () => { + const body = await response(['999999T9999', 'xyz']); + const details = body['response content']['details about your wait'] as any; + assert.ok(details.error.includes('malformed')); + assert.equal(body['response content']['disposition of your wait'], null); +}); + +test('awaitTest returns "completed" for an identifier with an already-available report', async () => { + const body = await response(['260101T0000', 'mix']); + const details = body['response content']['details about your wait'] as any; + assert.deepEqual(details['report identifier'], {timeStamp: '260101T0000', jobID: 'mix'}); + const disposition = body['response content']['disposition of your wait'] as any; + assert.equal(disposition.outcome, 'completed'); + assert.ok(disposition['how you can check for completion'].includes('listIssues')); +}); + +test('awaitTest returns an error for an identifier naming neither a report nor a pending job', async () => { + const body = await response(['260101T0000', 'zzz']); + const details = body['response content']['details about your wait'] as any; + assert.ok(details.error.includes('no report')); + assert.equal(body['response content']['disposition of your wait'], null); +}); + +test('awaitTest returns "failed" when the job has moved to the failed directory', async () => { + const failedPath = path.join(fixtureDBDir, 'jobs', 'failed', '260101T4444-flj.json'); + await fs.mkdir(path.dirname(failedPath), {recursive: true}); + await fs.writeFile(failedPath, JSON.stringify({target: {what: 'Failed Page', url: 'https://example.com/failed'}})); + try { + const body = await response(['260101T4444', 'flj']); + const disposition = body['response content']['disposition of your wait'] as any; + assert.equal(disposition.outcome, 'failed'); + assert.equal(disposition['how you can check for completion'], 'Not applicable.'); + } + finally { + await fs.unlink(failedPath); + } +}); + +test('awaitTest returns "timedOut" when a claimed job neither completes nor fails before the deadline', async () => { + const claimedPath = path.join(fixtureDBDir, 'jobs', 'claimed', '260101T5555-toj.json'); + await fs.writeFile(claimedPath, JSON.stringify({target: {what: 'Stuck Page', url: 'https://example.com/stuck'}})); + try { + const body = await response(['260101T5555', 'toj']); + const disposition = body['response content']['disposition of your wait'] as any; + assert.equal(disposition.outcome, 'timedOut'); + assert.ok(disposition['how you can check for completion'].includes('listIssues')); + } + finally { + await fs.unlink(claimedPath); + } +}); + +test('awaitTest response has no web UI equivalents for this request or its ancestor', async () => { + const body = await response(['260101T0000', 'mix']); + const similarWeb = body['URLs of similar requests for web users'] as any; + assert.equal(similarWeb['this request'], null); + assert.equal(similarWeb['closest ancestor request'], null); + const ancestor = (body['this request'] as any)['closest ancestor request']; + assert.equal(ancestor['tool name'], 'orderTest'); + assert.equal(ancestor.method, 'POST'); +}); diff --git a/api/awaitTest.ts b/api/awaitTest.ts new file mode 100644 index 00000000..bbe10182 --- /dev/null +++ b/api/awaitTest.ts @@ -0,0 +1,97 @@ +/* + awaitTest.ts + Waits for a job ordered via orderTest to complete, and returns its outcome. +*/ + +// IMPORTS + +import {z} from 'zod'; +import {getResponseMetadata, getThisHost, getToolsFacts} from './util.ts'; +import {awaitJob, isJobID, isTimeStamp} from '../util.ts'; +import {awaitTestResponseSchema} from './schemas.ts'; + +// TYPES + +// The response content defined by the response schema. +type ResponseContent = z.infer['response content']; + +// FUNCTIONS + +// Returns the response body. +export const response = async (args: string[]) => { + const [timeStamp = '', jobID = ''] = args; + const thisHost = getThisHost(); + // Initialize the response-content properties. + let requestDetails: ResponseContent['details about your wait']; + let requestDisposition: ResponseContent['disposition of your wait'] = null; + // If the timestamp or job ID is not syntactically valid: + if (!isTimeStamp(timeStamp) || !isJobID(jobID)) { + requestDetails = { + error: 'wait invalid: the report timestamp or job identifier is malformed' + }; + } + // Otherwise, i.e. if the identifier is syntactically valid: + else { + requestDetails = { + 'report identifier': {timeStamp, jobID} + }; + // Wait for the job to complete, fail, or time out. No progress notices are sent + // during this wait; the caller receives only this one, final response. + const outcome = await awaitJob(timeStamp, jobID); + if (outcome === 'notFound') { + requestDetails = { + error: `wait invalid: no report with timeStamp "${timeStamp}" and jobID "${jobID}" is completed, pending, or in progress` + }; + } + else if (outcome === 'completed') { + requestDisposition = { + outcome: 'completed', + 'what happened': 'The report is ready.', + 'how you can check for completion': `Call the listIssues tool with timeStamp "${timeStamp}" and jobID "${jobID}" to retrieve the report.` + }; + } + else if (outcome === 'failed') { + requestDisposition = { + outcome: 'failed', + 'what happened': 'The job failed and no report will be produced.', + 'how you can check for completion': 'Not applicable.' + }; + } + else { + requestDisposition = { + outcome: 'timedOut', + 'what happened': 'The job has not completed within the maximum wait time, but it may still be running.', + 'how you can check for completion': `Call the listIssues tool with timeStamp "${timeStamp}" and jobID "${jobID}" after a few minutes to check whether the report is ready.` + }; + } + } + // Create the response content. + const responseContent: ResponseContent = { + 'details about your wait': requestDetails, + 'disposition of your wait': requestDisposition + }; + // Create a response body. + const body = { + 'tool collection': getToolsFacts(), + 'tool name': 'awaitTest', + 'this request': { + description: 'Wait for a job ordered via the orderTest tool to complete. The timeStamp and jobID parameters are the report identifier that orderTest returned. This call blocks until the report is ready, the job fails, or a maximum wait time elapses.', + method: 'POST', + URL: `${thisHost}/api/awaitTest/${timeStamp}/${jobID}`, + 'closest ancestor request': { + 'tool name': 'orderTest', + description: 'Process my order to test a page about which no report is available yet, immediately and without manual approval.', + method: 'POST', + URL: `${thisHost}/api/orderTest` + } + }, + 'URLs of similar requests for web users': { + 'this request': null, + 'closest ancestor request': null + }, + 'response metadata': getResponseMetadata(), + 'response content': responseContent + }; + // Return it. + return body; +}; diff --git a/api/orderRetest.test.ts b/api/orderRetest.test.ts new file mode 100644 index 00000000..5e126704 --- /dev/null +++ b/api/orderRetest.test.ts @@ -0,0 +1,179 @@ +/* + orderRetest.test.ts + Tests for api/orderRetest.ts using the fixture corpus. +*/ + +// IMPORTS + +import {test, before, beforeEach, after} from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import {fixtureDBDir} from '../test/dbFixture.ts'; + +for (const key of ['MANAGER_EMAIL', 'ALERT_API_HOST', 'ALERT_API_PATH', 'ALERT_API_KEY', 'ALERT_FROM']) { + process.env[key] = ''; +} + +// SETUP AND TEARDOWN + +const savedDBDir = process.env.DB_DIR; +const queuePath = path.join(fixtureDBDir, 'jobs', 'queue'); +// A report added only to this test's disposable copy of the fixture database (never to +// the tracked test/fixtures/db corpus), matching the fixture requestRetest.test.ts uses +// for the same reason: a 3-character job ID satisfying isJobID without perturbing +// report-count tests elsewhere that copy the whole corpus. +const retestFixturePath = path.join(fixtureDBDir, 'reports', '260101T0004-ord.json'); +const retestFixtureReport = { + id: '260101T0004-ord', + what: 'Order Retest Fixture Page', + target: {what: 'Order Retest Fixture Page', url: 'https://example.com/orderretestfixture'}, + sources: {worker: 'test-worker'}, + acts: [], + jobData: {startTime: '26-01-01T00:00', endTime: '26-01-01T00:10', elapsedSeconds: 600, preventions: {}, issuelessRules: []}, + catalog: {}, + images: {}, + checkpoints: [] +}; + +before(async () => { + process.env.DB_DIR = fixtureDBDir; + await fs.writeFile(retestFixturePath, JSON.stringify(retestFixtureReport)); +}); + +beforeEach(async () => { + for (const fileName of await fs.readdir(queuePath)) { + await fs.unlink(path.join(queuePath, fileName)); + } +}); + +import {response} from './orderRetest.ts'; + +after(async () => { + await fs.unlink(retestFixturePath); + if (savedDBDir !== undefined) { + process.env.DB_DIR = savedDBDir; + } + else { + delete process.env.DB_DIR; + } +}); + +// TESTS + +test('orderRetest rejects a nonexistent report', async () => { + const body = await response(['251231T0000', 'zzz', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.ok(details.error.includes('does not exist')); +}); + +test('orderRetest rejects a malformed timestamp or job identifier', async () => { + const body = await response(['999999T9999', 'xyz', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.ok(details.error.includes('malformed')); +}); + +test('orderRetest rejects a reason shorter than 20 characters', async () => { + const body = await response(['260101T0004', 'ord', 'short']); + const details = body['response content']['details about your order'] as any; + assert.ok(details.error.includes('reason')); +}); + +test('orderRetest rejects a reason longer than 100 characters', async () => { + const longReason = 'x'.repeat(101); + const body = await response(['260101T0004', 'ord', longReason]); + const details = body['response content']['details about your order'] as any; + assert.ok(details.error.includes('reason')); +}); + +test('orderRetest rejects a superseded report', async () => { + // 260101T0000-mix is an earlier report of "Mixed Outcomes Page" than 260202T0000-new, + // so it is superseded. + const body = await response(['260101T0000', 'mix', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.equal(details.error, undefined); + const disposition = body['response content']['disposition of your order'] as any; + assert.ok( + disposition['what happens next'].includes('a later report about a page with the same description is available.') + ); + assert.equal(disposition['report identifier'], null); +}); + +test('orderRetest rejects a report matching a claimed job by description', async () => { + const claimedPath = path.join(fixtureDBDir, 'jobs', 'claimed', 'clm.json'); + await fs.writeFile(claimedPath, JSON.stringify({ + target: {what: 'Order Retest Fixture Page', url: 'https://example.com/unrelated'} + })); + try { + const body = await response(['260101T0004', 'ord', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.equal(details.error, undefined); + const disposition = body['response content']['disposition of your order'] as any; + assert.ok( + disposition['what happens next'] + .includes('a request to test a page with the same description is already approved.') + ); + } + finally { + await fs.unlink(claimedPath); + } +}); + +test('orderRetest rejects a report matching a queued job by URL', async () => { + const queuedFilePath = path.join(queuePath, 'que.json'); + await fs.writeFile(queuedFilePath, JSON.stringify({ + target: {what: 'Some Other Page', url: 'https://example.com/orderretestfixture'} + })); + try { + const body = await response(['260101T0004', 'ord', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.equal(details.error, undefined); + const disposition = body['response content']['disposition of your order'] as any; + assert.ok( + disposition['what happens next'] + .includes('a request to test a page with the same URL is already approved.') + ); + } + finally { + await fs.unlink(queuedFilePath); + } +}); + +test('orderRetest rejects an order once the job queue is full', async () => { + const fillerPath = path.join(queuePath, 'fil.json'); + await fs.writeFile(fillerPath, JSON.stringify({target: {what: 'Filler', url: 'https://example.com/filler'}})); + process.env.JOB_QUEUE_MAX = '1'; + try { + const body = await response(['260101T0004', 'ord', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.equal(details.error, undefined); + const disposition = body['response content']['disposition of your order'] as any; + assert.ok(disposition['what happens next'].includes('too many requests are awaiting approval right now.')); + } finally { + delete process.env.JOB_QUEUE_MAX; + await fs.unlink(fillerPath); + } +}); + +test('orderRetest accepts a valid retest order and enqueues a job directly', async () => { + const body = await response(['260101T0004', 'ord', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.equal(details.error, undefined); + assert.equal(details['page to be retested'].description, 'Order Retest Fixture Page'); + const disposition = body['response content']['disposition of your order'] as any; + assert.ok(disposition['report identifier']); + assert.ok(disposition['what happens next'].includes('Testing has begun.')); + assert.ok(disposition['do you want to wait for completion'].includes('awaitTest')); + const queuedFiles = await fs.readdir(queuePath); + assert.equal(queuedFiles.length, 1); + const job = JSON.parse(await fs.readFile(path.join(queuePath, queuedFiles[0]!), 'utf8')); + assert.equal(job.target.what, 'Order Retest Fixture Page'); + assert.equal(job.sources.reason, 'A reason that is long enough.'); +}); + +test('orderRetest response omits a web UI URL for this request but includes the ancestor', async () => { + const body = await response(['260101T0004', 'ord', 'A reason that is long enough.']); + const similarWeb = body['URLs of similar requests for web users'] as any; + assert.equal(similarWeb['this request'], null); + assert.ok(similarWeb['closest ancestor request'].includes('listIssues.html')); +}); diff --git a/api/orderRetest.ts b/api/orderRetest.ts new file mode 100644 index 00000000..e7c127cd --- /dev/null +++ b/api/orderRetest.ts @@ -0,0 +1,119 @@ +/* + orderRetest.ts + Processes an order to retest a page immediately, without manual approval, and returns an acknowledgement. +*/ + +// IMPORTS + +import {z} from 'zod'; +import {getRequestFailureReason, getResponseMetadata, getThisHost, getToolsFacts} from './util.ts'; +import {checkLength, isJobID, isTimeStamp, orderJob} from '../util.ts'; +import {orderRetestResponseSchema} from './schemas.ts'; + +// TYPES + +// The response content defined by the response schema. +type ResponseContent = z.infer['response content']; + +// FUNCTIONS + +// Returns the response body. +export const response = async (args: string[]) => { + const [timeStamp = '', jobID = '', reason = ''] = args; + const thisHost = getThisHost(); + const reasonCheck = checkLength(reason, 20, 100, 'reason'); + // Initialize the response-content properties. + let requestDetails: ResponseContent['details about your order']; + let requestDisposition: ResponseContent['disposition of your order'] = null; + // If the timestamp or job ID is not syntactically valid: + if (!isTimeStamp(timeStamp) || !isJobID(jobID)) { + requestDetails = { + error: 'order invalid: the report timestamp or job identifier is malformed' + }; + } + // Otherwise, if the encoded reason is too short or too long: + else if (reasonCheck.status === 'error') { + requestDetails = { + error: `order invalid: your ${reasonCheck.message}` + }; + } + // Otherwise, i.e. if the order is facially valid: + else { + // Order the retest. Unlike processTestRequest, this enqueues the job directly, with + // no manual-approval step, gated only by the lightweight checks orderJob itself + // performs (the target is not already claimed or queued, and the job queue is not + // already full). This resolves the cited report's description and URL itself. + const orderResult = await orderJob({timeStamp, jobID}, reason); + // If the cited report does not exist: + if (orderResult.result === 'nonreport') { + requestDetails = { + error: 'order invalid: the specified report does not exist' + }; + } + // Otherwise, i.e. if the cited report was found: + else { + // Add details about the order. + requestDetails = { + 'date and time received': new Date().toISOString(), + 'page to be retested': { + description: orderResult.description, + URL: orderResult.url + }, + 'reason why the page should be retested': reason + }; + // Add information about the disposition of the order. + if (orderResult.result === 'ok') { + const [reportTimeStamp, reportJobID] = orderResult.jobID.split('-') as [string, string]; + requestDisposition = { + 'what happens next': 'Testing has begun. Once it finishes, the report will be available via the listIssues tool.', + 'report identifier': {timeStamp: reportTimeStamp, jobID: reportJobID}, + 'how you can check for completion': `Call the listIssues tool with timeStamp "${reportTimeStamp}" and jobID "${reportJobID}" to learn whether the report is ready.`, + 'do you want to wait for completion': 'Testing typically takes about 2 minutes and occasionally up to 4 minutes. Do you want to wait (and be notified of completion) instead of checking back later? If so, call the awaitTest tool with the same timeStamp and jobID.' + }; + } + else { + requestDisposition = { + 'what happens next': `Your order will not be processed, because ${getRequestFailureReason( + orderResult.result, + {result: 'superseded', reason: 'a later report about a page with the same description is available.'} + )}`, + 'report identifier': null, + 'how you can check for completion': 'Not applicable.', + 'do you want to wait for completion': 'Not applicable.' + }; + } + } + } + // Create the response content. + const responseContent: ResponseContent = { + 'details about your order': requestDetails, + 'disposition of your order': requestDisposition + }; + // Create a response body. + const body = { + 'tool collection': getToolsFacts(), + 'tool name': 'orderRetest', + 'this request': { + description: 'Process my order to retest a page, immediately and without manual approval. The timeStamp and jobID parameters identify the latest available report about the page. Those parameters were in the response to my earlier listIssues request. The reason property of the request body is the reason why the page should be retested.', + method: 'POST', + URL: `${thisHost}/api/orderRetest/${timeStamp}/${jobID}`, + body: { + reason + }, + 'closest ancestor request': { + 'tool name': 'listIssues', + description: 'Provide details about one report, including basics about the issues reported in it.', + method: 'GET', + URL: `${thisHost}/api/listIssues/${timeStamp}/${jobID}` + } + }, + 'URLs of similar requests for web users': { + 'this request': null, + 'closest ancestor request': `${thisHost}/listIssues.html/${timeStamp}/${jobID}` + }, + 'response metadata': getResponseMetadata(), + 'response content': responseContent + }; + // Return it. + return body; +}; diff --git a/api/orderTest.test.ts b/api/orderTest.test.ts new file mode 100644 index 00000000..4d117f98 --- /dev/null +++ b/api/orderTest.test.ts @@ -0,0 +1,213 @@ +/* + orderTest.test.ts + Tests for api/orderTest.ts using the fixture corpus. +*/ + +// IMPORTS + +import {test, before, beforeEach, after} from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import {fixtureDBDir} from '../test/dbFixture.ts'; + +// Blank the alert configuration unconditionally, before ./orderTest.ts (whose orderJob +// calls no alert directly, but whose redirect-rejection branch does) is ever imported +// below, so this file sends no real alert emails even when run directly rather than via +// `npm test`. +for (const key of ['MANAGER_EMAIL', 'ALERT_API_HOST', 'ALERT_API_PATH', 'ALERT_API_KEY', 'ALERT_FROM']) { + process.env[key] = ''; +} +// Allow internal targets by default, so tests that submit an ordinary https://example.com/... +// URL do not depend on real DNS/network access to pass the resolution check that +// isAllowedTarget performs. +process.env.ALLOW_INTERNAL_TARGETS = 'true'; + +// SETUP AND TEARDOWN + +const savedDBDir = process.env.DB_DIR; +const queuePath = path.join(fixtureDBDir, 'jobs', 'queue'); +let logged: any[] = []; +const originalLog = console.log; + +before(() => { + process.env.DB_DIR = fixtureDBDir; +}); + +// Empty the job queue and capture console.log before each test, so tests are +// order-independent and any queued jobs a test creates do not leak into later tests. +beforeEach(async () => { + for (const fileName of await fs.readdir(queuePath)) { + await fs.unlink(path.join(queuePath, fileName)); + } + logged = []; + console.log = (...args) => logged.push(args.join(' ')); +}); + +import {response} from './orderTest.ts'; + +after(() => { + console.log = originalLog; + if (savedDBDir !== undefined) { + process.env.DB_DIR = savedDBDir; + } + else { + delete process.env.DB_DIR; + } +}); + +// TESTS + +test('orderTest rejects an empty description', async () => { + const body = await response(['', 'https://example.com/test', 'A reason that is long enough.']); + assert.ok((body['response content']['details about your order'] as any).error); +}); + +test('orderTest rejects a reason shorter than 20 characters', async () => { + const body = await response(['Ordered Page', 'https://example.com/ordered', 'short']); + const details = body['response content']['details about your order'] as any; + assert.ok(details.error.includes('reason')); +}); + +test('orderTest rejects a reason longer than 100 characters', async () => { + const longReason = 'x'.repeat(101); + const body = await response(['Ordered Page', 'https://example.com/ordered', longReason]); + const details = body['response content']['details about your order'] as any; + assert.ok(details.error.includes('reason')); +}); + +test('orderTest rejects a URL shorter than 12 characters', async () => { + const body = await response(['Test Page', 'short', 'A reason that is long enough.']); + assert.ok((body['response content']['details about your order'] as any).error); +}); + +test('orderTest rejects a syntactically invalid URL with the correct length', async () => { + const body = await response(['Test Page', 'not-a-valid-url', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.ok(details.error.includes('invalid URL')); +}); + +test('orderTest rejects a private-address URL unless internal targets are allowed', async () => { + delete process.env.ALLOW_INTERNAL_TARGETS; + try { + const body = await response(['Internal Page', 'https://192.168.1.1/page', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.ok(details.error.includes('does not resolve to a page that this deployment can test')); + } finally { + process.env.ALLOW_INTERNAL_TARGETS = 'true'; + } +}); + +test('orderTest rejects a URL that redirects to a disallowed target, and alerts a manager', async (t) => { + delete process.env.ALLOW_INTERNAL_TARGETS; + t.mock.method(globalThis, 'fetch', async () => ({url: 'https://10.0.0.5/page'}) as Response); + try { + const body = await response(['Redirecting Page', 'https://example.com/redirector', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.ok(details.error.includes('does not resolve to a page that this deployment can test')); + assert.ok(logged.some(line => + line.includes('WARNING (Kilotest: instant-test order rejected for redirecting to a disallowed target)') + && line.includes('https://example.com/redirector') + )); + } finally { + process.env.ALLOW_INTERNAL_TARGETS = 'true'; + } +}); + +test('orderTest rejects an already-tested page', async () => { + const body = await response(['Mixed Outcomes Page', 'https://example.com/mixed', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.equal(details.error, undefined); + const disposition = body['response content']['disposition of your order'] as any; + assert.ok( + disposition['what happens next'] + .includes('a report about a page with the same description and URL is available.') + ); + assert.equal(disposition['report identifier'], null); +}); + +test('orderTest rejects a page matching a claimed job by description', async () => { + const claimedPath = path.join(fixtureDBDir, 'jobs', 'claimed', 'clm.json'); + await fs.writeFile(claimedPath, JSON.stringify({ + target: {what: 'Claimed Test Page', url: 'https://example.com/claimed-job'} + })); + try { + const body = await response(['Claimed Test Page', 'https://example.com/other-url', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.equal(details.error, undefined); + const disposition = body['response content']['disposition of your order'] as any; + assert.ok( + disposition['what happens next'] + .includes('a request to test a page with the same description is already approved.') + ); + } + finally { + await fs.unlink(claimedPath); + } +}); + +test('orderTest rejects a page matching a queued job by URL', async () => { + const queuedPath = path.join(fixtureDBDir, 'jobs', 'queue', 'que.json'); + await fs.writeFile(queuedPath, JSON.stringify({ + target: {what: 'Some Other Page', url: 'https://example.com/queued-url'} + })); + try { + const body = await response(['A Different Page', 'https://example.com/queued-url', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.equal(details.error, undefined); + const disposition = body['response content']['disposition of your order'] as any; + assert.ok( + disposition['what happens next'] + .includes('a request to test a page with the same URL is already approved.') + ); + } + finally { + await fs.unlink(queuedPath); + } +}); + +test('orderTest rejects an order once the job queue is full', async () => { + process.env.JOB_QUEUE_MAX = '0'; + const fillerPaths = await Promise.all(Array.from({length: 1}, async (_, i) => { + const fillerPath = path.join(fixtureDBDir, 'jobs', 'queue', `fil${i}.json`); + await fs.writeFile(fillerPath, JSON.stringify({target: {what: `Filler ${i}`, url: `https://example.com/filler${i}`}})); + return fillerPath; + })); + delete process.env.JOB_QUEUE_MAX; + process.env.JOB_QUEUE_MAX = '1'; + try { + const body = await response(['One Too Many Page', 'https://example.com/one-too-many', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.equal(details.error, undefined); + const disposition = body['response content']['disposition of your order'] as any; + assert.ok(disposition['what happens next'].includes('too many requests are awaiting approval right now.')); + } finally { + delete process.env.JOB_QUEUE_MAX; + await Promise.all(fillerPaths.map(p => fs.unlink(p))); + } +}); + +test('orderTest accepts a valid new page order and enqueues a job directly', async () => { + const body = await response(['Brand New Ordered Page', 'https://example.com/brandnewordered', 'A reason that is long enough.']); + const details = body['response content']['details about your order'] as any; + assert.equal(details.error, undefined); + assert.equal(details['page to be tested'].description, 'Brand New Ordered Page'); + const disposition = body['response content']['disposition of your order'] as any; + assert.ok(disposition['report identifier']); + assert.match(disposition['report identifier'].timeStamp, /^\d{6}T\d{4}$/); + assert.match(disposition['report identifier'].jobID, /^[a-z0-9]{3}$/); + assert.ok(disposition['what happens next'].includes('Testing has begun.')); + assert.ok(disposition['do you want to wait for completion'].includes('awaitTest')); + const queuedFiles = await fs.readdir(queuePath); + assert.equal(queuedFiles.length, 1); + const job = JSON.parse(await fs.readFile(path.join(queuePath, queuedFiles[0]!), 'utf8')); + assert.equal(job.target.what, 'Brand New Ordered Page'); + assert.equal(job.sources.reason, 'A reason that is long enough.'); +}); + +test('orderTest response omits a web UI URL for this request but includes the ancestor', async () => { + const body = await response(['Brand New Ordered Page 2', 'https://example.com/brandnewordered2', 'A reason that is long enough.']); + const similarWeb = body['URLs of similar requests for web users'] as any; + assert.equal(similarWeb['this request'], null); + assert.ok(similarWeb['closest ancestor request'].includes('listReports.html')); +}); diff --git a/api/orderTest.ts b/api/orderTest.ts new file mode 100644 index 00000000..0c18f43c --- /dev/null +++ b/api/orderTest.ts @@ -0,0 +1,145 @@ +/* + orderTest.ts + Processes an order to test an untested page immediately, without manual approval, and returns an acknowledgement. +*/ + +// IMPORTS + +import {z} from 'zod'; +import {getRequestFailureReason, getResponseMetadata, getThisHost, getToolsFacts} from './util.ts'; +import {checkLength, isAllowedRedirectTarget, isAllowedTarget, isURL, orderJob} from '../util.ts'; +import {orderTestResponseSchema} from './schemas.ts'; +import {sendAlert} from '../alerts.ts'; + +// TYPES + +// The response content defined by the response schema. +type ResponseContent = z.infer['response content']; + +// FUNCTIONS + +// Returns the response body. +export const response = async (args: string[]) => { + const [description = '', url = '', reason = ''] = args; + const thisHost = getThisHost(); + // Initialize the response-content properties. + let requestDetails: ResponseContent['details about your order']; + let requestDisposition: ResponseContent['disposition of your order'] = null; + const descriptionCheck = checkLength(description, 1, 100, 'description of the page'); + const reasonCheck = checkLength(reason, 20, 100, 'reason'); + // If the description is invalid: + if (descriptionCheck.status === 'error') { + requestDetails = { + error: `order invalid: your ${descriptionCheck.message}` + }; + } + // Otherwise, if the reason is too short or too long: + else if (reasonCheck.status === 'error') { + requestDetails = { + error: `order invalid: your ${reasonCheck.message}` + }; + } + // Otherwise, i.e. if the URL is too short or too long: + else if (url.length < 12 || url.length > 300) { + requestDetails = { + error: 'order invalid: you specified a URL for the page that is not between 12 and 300 characters long' + }; + } + // Otherwise, i.e. if the URL is invalid: + else if (!isURL(url)) { + requestDetails = { + error: 'order invalid: you specified an invalid URL for the page' + }; + } + // Otherwise, i.e. if the URL does not resolve to an allowed target: + else if (!(await isAllowedTarget(url))) { + requestDetails = { + error: 'order invalid: the URL you specified does not resolve to a page that this deployment can test' + }; + } + // Otherwise, i.e. if the URL redirects to a target this deployment does not allow: + else if (!(await isAllowedRedirectTarget(url))) { + requestDetails = { + error: 'order invalid: the URL you specified does not resolve to a page that this deployment can test' + }; + // Alert a manager, since a redirect to a disallowed target on an order that + // itself named an allowed hostname suggests an attempted or accidental SSRF, + // rather than an ordinary mistake by a legitimate submitter. + await sendAlert( + 'Kilotest: instant-test order rejected for redirecting to a disallowed target', + `An order to test ${url} (described as "${description}") was rejected because it redirects to a target this deployment does not allow (e.g. a private, loopback, or link-local address).` + ); + } + // Otherwise, i.e. if the description and URL are valid: + else { + // Order the test. Unlike processTestRequest, this enqueues the job directly, with no + // manual-approval step, gated only by the lightweight checks orderJob itself performs + // (the target is not already claimed, queued, or reported, and the job queue is not + // already full). + const orderResult = await orderJob({description, url}, reason); + // Add details about the order. + requestDetails = { + 'date and time received': new Date().toISOString(), + 'page to be tested': { + description, + URL: url + }, + 'reason why the page should be tested': reason + }; + // Add information about the disposition of the order. + if (orderResult.result === 'ok') { + const [timeStamp, reportJobID] = orderResult.jobID.split('-') as [string, string]; + requestDisposition = { + 'what happens next': 'Testing has begun. Once it finishes, the report will be available via the listIssues tool.', + 'report identifier': {timeStamp, jobID: reportJobID}, + 'how you can check for completion': `Call the listIssues tool with timeStamp "${timeStamp}" and jobID "${reportJobID}" to learn whether the report is ready.`, + 'do you want to wait for completion': 'Testing typically takes about 2 minutes and occasionally up to 4 minutes. Do you want to wait (and be notified of completion) instead of checking back later? If so, call the awaitTest tool with the same timeStamp and jobID.' + }; + } + else { + requestDisposition = { + 'what happens next': `Your order will not be processed, because ${getRequestFailureReason( + orderResult.result, + {result: 'retest', reason: 'a report about a page with the same description and URL is available.'} + )}`, + 'report identifier': null, + 'how you can check for completion': 'Not applicable.', + 'do you want to wait for completion': 'Not applicable.' + }; + } + } + // Create the response content. + const responseContent: ResponseContent = { + 'details about your order': requestDetails, + 'disposition of your order': requestDisposition + }; + // Create a response body. + const body = { + 'tool collection': getToolsFacts(), + 'tool name': 'orderTest', + 'this request': { + description: 'Process my order to test a page about which no report is available yet, immediately and without manual approval. I have provided a description and the URL of the page and a reason why it should be tested.', + method: 'POST', + URL: `${thisHost}/api/orderTest`, + body: { + description, + URL: url, + reason + }, + 'closest ancestor request': { + 'tool name': 'listReports', + description: 'Provide basics about all available reports.', + method: 'GET', + URL: `${thisHost}/api/listReports` + } + }, + 'URLs of similar requests for web users': { + 'this request': null, + 'closest ancestor request': `${thisHost}/listReports.html` + }, + 'response metadata': getResponseMetadata(), + 'response content': responseContent + }; + // Return it. + return body; +}; diff --git a/api/requestRetest.ts b/api/requestRetest.ts index 43266a52..5b5ee387 100644 --- a/api/requestRetest.ts +++ b/api/requestRetest.ts @@ -6,7 +6,7 @@ // IMPORTS import {z} from 'zod'; -import {getResponseMetadata, getThisHost, getToolsFacts} from './util.ts'; +import {buildRequestDisposition, getResponseMetadata, getThisHost, getToolsFacts} from './util.ts'; import {checkLength, isJobID, isTimeStamp, processTestRequest} from '../util.ts'; import {requestRetestResponseSchema} from './schemas.ts'; @@ -60,38 +60,15 @@ export const response = async (args: string[]) => { 'reason why the page should be retested': reason }; // Add information about the disposition of the request. - if (requestResult === 'ok') { - requestDisposition = { - 'what happens next': 'Your request is likely to be approved and processed within 1 hour to 1 day.', - 'how you can check for completion': 'You can call the listReports tool to learn whether the page has been retested and a new report is available.', - 'how a web user can check for completion': `A web user can visit ${thisHost}/listReports.html to learn whether the page has been retested and a new report is available.` - }; - } - else { - const failureFact = 'Your request will not be processed, because '; - let failureReason: string; - if (requestResult === 'description') { - failureReason = 'a request to test a page with the same description is already approved.' - } - else if (requestResult === 'url') { - failureReason = 'a request to test a page with the same URL is already approved.' - } - else if (requestResult === 'superseded') { - failureReason = 'a later report about a page with the same description is available.' - } - else if (requestResult === 'queueFull') { - failureReason = 'too many requests are awaiting approval right now. Please try again later, ' + - 'or post your request at https://github.com/jrpool/kilotest/issues or email info@kilotest.com.' - } - else { - failureReason = 'an identical request is already awaiting approval.' - } - requestDisposition = { - 'what happens next': `${failureFact}${failureReason}`, - 'how you can check for completion': 'Not applicable.', - 'how a web user can check for completion': 'Not applicable.' - } - } + requestDisposition = buildRequestDisposition( + requestResult, + { + whatHappensNext: 'Your request is likely to be approved and processed within 1 hour to 1 day.', + howToCheck: 'You can call the listReports tool to learn whether the page has been retested and a new report is available.', + howWebUserChecks: `A web user can visit ${thisHost}/listReports.html to learn whether the page has been retested and a new report is available.` + }, + {result: 'superseded', reason: 'a later report about a page with the same description is available.'} + ); } } // Create the response content. diff --git a/api/requestTest.ts b/api/requestTest.ts index 8c5e8f1c..aef12211 100644 --- a/api/requestTest.ts +++ b/api/requestTest.ts @@ -6,7 +6,7 @@ // IMPORTS import {z} from 'zod'; -import {getResponseMetadata, getThisHost, getToolsFacts} from './util.ts'; +import {buildRequestDisposition, getResponseMetadata, getThisHost, getToolsFacts} from './util.ts'; import {checkLength, isAllowedRedirectTarget, isAllowedTarget, isURL, processTestRequest} from '../util.ts'; import {requestTestResponseSchema} from './schemas.ts'; import {sendAlert} from '../alerts.ts'; @@ -77,38 +77,15 @@ export const response = async (args: string[]) => { 'reason why the page should be tested': reason }; // Add information about the disposition of the request. - if (requestResult === 'ok') { - requestDisposition = { - 'what happens next': 'Your request is likely to be approved and processed within 1 hour to 1 day.', - 'how you can check for completion': 'You can call the listReports tool to learn whether the page has been tested and a report is available.', - 'how a web user can check for completion': `A web user can visit ${thisHost}/listReports.html to learn whether the page has been tested and a report is available.` - }; - } - else { - const failureFact = 'Your request will not be processed, because '; - let failureReason: string; - if (requestResult === 'description') { - failureReason = 'a request to test a page with the same description is already approved.' - } - else if (requestResult === 'url') { - failureReason = 'a request to test a page with the same URL is already approved.' - } - else if (requestResult === 'retest') { - failureReason = 'a report about a page with the same description and URL is available.' - } - else if (requestResult === 'queueFull') { - failureReason = 'too many requests are awaiting approval right now. Please try again later, ' + - 'or post your request at https://github.com/jrpool/kilotest/issues or email info@kilotest.com.' - } - else { - failureReason = 'an identical request is already awaiting approval.' - } - requestDisposition = { - 'what happens next': `${failureFact}${failureReason}`, - 'how you can check for completion': 'Not applicable.', - 'how a web user can check for completion': 'Not applicable.' - } - } + requestDisposition = buildRequestDisposition( + requestResult, + { + whatHappensNext: 'Your request is likely to be approved and processed within 1 hour to 1 day.', + howToCheck: 'You can call the listReports tool to learn whether the page has been tested and a report is available.', + howWebUserChecks: `A web user can visit ${thisHost}/listReports.html to learn whether the page has been tested and a report is available.` + }, + {result: 'retest', reason: 'a report about a page with the same description and URL is available.'} + ); } // Create the response content. const responseContent: ResponseContent = { diff --git a/api/routes.test.ts b/api/routes.test.ts index eee487da..68a32423 100644 --- a/api/routes.test.ts +++ b/api/routes.test.ts @@ -11,19 +11,22 @@ import {routes} from './routes.ts'; // TESTS -test('routes exports an array with 8 operations', () => { +test('routes exports an array with 11 operations', () => { assert.ok(Array.isArray(routes)); - assert.equal(routes.length, 8); + assert.equal(routes.length, 11); }); test('routes includes all expected operation IDs', () => { const ids = routes.map(r => r.operationId).sort(); assert.deepEqual(ids, [ + 'awaitTest', 'getReport', 'listDiagnoses', 'listIssues', 'listReports', 'listViolators', + 'orderRetest', + 'orderTest', 'requestFeature', 'requestRetest', 'requestTest' diff --git a/api/routes.ts b/api/routes.ts index 72dd4992..fcebf6f7 100644 --- a/api/routes.ts +++ b/api/routes.ts @@ -14,6 +14,9 @@ import { requestTestSchema, requestRetestSchema, requestFeatureSchema, + orderTestSchema, + orderRetestSchema, + awaitTestSchema, listReportsResponseSchema, listIssuesResponseSchema, listViolatorsResponseSchema, @@ -21,7 +24,10 @@ import { getReportResponseSchema, requestTestResponseSchema, requestRetestResponseSchema, - requestFeatureResponseSchema + requestFeatureResponseSchema, + orderTestResponseSchema, + orderRetestResponseSchema, + awaitTestResponseSchema } from './schemas.ts'; // ROUTES @@ -93,5 +99,37 @@ export const routes = [ summary: 'Request a feature improvement or new feature', bodySchema: z.object(requestFeatureSchema), responseSchema: requestFeatureResponseSchema + }, + { + operationId: 'orderTest', + method: 'post', + path: '/api/orderTest', + summary: 'Order that a page be tested immediately, without manual approval', + bodySchema: z.object(orderTestSchema), + responseSchema: orderTestResponseSchema + }, + { + operationId: 'orderRetest', + method: 'post', + path: '/api/orderRetest/{timeStamp}/{jobID}', + summary: 'Order that a page be retested immediately, without manual approval', + pathParamsSchema: z.object({ + timeStamp: orderRetestSchema.timeStamp, + jobID: orderRetestSchema.jobID + }), + bodySchema: z.object({reason: orderRetestSchema.reason}), + responseSchema: orderRetestResponseSchema + }, + { + operationId: 'awaitTest', + method: 'post', + path: '/api/awaitTest/{timeStamp}/{jobID}', + summary: 'Wait for a job ordered via orderTest or orderRetest to complete', + pathParamsSchema: z.object(awaitTestSchema), + // No caller-supplied fields beyond the path parameters; an empty body schema (rather + // than omitting bodySchema) keeps this route consistent with the "every POST route + // has a bodySchema" invariant the rest of this table and its own test maintain. + bodySchema: z.object({}), + responseSchema: awaitTestResponseSchema } ]; diff --git a/api/schemas.ts b/api/schemas.ts index 452639ff..c8baa672 100644 --- a/api/schemas.ts +++ b/api/schemas.ts @@ -60,6 +60,26 @@ export const requestFeatureSchema = { feature: z.string().describe('20- to 1000-character description of requested feature improvement or new feature') }; +// orderTest: POST /api/orderTest +export const orderTestSchema = { + description: z.string().describe('1- to 100-character description of the page conforming to the naming convention used in the listReports output'), + URL: z.string().describe('12- to 300-character URL of the page, including the https:// scheme and any query'), + reason: z.string().describe('20- to 100-character reason why the page should be tested') +}; + +// orderRetest: POST /api/orderRetest/{timeStamp}/{jobID} +export const orderRetestSchema = { + timeStamp: z.string().describe('Timestamp of the latest report about the page in YYMMDDTHHmm format (example: 260503T0432)'), + jobID: z.string().describe('Job identifier of the latest report about the page (example: x9z)'), + reason: z.string().describe('20- to 100-character reason why the page should be retested') +}; + +// awaitTest: POST /api/awaitTest +export const awaitTestSchema = { + timeStamp: z.string().describe('Timestamp of the ordered job, from the report identifier returned by orderTest, in YYMMDDTHHmm format (example: 260503T0432)'), + jobID: z.string().describe('Job identifier of the ordered job, from the report identifier returned by orderTest (example: x9z)') +}; + // listReports takes no input; omitted (mcp.ts already uses inputSchema: {}). // RESPONSE SCHEMAS @@ -93,11 +113,13 @@ const toolsFactsSchema = z.object({ 'web users can obtain similar functionalities at': z.string() }).meta({id: 'ToolsFacts'}); -// Identifies a related request (e.g., the closest ancestor request in the drill-down hierarchy). +// Identifies a related request (e.g., the closest ancestor request in the drill-down +// hierarchy). method is usually GET (every drill-down tool's ancestor is a read), except +// for awaitTest, whose only ancestor, orderTest, is a POST (it creates a job). const requestReferenceSchema = z.object({ 'tool name': z.string(), description: z.string(), - method: z.literal('GET'), + method: z.enum(['GET', 'POST']), URL: z.string() }).meta({id: 'RequestReference'}); @@ -320,3 +342,81 @@ export const requestFeatureResponseSchema = envelope( z.object(requestFeatureSchema), z.null() ); + +// Identifies the report a successfully ordered or awaited job will produce, once complete. +const reportIdentifierSchema = z.object({ + timeStamp: z.string().describe('Timestamp to pass to listIssues once the report is ready.'), + jobID: z.string().describe('Job identifier to pass to listIssues once the report is ready.') +}).meta({id: 'ReportIdentifier'}); + +export const orderTestResponseSchema = envelope( + 'POST', + z.object({ + 'details about your order': z.union([ + z.object({error: z.string()}), + z.object({ + 'date and time received': z.string(), + 'page to be tested': z.object({description: z.string(), URL: z.string()}), + 'reason why the page should be tested': z.string() + }) + ]), + 'disposition of your order': z.object({ + 'what happens next': z.string(), + 'report identifier': reportIdentifierSchema.nullable(), + 'how you can check for completion': z.string(), + 'do you want to wait for completion': z.string() + }).nullable() + }), + z.object(orderTestSchema), + z.object({ + 'this request': z.null(), + 'closest ancestor request': z.string().nullable() + }) +); + +export const orderRetestResponseSchema = envelope( + 'POST', + z.object({ + 'details about your order': z.union([ + z.object({error: z.string()}), + z.object({ + 'date and time received': z.string(), + 'page to be retested': z.object({description: z.string(), URL: z.string()}), + 'reason why the page should be retested': z.string() + }) + ]), + 'disposition of your order': z.object({ + 'what happens next': z.string(), + 'report identifier': reportIdentifierSchema.nullable(), + 'how you can check for completion': z.string(), + 'do you want to wait for completion': z.string() + }).nullable() + }), + z.object({reason: orderRetestSchema.reason}), + z.object({ + 'this request': z.null(), + 'closest ancestor request': z.string().nullable() + }) +); + +export const awaitTestResponseSchema = envelope( + 'POST', + z.object({ + 'details about your wait': z.union([ + z.object({error: z.string()}), + z.object({ + 'report identifier': reportIdentifierSchema + }) + ]), + 'disposition of your wait': z.object({ + outcome: z.enum(['completed', 'failed', 'timedOut']), + 'what happened': z.string(), + 'how you can check for completion': z.string() + }).nullable() + }), + undefined, + z.object({ + 'this request': z.null(), + 'closest ancestor request': z.null() + }) +); diff --git a/api/util.ts b/api/util.ts index 90d62a7a..f09771c2 100644 --- a/api/util.ts +++ b/api/util.ts @@ -15,7 +15,7 @@ import { objectSort, ruleEngines } from '../util.ts'; -import type {ReportExtract} from '../util.ts'; +import type {ReportExtract, TestRequestResult} from '../util.ts'; import {issues as issueSpecs} from 'testaro-issues'; // TYPES @@ -29,6 +29,15 @@ export type ReportBasics = { 'whether a later report about the same page exists': boolean; }; +// The disposition of a test/retest/instant-test request, as returned by +// buildRequestDisposition, or null when the request itself was invalid (in which case +// details about your request already carries the error, and no disposition applies). +export type RequestDisposition = { + 'what happens next': string; + 'how you can check for completion': string; + 'how a web user can check for completion': string; +} | null; + // FUNCTIONS // Returns the base URL of this Kilotest host. @@ -59,6 +68,54 @@ export const getToolsFacts = () => ({ 'URL': `${getThisHost()}/mcp`, 'web users can obtain similar functionalities at': getThisHost() }); +// Returns the human-readable reason a test/retest/instant-test request was not +// processed, given the result processTestRequest returned. requestResult is always a +// rejection reason here ('description', 'url', 'queueFull', or an unrecognized value +// meaning 'duplicate'); callers pass their own extra branch for a requestResult value +// specific to their own request type ('retest' for requestTest, 'superseded' for +// requestRetest), since that is the one branch not shared between them. +export const getRequestFailureReason = ( + requestResult: Exclude, + extraBranch?: {result: TestRequestResult; reason: string} +): string => { + if (requestResult === 'description') { + return 'a request to test a page with the same description is already approved.'; + } + if (requestResult === 'url') { + return 'a request to test a page with the same URL is already approved.'; + } + if (extraBranch && requestResult === extraBranch.result) { + return extraBranch.reason; + } + if (requestResult === 'queueFull') { + return 'too many requests are awaiting approval right now. Please try again later, ' + + 'or post your request at https://github.com/jrpool/kilotest/issues or email info@kilotest.com.'; + } + return 'an identical request is already awaiting approval.'; +}; +// Returns the disposition of a test/retest/instant-test request: an 'ok' disposition +// describing how to check for completion, or a rejection disposition naming why the +// request was not processed. Shared by requestTest, requestRetest, and orderTest, +// whose only differences are the wording of the 3 completion-related sentences and, for +// a rejection, the one extra requestResult branch getRequestFailureReason takes. +export const buildRequestDisposition = ( + requestResult: TestRequestResult, + okText: {whatHappensNext: string; howToCheck: string; howWebUserChecks: string}, + extraBranch?: {result: TestRequestResult; reason: string} +): RequestDisposition => { + if (requestResult === 'ok') { + return { + 'what happens next': okText.whatHappensNext, + 'how you can check for completion': okText.howToCheck, + 'how a web user can check for completion': okText.howWebUserChecks + }; + } + return { + 'what happens next': `Your request will not be processed, because ${getRequestFailureReason(requestResult, extraBranch)}`, + 'how you can check for completion': 'Not applicable.', + 'how a web user can check for completion': 'Not applicable.' + }; +}; // Returns the facts about a rule engine. export const getRuleEngineFacts = (ruleEngineID: string) => { const ruleEngineData = ruleEngines[ruleEngineID] || [null, null]; diff --git a/capability.md b/capability.md index a761a24c..4265dff2 100644 --- a/capability.md +++ b/capability.md @@ -36,7 +36,7 @@ A list of available reports. ## Side effects -Kilotest reads the specified web page during testing. It does not modify the target site. Testing is asynchronous and may take up to approximately one day before reports are available for newly requested pages. +Kilotest reads the specified web page during testing. It does not modify the target site. A manually approved test request is asynchronous and may take up to approximately one day before a report is available; an automatically approved, immediate test order is enqueued right away and typically completes within a few minutes. ## Cost diff --git a/env.example b/env.example index 3ace71a4..e79e498f 100644 --- a/env.example +++ b/env.example @@ -47,6 +47,10 @@ TUTORIAL_WEB_COMMENTS_MAX=__placeholder__ TUTORIAL_AI_COMMENTS_MAX=__placeholder__ # Maximum number of feature requests held via the MCP server's requestFeature tool. FEATURE_REQUESTS_MAX=__placeholder__ +# Maximum number of jobs allowed in the job queue (db/jobs/queue) at once. Shared by every +# path that enqueues a job: manual approval of a pending test/retest request, and the +# orderTest MCP tool, which enqueues directly. +JOB_QUEUE_MAX=__placeholder__ # Alert configuration: diff --git a/index.test.ts b/index.test.ts index 1b3f187e..f53cd2de 100644 --- a/index.test.ts +++ b/index.test.ts @@ -544,6 +544,45 @@ test('POST /api/requestFeature with valid JSON returns a JSON response and recor assert.equal(await getMetricCount('apiOperations', 'requestFeature'), countBefore + 1); }); +test('POST /api/orderTest with valid JSON enqueues a job directly and records an API-operation metric', {timeout: 500}, async () => { + const countBefore = await getMetricCount('apiOperations', 'orderTest'); + const res = await request('POST', '/api/orderTest', { + description: `API Ordered Page ${uniqueStamp}`, + URL: `https://example.com/api-ordered-${uniqueStamp}`, + reason: 'Because accessibility matters here' + }); + assert.equal(res.statusCode, 200); + const body = jsonBody(res); + assert.ok(body['tool name'] || body['response content']); + assert.equal(await getMetricCount('apiOperations', 'orderTest'), countBefore + 1); + const reportIdentifier = body['response content']['disposition of your order']['report identifier']; + assert.ok(reportIdentifier); + await fs.unlink(path.join(fixtureDBDir, 'jobs', 'queue', `${reportIdentifier.timeStamp}-${reportIdentifier.jobID}.json`)); +}); + +test('POST /api/orderRetest/260202T0000/new with valid JSON enqueues a job directly and records an API-operation metric', async () => { + const countBefore = await getMetricCount('apiOperations', 'orderRetest'); + const res = await request('POST', '/api/orderRetest/260202T0000/new', { + reason: 'Because the report is obsolete and needs refreshing' + }); + assert.equal(res.statusCode, 200); + const body = jsonBody(res); + assert.ok(body['tool name'] || body['response content']); + assert.equal(await getMetricCount('apiOperations', 'orderRetest'), countBefore + 1); + const reportIdentifier = body['response content']['disposition of your order']['report identifier']; + assert.ok(reportIdentifier); + await fs.unlink(path.join(fixtureDBDir, 'jobs', 'queue', `${reportIdentifier.timeStamp}-${reportIdentifier.jobID}.json`)); +}); + +test('POST /api/awaitTest/260101T0000/mix with an already-completed report records an API-operation metric', async () => { + const countBefore = await getMetricCount('apiOperations', 'awaitTest'); + const res = await request('POST', '/api/awaitTest/260101T0000/mix', {}); + assert.equal(res.statusCode, 200); + const body = jsonBody(res); + assert.equal(body['response content']['disposition of your wait'].outcome, 'completed'); + assert.equal(await getMetricCount('apiOperations', 'awaitTest'), countBefore + 1); +}); + test('POST /api/invalidService returns an error', async () => { const res = await request('POST', '/api/invalidService', {data: 'test'}); assert.equal(res.statusCode, 400); diff --git a/index.ts b/index.ts index 0020ed51..6413db58 100644 --- a/index.ts +++ b/index.ts @@ -80,6 +80,9 @@ import {response as listViolatorsAPI} from './api/listViolators.ts'; import {response as requestFeatureAPI} from './api/requestFeature.ts'; import {response as requestRetestAPI} from './api/requestRetest.ts'; import {response as requestTestAPI} from './api/requestTest.ts'; +import {response as orderTestAPI} from './api/orderTest.ts'; +import {response as orderRetestAPI} from './api/orderRetest.ts'; +import {response as awaitTestAPI} from './api/awaitTest.ts'; // ENVIRONMENT @@ -171,6 +174,9 @@ const apiRespond: { requestFeature: ApiResponder; requestRetest: ApiResponder; requestTest: ApiResponder; + orderTest: ApiResponder; + orderRetest: ApiResponder; + awaitTest: ApiResponder; } = { getReport: getReportAPI, listDiagnoses: listDiagnosesAPI, @@ -179,7 +185,10 @@ const apiRespond: { listViolators: listViolatorsAPI, requestFeature: requestFeatureAPI, requestRetest: requestRetestAPI, - requestTest: requestTestAPI + requestTest: requestTestAPI, + orderTest: orderTestAPI, + orderRetest: orderRetestAPI, + awaitTest: awaitTestAPI }; // CONSTANTS @@ -1255,6 +1264,35 @@ const handleRequest = async (request: IncomingMessage, response: ServerResponse) setHeaders('application/json', null, 'ultra'); response.end(JSON.stringify(responseBody)); } + // Otherwise, if the service is to order an immediate, automatically approved test: + else if (segments[0] === 'orderTest') { + const {description, URL, reason} = postData as {description: string; URL: string; reason: string}; + // Get the response body. + const responseBody = await apiRespond.orderTest([description, URL, reason]); + await recordPageMetric('apiOperations', 'orderTest'); + // Send it. + setHeaders('application/json', null, 'ultra'); + response.end(JSON.stringify(responseBody)); + } + // Otherwise, if the service is to order an immediate, automatically approved retest: + else if (segments[0] === 'orderRetest') { + const {reason} = postData as {reason: string}; + // Get the response body. + const responseBody = await apiRespond.orderRetest(segments.slice(1).concat(reason)); + await recordPageMetric('apiOperations', 'orderRetest'); + // Send it. + setHeaders('application/json', null, 'ultra'); + response.end(JSON.stringify(responseBody)); + } + // Otherwise, if the service is to wait for an ordered job to complete: + else if (segments[0] === 'awaitTest') { + // Get the response body. + const responseBody = await apiRespond.awaitTest(segments.slice(1)); + await recordPageMetric('apiOperations', 'awaitTest'); + // Send it. + setHeaders('application/json', null, 'ultra'); + response.end(JSON.stringify(responseBody)); + } // Otherwise, i.e. if the service is invalid: else { await serveError( diff --git a/llms-full.txt b/llms-full.txt index 502432b3..120c8886 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -132,6 +132,35 @@ Kilotest offers these tools: - path: `/api/requestFeature` - payload properties - `feature`: description and motivation of the feature improvement or the new feature +- `orderTest` + - method: `POST` + - purpose: order immediate, automatically approved testing of one web page about which no report is available, instead of waiting for manual approval + - path: `/api/orderTest` + - payload properties + - `description`: description of the page to be tested + - `URL`: URL of the page to be tested + - `reason`: reason for testing the page + - response includes a report identifier (`timeStamp` and `jobID`) once the job is enqueued +- `orderRetest` + - method: `POST` + - purpose: order immediate, automatically approved retesting of one web page about which a report is available, instead of waiting for manual approval + - path: `/api/orderRetest/{timeStamp}/{jobID}` + - parameters + - `timeStamp`: initial segment of report identifier + - `jobID`: final segment of report identifier + - source of parameters: response to a `listIssues` request + - payload properties + - `reason`: reason for retesting the page + - response includes a report identifier (`timeStamp` and `jobID`) once the job is enqueued +- `awaitTest` + - method: `POST` + - purpose: wait for a job ordered via `orderTest` or `orderRetest` to complete; blocks until the report is ready, the job fails, or a maximum wait time elapses, then returns + - path: `/api/awaitTest/{timeStamp}/{jobID}` + - parameters + - `timeStamp`: initial segment of the report identifier returned by `orderTest` or `orderRetest` + - `jobID`: final segment of the report identifier returned by `orderTest` or `orderRetest` + - source of parameters: response to an `orderTest` or `orderRetest` request + - caveat: testing typically takes about 2 minutes and occasionally up to 4 minutes; sends no interim notices while waiting ## After a test, retest, or feature request @@ -147,6 +176,8 @@ When you request a test or a retest: - check test completion by calling `listReports` and looking for the page and completion date in the list - you advise the user to check in approximately 1 day for the test results on the Kilotest website, starting at `https://kilotest.com/listReports.html` +For immediate testing without waiting for manual approval, use `orderTest` or `orderRetest` instead of `requestTest` or `requestRetest`. Each enqueues the job right away and returns a report identifier along with a question about whether to wait for completion. To wait, call `awaitTest` with that identifier; it blocks until the report is ready, the job fails, or a maximum wait time elapses. To check later instead, call `listIssues` with the same identifier. + An [OpenAPI specification for Kilotest](https://kilotest.com/openapi.yaml) is available. ## How to report results honestly to users diff --git a/llms.txt b/llms.txt index bb2e889c..4e2481d9 100644 --- a/llms.txt +++ b/llms.txt @@ -12,6 +12,9 @@ Kilotest helps language models and AI platforms to give thorough, accurate and t - Request testing of a web page: `/api/requestTest` (`POST`) - Request retesting of a web page: `/api/requestRetest/{timeStamp}/{jobID}` (`POST`) - Request adding or improving a feature: `/api/requestFeature` (`POST`) +- Order immediate, automatically approved testing of a web page: `/api/orderTest` (`POST`) +- Order immediate, automatically approved retesting of a web page: `/api/orderRetest/{timeStamp}/{jobID}` (`POST`) +- Wait for a job ordered via orderTest or orderRetest to complete: `/api/awaitTest/{timeStamp}/{jobID}` (`POST`) ## MCP Server diff --git a/mcp.test.ts b/mcp.test.ts index d122cc1e..3a9fadd5 100644 --- a/mcp.test.ts +++ b/mcp.test.ts @@ -37,10 +37,10 @@ test('mcpPath is /mcp', () => { assert.equal(mcpPath, '/mcp'); }); -test('createMCPServer registers all 9 tools', () => { +test('createMCPServer registers all 12 tools', () => { const server = createMCPServer(); const toolNames = Object.keys((server as any)._registeredTools); - assert.equal(toolNames.length, 9); + assert.equal(toolNames.length, 12); assert.deepEqual(toolNames, [ 'getKilotestOverview', 'listReports', @@ -50,7 +50,10 @@ test('createMCPServer registers all 9 tools', () => { 'getReport', 'requestTest', 'requestRetest', - 'requestFeature' + 'requestFeature', + 'orderTest', + 'orderRetest', + 'awaitTest' ]); }); @@ -199,6 +202,60 @@ test('requestFeature handler returns content and structuredContent', async () => assert.equal(getToolCallCount('requestFeature'), countBefore + 1); }); +test('orderTest handler returns content and structuredContent, and enqueues a job', async () => { + const {jobsPath} = await import('./util.ts'); + const server = createMCPServer(); + const countBefore = getToolCallCount('orderTest'); + const result = await (server as any)._registeredTools.orderTest.handler({ + description: 'MCP Ordered Page', + URL: 'https://example.com/mcp-ordered', + reason: 'Because accessibility matters here' + }); + assert.ok(result.content); + assert.equal(result.content[0].type, 'text'); + assert.ok(result.structuredContent); + assert.equal(getToolCallCount('orderTest'), countBefore + 1); + const reportIdentifier = result.structuredContent['response content']['disposition of your order']['report identifier']; + assert.ok(reportIdentifier); + const queuedPath = `${jobsPath()}/queue/${reportIdentifier.timeStamp}-${reportIdentifier.jobID}.json`; + fs.unlinkSync(queuedPath); +}); + +test('orderRetest handler returns content and structuredContent, and enqueues a job', async () => { + const {jobsPath} = await import('./util.ts'); + const server = createMCPServer(); + const countBefore = getToolCallCount('orderRetest'); + // 260202T0000-new is the latest report of "Mixed Outcomes Page" (260101T0000-mix is an + // earlier, superseded report of the same page), so this order is accepted. + const result = await (server as any)._registeredTools.orderRetest.handler({ + timeStamp: '260202T0000', + jobID: 'new', + reason: 'Because the report is obsolete' + }); + assert.ok(result.content); + assert.equal(result.content[0].type, 'text'); + assert.ok(result.structuredContent); + assert.equal(getToolCallCount('orderRetest'), countBefore + 1); + const reportIdentifier = result.structuredContent['response content']['disposition of your order']['report identifier']; + assert.ok(reportIdentifier); + const queuedPath = `${jobsPath()}/queue/${reportIdentifier.timeStamp}-${reportIdentifier.jobID}.json`; + fs.unlinkSync(queuedPath); +}); + +test('awaitTest handler returns content and structuredContent for an already-completed report', async () => { + const server = createMCPServer(); + const countBefore = getToolCallCount('awaitTest'); + const result = await (server as any)._registeredTools.awaitTest.handler({ + timeStamp: '260101T0000', + jobID: 'mix' + }); + assert.ok(result.content); + assert.equal(result.content[0].type, 'text'); + assert.ok(result.structuredContent); + assert.equal(result.structuredContent['response content']['disposition of your wait'].outcome, 'completed'); + assert.equal(getToolCallCount('awaitTest'), countBefore + 1); +}); + test('listIssues handler returns an error for a nonexistent report', async () => { const server = createMCPServer(); const result = await (server as any)._registeredTools.listIssues.handler({ @@ -301,7 +358,7 @@ test('handleMCP responds to initialize with server info', async () => { } }); -test('handleMCP lists all 9 tools via tools/list', async () => { +test('handleMCP lists all 12 tools via tools/list', async () => { const server = await startMCPServer(); try { const port = server.address().port; @@ -309,7 +366,7 @@ test('handleMCP lists all 9 tools via tools/list', async () => { assert.equal(res.statusCode, 200); const result = parseSSEResult(res.body); const toolNames = result.result.tools.map((t: any) => t.name); - assert.equal(toolNames.length, 9); + assert.equal(toolNames.length, 12); assert.deepEqual(toolNames, [ 'getKilotestOverview', 'listReports', @@ -319,7 +376,10 @@ test('handleMCP lists all 9 tools via tools/list', async () => { 'getReport', 'requestTest', 'requestRetest', - 'requestFeature' + 'requestFeature', + 'orderTest', + 'orderRetest', + 'awaitTest' ]); } finally { diff --git a/mcp.ts b/mcp.ts index 279f918b..63ec0a71 100644 --- a/mcp.ts +++ b/mcp.ts @@ -16,6 +16,9 @@ import * as listDiagnosesAPI from './api/listDiagnoses.ts'; import * as requestTestAPI from './api/requestTest.ts'; import * as requestRetestAPI from './api/requestRetest.ts'; import * as requestFeatureAPI from './api/requestFeature.ts'; +import * as orderTestAPI from './api/orderTest.ts'; +import * as orderRetestAPI from './api/orderRetest.ts'; +import * as awaitTestAPI from './api/awaitTest.ts'; import {version} from './api/version.ts'; import {recordMetric} from './util.ts'; @@ -27,6 +30,9 @@ import { requestTestSchema, requestRetestSchema, requestFeatureSchema, + orderTestSchema, + orderRetestSchema, + awaitTestSchema, listReportsResponseSchema, listIssuesResponseSchema, listViolatorsResponseSchema, @@ -34,7 +40,10 @@ import { getReportResponseSchema, requestTestResponseSchema, requestRetestResponseSchema, - requestFeatureResponseSchema + requestFeatureResponseSchema, + orderTestResponseSchema, + orderRetestResponseSchema, + awaitTestResponseSchema } from './api/schemas.ts'; // CONSTANTS @@ -54,6 +63,7 @@ const kilotestOverview = [ 'Kilotest integrates an ensemble of twelve independent rule engines to test a web page and stores the results as a structured report.', 'Results are organized as a hierarchy. A report contains issues. An issue has violators: elements of the tested page reported as exhibiting the issue. A violator has diagnoses: explanations from rule engines of how the element exhibited the issue.', 'Typical workflow: call the listReports tool to learn whether a report about the page already exists. If it does, drill down with the listIssues, listViolators, and listDiagnoses tools, or retrieve the entire report with the getReport tool. If no report exists, request a test with the requestTest tool. If the latest report is obsolete, request a retest with the requestRetest tool.', + 'For immediate, automatically approved testing, use the orderTest tool (for a new page) or the orderRetest tool (for a page with an obsolete report) instead of requestTest or requestRetest. Each enqueues the job right away, with no manual approval step, and returns a report identifier along with a question about whether to wait for completion. To wait, call the awaitTest tool with that identifier; it blocks until the report is ready, the job fails, or a maximum wait time elapses. To check later instead, call the listIssues tool with the same identifier.', 'More documentation is available at https://kilotest.com/llms.txt and https://kilotest.com/llms-full.txt.' ].join('\n\n'); @@ -71,7 +81,7 @@ export const createMCPServer = (): McpServer => { description: 'Tools that test web pages for front-end quality (accessibility, usability, and standards conformity) and make test results available' }, { - instructions: `${sharedContext} Use the listReports tool to start. If it shows that there is a report available about the page you want facts about, drill down with the listIssues, listViolators, and listDiagnoses tools. If not, use the requestTest tool to request that the page be tested. If the latest report about the page is obsolete, use the requestRetest tool to request that the page be retested. For detailed documentation, call the getKilotestOverview tool.` + instructions: `${sharedContext} Use the listReports tool to start. If it shows that there is a report available about the page you want facts about, drill down with the listIssues, listViolators, and listDiagnoses tools. If not, use the requestTest tool to request that the page be tested, or the orderTest tool for immediate, automatically approved testing. If the latest report about the page is obsolete, use the requestRetest tool to request that the page be retested, or the orderRetest tool for immediate, automatically approved retesting. For detailed documentation, call the getKilotestOverview tool.` }); server.registerTool( 'getKilotestOverview', @@ -251,6 +261,66 @@ export const createMCPServer = (): McpServer => { return {content: [{type: 'text', text: JSON.stringify(result)}], structuredContent: result}; } ); + server.registerTool( + 'orderTest', + { + description: toolDoc('Process my order to test a page about which no report is available yet, immediately and automatically approved, instead of waiting for manual approval. Testing typically takes about 2 minutes and occasionally up to 4 minutes.'), + inputSchema: orderTestSchema, + outputSchema: orderTestResponseSchema, + annotations: { + title: toolDoc('Process my order to test a page about which no report is available yet, immediately and automatically approved.'), + readOnlyHint: false, + idempotentHint: false, + destructiveHint: false, + openWorldHint: false + } + }, + async ({description, URL, reason}) => { + const result = await orderTestAPI.response([description, URL, reason]); + await recordMetric('mcpToolCalls', 'orderTest'); + return {content: [{type: 'text', text: JSON.stringify(result)}], structuredContent: result}; + } + ); + server.registerTool( + 'orderRetest', + { + description: toolDoc('Process my order to retest a page immediately and automatically approved, instead of waiting for manual approval. Testing typically takes about 2 minutes and occasionally up to 4 minutes.'), + inputSchema: orderRetestSchema, + outputSchema: orderRetestResponseSchema, + annotations: { + title: toolDoc('Process my order to retest a page immediately and automatically approved.'), + readOnlyHint: false, + idempotentHint: false, + destructiveHint: false, + openWorldHint: false + } + }, + async ({timeStamp, jobID, reason}) => { + const result = await orderRetestAPI.response([timeStamp, jobID, reason]); + await recordMetric('mcpToolCalls', 'orderRetest'); + return {content: [{type: 'text', text: JSON.stringify(result)}], structuredContent: result}; + } + ); + server.registerTool( + 'awaitTest', + { + description: toolDoc('Wait for a job ordered via the orderTest tool to complete, using the report identifier orderTest returned. Blocks until the report is ready, the job fails, or a maximum wait time elapses; sends no interim notices.'), + inputSchema: awaitTestSchema, + outputSchema: awaitTestResponseSchema, + annotations: { + title: toolDoc('Wait for a job ordered via the orderTest tool to complete.'), + readOnlyHint: true, + idempotentHint: false, + destructiveHint: false, + openWorldHint: false + } + }, + async ({timeStamp, jobID}) => { + const result = await awaitTestAPI.response([timeStamp, jobID]); + await recordMetric('mcpToolCalls', 'awaitTest'); + return {content: [{type: 'text', text: JSON.stringify(result)}], structuredContent: result}; + } + ); server.registerResource( 'kilotest-overview', 'docs://kilotest/overview', diff --git a/memory/MEMORY.md b/memory/MEMORY.md index ce6c168b..48f212e8 100644 --- a/memory/MEMORY.md +++ b/memory/MEMORY.md @@ -5,3 +5,4 @@ This directory holds persistent context files for the Kilotest project. Each fil ## Index - [user_maintainer_context.md](user_maintainer_context.md) — user: Maintainer is 84 years old, sole contributor, prioritizes legibility for future maintainers over short-term velocity +- [plan_issue_111_instant_test.md](plan_issue_111_instant_test.md) — project: orderTest/orderRetest/awaitTest (issue #111), implemented on orderTest branch with full REST+MCP parity diff --git a/memory/plan_issue_111_instant_test.md b/memory/plan_issue_111_instant_test.md new file mode 100644 index 00000000..b8edb538 --- /dev/null +++ b/memory/plan_issue_111_instant_test.md @@ -0,0 +1,184 @@ +--- +type: project +name: Plan for issue #111 (instant testing) +description: Design and implementation of orderTest/orderRetest/awaitTest, the three-tool MCP addition (with REST parity) for immediate, auto-approved testing with an optional wait for completion. Implemented on the orderTest branch. +--- + +# Issue #111: instant-testing MCP tools (implemented) + +Issue: [jrpool/kilotest#111](https://github.com/jrpool/kilotest/issues/111). The issue's own comment already +narrowed scope to a minimal increment: keep the existing single-worker Testaro polling +model, add new MCP tools, skip cost recovery, add only a lightweight anti-abuse gate. +This was implemented on the `orderTest` branch as three tools, `orderTest`, `orderRetest`, +and `awaitTest`, each with REST (`/api/*`) and MCP exposure, at the user's request for +full parity with the existing `requestTest`/`requestRetest` tools. + +## Why two tools, not one + +MCP `tools/call` is one request, one response; a tool cannot pose a question mid-call +and get an answer in the same call. The natural interaction is: order the test (fast, +always returns quickly), then, as a genuinely separate second call, decide whether to +wait for the result. Splitting into two tools avoids needing any new MCP machinery +(progress tokens, progress notifications) to solve the timeout problem below. + +### The problem that killed the single-tool, blocking-wait design + +An earlier design considered one tool with a `waitForCompletion` flag, where a `true` +call would enqueue and then block up to ~4 minutes for the report. This breaks if the +MCP client times out client-side before Kilotest responds: the client has stopped +listening, so nothing the server does (a final response, a progress notification) can +reach it, unless the client supplied a `progressToken` up front and progress +notifications were emitted early enough to carry the job identifier before the +timeout hit. That makes the wait's safety depend on client capabilities the caller may +not have, with no way for the server to detect or work around the gap. + +The fix is not "notify them anyway", it is: never let identifier delivery depend on a +wait completing. `orderTest` returns immediately, always, with the identifier already +in hand, before any wait is ever proposed. This makes the whole progress-token / +client-timeout question moot for the ordering step; the identifier is guaranteed +delivered before any risky wait begins. + +### Why a caller could never misuse awaitTest on a manually-approved job + +Considered and rejected: restricting `awaitTest` to jobs specifically created by +`orderTest`, versus letting it work generically on any pending job by `timeStamp`/ +`jobID`. This turned out to be moot: `requestTest`/`requestRetest` never hand the +caller a `timeStamp`/`jobID` at all (nothing exists yet), only instructions to poll +`listReports` later by description/URL. An identifier for a report only becomes +visible once the report is complete. So no caller can ever construct an `awaitTest` +call naming a manually-approved, still-pending job; the scenario that would need a +restriction cannot arise. No state-check or restriction is needed in `awaitTest`. +`orderTest` is the only path that exposes a pre-completion identifier at all, because +it enqueues synchronously and returns the job's own identifier immediately. + +## Implementation notes + +- `orderJob(target, reason)` in `util.ts` is the shared enqueue function backing both + `orderTest` (target `{description, url}`) and `orderRetest` (target `{timeStamp, + jobID}`), mirroring `processTestRequest`'s own dual-target design. It shares + `testRequestsLock` with `processTestRequest` so the two can never race to enqueue + duplicate jobs for the same page. +- `awaitJob(timeStamp, jobID)` in `util.ts` is the shared poll function backing + `awaitTest`; it works for jobs from either `orderTest` or `orderRetest` (or, in + principle, any job in the queue/claimed pipeline), since it only inspects + `getJobNames()`/`getReportStats()`, not job origin. Poll interval and timeout are + injectable via `AWAIT_JOB_POLL_MS`/`AWAIT_JOB_TIMEOUT_MS` so tests do not need to + wait the real ~2-4 minutes. +- Full REST+MCP parity: `api/routes.ts`, `index.ts`'s `/api/*` POST dispatch, and + `openapi.yaml` (regenerated via `npm run generate:openapi`) all cover the three new + operations, alongside the MCP tool registrations in `mcp.ts`. `smokeTest.ts` needed + no change, since it already smoke-tests the `/api/*` wildcard generically via one + representative path, not every individual operation. +- Web-UI-equivalent URL fields (`'URLs of similar requests for web users'`) are `null` + for `orderTest`/`orderRetest`/`awaitTest`'s own request, per explicit instruction: + "web users will not be offered an order option." The *ancestor* request's web URL + (e.g. `listReports.html`, `listIssues.html`) is still populated where one exists. +- `requestReferenceSchema` (`api/schemas.ts`) had to be widened from `method: + z.literal('GET')` to `z.enum(['GET', 'POST'])`, since `awaitTest`'s only ancestor, + `orderTest`/`orderRetest`, is a POST, unlike every other tool's GET ancestor. +- The caller's stated `reason` is persisted onto the job itself (`job.sources.reason`), + surviving into the eventual report exactly as `sources.worker` already does. No + approval rule consults it yet; the user's rationale for keeping the field even though + there is no human review is to accumulate real callers' stated reasons as data to + inform more sophisticated, reason-aware approval rules in a later iteration. +- Shared response-shaping helpers landed in `api/util.ts`: `getRequestFailureReason` + and `buildRequestDisposition`, extracted from `requestTest.ts`/`requestRetest.ts` + (now refactored to use them) before `orderTest.ts`/`orderRetest.ts` were written + against the same helpers. + +## Tool 1: orderTest + +- Reuses `requestTest.ts`'s existing validation (length/URL/target-safety checks + already in top-level `util.ts`: `checkLength`, `isURL`, `isAllowedTarget`, + `isAllowedRedirectTarget`). +- On success, enqueues directly into `db/jobs/queue` (bypassing the manual-approval + inbox `db/jobs/testRequests.json` entirely), following the job-template population + logic already in `web/enqueue/index.ts` (`jobName = ${nowStamp}-${jobIDSuffix}`, + etc.). +- Approval gate (lightweight, per the issue's own comment, no human-in-the-loop): + - Queue-size cap: env var **`JOB_QUEUE_MAX`** (not "instant-test"-scoped; `db/jobs/queue` + is a shared resource fed by manual approvals, the web UI, and this tool, so the cap + name must not imply a per-source quota). Checked against the live count of + `db/jobs/queue` at call time via the existing `queuePath()` helper, not a + self-maintained counter. Parallels the existing `TEST_REQUEST_QUEUE_MAX` (which caps + the earlier-stage pending-approval inbox), as the sibling cap one stage further down + the same pipeline. + - Dedup against `queue`/`claimed`, reusing/extending `processTestRequest`'s existing + per-URL/description dedup logic. +- Response, in all cases (accepted or rejected), returns immediately: + - If rejected: says why (queue full, duplicate), no identifier. + - If accepted: includes the job's `timeStamp`/`jobID` (or equivalent identifier), + states that testing has begun and the report will be available via `listIssues` + with those parameters, and asks the literal question: "Do you want to wait + (typically about 2 minutes, occasionally up to 4 minutes) and be notified of + completion?" The calling model decides and, if yes, makes a separate `awaitTest` + call. + +## Tool 2: awaitTest + +- Takes the identifier (`timeStamp`/`jobID`) `orderTest` returned. +- Blocks, polling for report completion, until the report exists or a ~4-minute cap + elapses. +- No progress notifications during the wait ("no notices until job completion or job + timeout", per explicit instruction). This is a pure bounded poll. +- On completion or timeout, returns a response pointing at the same `listIssues` + parameters already given, so both outcomes converge on the same follow-up action. +- Needs zero Testaro changes and zero MCP progress-notification plumbing: it derives + status purely from Kilotest's own already-tracked state (`db/jobs/claimed/` vs. a + completed report existing), not from any per-rule-engine signal. + +## Deferred: detailed (per-rule-engine) progress reporting + +Explicitly postponed; "the work is substantial even with generic progress reporting." +Findings, kept for whenever this is revisited: + +- Testaro's `doActs.js` already has the needed hook, `opts.onProgress`, firing + `actStart`/`actEnd` per act (including per rule-engine `test` act, with `which` + naming the tool and `actEnd` carrying `outcome`/`elapsedMs`), built for Testaro's own + issue #44. +- However, **no current Testaro entry point supplies `opts`**: `call.js`, `dirWatch.js`, + and `netWatch.js` (the one a Kilotest Testaro worker actually runs) all call + `doJob(job)` with a single argument. `opts.onProgress` is dead code from any shipped + Testaro CLI/watch surface today. +- Consuming detailed progress therefore requires a **Testaro-side revision** to + `netWatch.js` (to accept/forward progress, plausibly keyed off a job property rather + than a runtime `opts` argument, since `netWatch.js`'s call site has no second argument + to attach a callback to) plus a **new worker-to-Kilotest channel** (the worker + currently only talks to Kilotest at claim-time and report-time; no mid-job channel + exists). This is real, separate scope: a Testaro release plus a worker deployment + update, before Kilotest's side could even be attempted. File as its own Testaro issue + when picked up, not bundled into this work. +- Generic progress (this plan) needed none of this, since Kilotest already knows + claimed-vs-reported from its own state. + +## Tool 3: orderRetest + +Same shape as `orderTest`, but for retesting a page with an existing report: target is +`{timeStamp, jobID}` of the report to retest, matching `requestRetest`'s own parameters. +Added at the user's explicit request, alongside full REST/MCP parity, for symmetry with +`orderTest`. Reuses the same `orderJob` function (see Implementation notes above) and the +same `awaitTest` tool for waiting; no separate wait tool was needed. + +## Out of scope for this increment + +- Cost recovery (explicit in the issue's own comment). +- Any change to the Testaro worker or its single-worker polling model. +- Any human-approval step for `orderTest`/`orderRetest`-originated jobs. + +## Likely shared-code extraction + +`requestTest.ts` and `requestRetest.ts` already duplicate a failure-reason mapping +(`requestResult` → human-readable string) and a `requestDisposition` envelope shape +(`'what happens next'` / `'how you can check for completion'` / `'how a web user can +check for completion'`). A third near-identical need in `orderTest.ts` makes this real +duplication (three call sites), worth extracting into small helpers in `api/util.ts` +before or alongside writing `orderTest.ts`. The outer response-body envelope +(`'tool collection'`, `'this request'`, etc.) is NOT worth extracting further; it is +already assembled from `getToolsFacts()`/`getResponseMetadata()` and the remaining +per-tool differences (method, URL, body shape, ancestor request) are genuinely +tool-specific. + +See also `user_maintainer_context` for the legibility-for-future-maintainers priority +this design tries to respect (explicit state checks over implicit invariants, e.g. why +`awaitTest` needed no special-casing once the actual constraint was understood, rather +than adding a defensive check for a scenario that cannot occur). diff --git a/openapi.yaml b/openapi.yaml index 777fcded..44b1fd38 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1181,6 +1181,436 @@ paths: - response metadata - response content additionalProperties: false + /api/orderTest: + post: + operationId: orderTest + summary: Order that a page be tested immediately, without manual approval + requestBody: + content: + application/json: + schema: + type: object + properties: + description: + type: string + description: 1- to 100-character description of the page conforming to the naming convention used in the listReports output + URL: + type: string + description: 12- to 300-character URL of the page, including the https:// scheme and any query + reason: + type: string + description: 20- to 100-character reason why the page should be tested + required: + - description + - URL + - reason + responses: + '200': + description: Order that a page be tested immediately, without manual approval + content: + application/json: + schema: + type: object + properties: + tool collection: + $ref: '#/components/schemas/ToolsFacts' + tool name: + type: string + this request: + type: object + properties: + description: + type: string + method: + type: string + const: POST + URL: + type: string + body: + type: object + properties: + description: + type: string + description: 1- to 100-character description of the page conforming to the naming convention used in the listReports output + URL: + type: string + description: 12- to 300-character URL of the page, including the https:// scheme and any query + reason: + type: string + description: 20- to 100-character reason why the page should be tested + required: + - description + - URL + - reason + additionalProperties: false + closest ancestor request: + anyOf: + - $ref: '#/components/schemas/RequestReference' + - type: 'null' + required: + - description + - method + - URL + - body + - closest ancestor request + additionalProperties: false + URLs of similar requests for web users: + type: object + properties: + this request: + type: 'null' + closest ancestor request: + type: + - string + - 'null' + required: + - this request + - closest ancestor request + additionalProperties: false + response metadata: + $ref: '#/components/schemas/ResponseMetadata' + response content: + type: object + properties: + details about your order: + anyOf: + - type: object + properties: + error: + type: string + required: + - error + additionalProperties: false + - type: object + properties: + date and time received: + type: string + page to be tested: + type: object + properties: + description: + type: string + URL: + type: string + required: + - description + - URL + additionalProperties: false + reason why the page should be tested: + type: string + required: + - date and time received + - page to be tested + - reason why the page should be tested + additionalProperties: false + disposition of your order: + anyOf: + - type: object + properties: + what happens next: + type: string + report identifier: + anyOf: + - $ref: '#/components/schemas/ReportIdentifier' + - type: 'null' + how you can check for completion: + type: string + do you want to wait for completion: + type: string + required: + - what happens next + - report identifier + - how you can check for completion + - do you want to wait for completion + additionalProperties: false + - type: 'null' + required: + - details about your order + - disposition of your order + additionalProperties: false + required: + - tool collection + - tool name + - this request + - URLs of similar requests for web users + - response metadata + - response content + additionalProperties: false + /api/orderRetest/{timeStamp}/{jobID}: + post: + operationId: orderRetest + summary: Order that a page be retested immediately, without manual approval + parameters: + - in: path + name: timeStamp + schema: + type: string + description: 'Timestamp of the latest report about the page in YYMMDDTHHmm format (example: 260503T0432)' + required: true + description: 'Timestamp of the latest report about the page in YYMMDDTHHmm format (example: 260503T0432)' + - in: path + name: jobID + schema: + type: string + description: 'Job identifier of the latest report about the page (example: x9z)' + required: true + description: 'Job identifier of the latest report about the page (example: x9z)' + requestBody: + content: + application/json: + schema: + type: object + properties: + reason: + type: string + description: 20- to 100-character reason why the page should be retested + required: + - reason + responses: + '200': + description: Order that a page be retested immediately, without manual approval + content: + application/json: + schema: + type: object + properties: + tool collection: + $ref: '#/components/schemas/ToolsFacts' + tool name: + type: string + this request: + type: object + properties: + description: + type: string + method: + type: string + const: POST + URL: + type: string + body: + type: object + properties: + reason: + type: string + description: 20- to 100-character reason why the page should be retested + required: + - reason + additionalProperties: false + closest ancestor request: + anyOf: + - $ref: '#/components/schemas/RequestReference' + - type: 'null' + required: + - description + - method + - URL + - body + - closest ancestor request + additionalProperties: false + URLs of similar requests for web users: + type: object + properties: + this request: + type: 'null' + closest ancestor request: + type: + - string + - 'null' + required: + - this request + - closest ancestor request + additionalProperties: false + response metadata: + $ref: '#/components/schemas/ResponseMetadata' + response content: + type: object + properties: + details about your order: + anyOf: + - type: object + properties: + error: + type: string + required: + - error + additionalProperties: false + - type: object + properties: + date and time received: + type: string + page to be retested: + type: object + properties: + description: + type: string + URL: + type: string + required: + - description + - URL + additionalProperties: false + reason why the page should be retested: + type: string + required: + - date and time received + - page to be retested + - reason why the page should be retested + additionalProperties: false + disposition of your order: + anyOf: + - type: object + properties: + what happens next: + type: string + report identifier: + anyOf: + - $ref: '#/components/schemas/ReportIdentifier' + - type: 'null' + how you can check for completion: + type: string + do you want to wait for completion: + type: string + required: + - what happens next + - report identifier + - how you can check for completion + - do you want to wait for completion + additionalProperties: false + - type: 'null' + required: + - details about your order + - disposition of your order + additionalProperties: false + required: + - tool collection + - tool name + - this request + - URLs of similar requests for web users + - response metadata + - response content + additionalProperties: false + /api/awaitTest/{timeStamp}/{jobID}: + post: + operationId: awaitTest + summary: Wait for a job ordered via orderTest or orderRetest to complete + parameters: + - in: path + name: timeStamp + schema: + type: string + description: 'Timestamp of the ordered job, from the report identifier returned by orderTest, in YYMMDDTHHmm format (example: 260503T0432)' + required: true + description: 'Timestamp of the ordered job, from the report identifier returned by orderTest, in YYMMDDTHHmm format (example: 260503T0432)' + - in: path + name: jobID + schema: + type: string + description: 'Job identifier of the ordered job, from the report identifier returned by orderTest (example: x9z)' + required: true + description: 'Job identifier of the ordered job, from the report identifier returned by orderTest (example: x9z)' + requestBody: + content: + application/json: + schema: + type: object + properties: {} + responses: + '200': + description: Wait for a job ordered via orderTest or orderRetest to complete + content: + application/json: + schema: + type: object + properties: + tool collection: + $ref: '#/components/schemas/ToolsFacts' + tool name: + type: string + this request: + type: object + properties: + description: + type: string + method: + type: string + const: POST + URL: + type: string + closest ancestor request: + anyOf: + - $ref: '#/components/schemas/RequestReference' + - type: 'null' + required: + - description + - method + - URL + - closest ancestor request + additionalProperties: false + URLs of similar requests for web users: + type: object + properties: + this request: + type: 'null' + closest ancestor request: + type: 'null' + required: + - this request + - closest ancestor request + additionalProperties: false + response metadata: + $ref: '#/components/schemas/ResponseMetadata' + response content: + type: object + properties: + details about your wait: + anyOf: + - type: object + properties: + error: + type: string + required: + - error + additionalProperties: false + - type: object + properties: + report identifier: + $ref: '#/components/schemas/ReportIdentifier' + required: + - report identifier + additionalProperties: false + disposition of your wait: + anyOf: + - type: object + properties: + outcome: + type: string + enum: + - completed + - failed + - timedOut + what happened: + type: string + how you can check for completion: + type: string + required: + - outcome + - what happened + - how you can check for completion + additionalProperties: false + - type: 'null' + required: + - details about your wait + - disposition of your wait + additionalProperties: false + required: + - tool collection + - tool name + - this request + - URLs of similar requests for web users + - response metadata + - response content + additionalProperties: false components: schemas: ToolsFacts: @@ -1253,7 +1683,9 @@ components: type: string method: type: string - const: GET + enum: + - GET + - POST URL: type: string required: @@ -1394,3 +1826,16 @@ components: required: - error additionalProperties: false + ReportIdentifier: + type: object + properties: + timeStamp: + type: string + description: Timestamp to pass to listIssues once the report is ready. + jobID: + type: string + description: Job identifier to pass to listIssues once the report is ready. + required: + - timeStamp + - jobID + additionalProperties: false diff --git a/package.json b/package.json index d6632fa8..b1aa49f0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@jrpool/kilotest", - "version": "55.0.0", + "version": "56.0.0", "description": "Ensemble testing for front-end web quality", "type": "module", "main": "index.ts", diff --git a/util.test.ts b/util.test.ts index 173a2ee9..e841786e 100644 --- a/util.test.ts +++ b/util.test.ts @@ -21,6 +21,7 @@ for (const key of ['MANAGER_EMAIL', 'ALERT_API_HOST', 'ALERT_API_PATH', 'ALERT_A import { annotateReportObject, + awaitJob, checkCommentDuplicate, checkCommentLength, checkLength, @@ -66,6 +67,7 @@ import { makeBreakable, minifyURL, objectSort, + orderJob, processTestRequest, testRequestsLock, testRequestsPath, @@ -1852,3 +1854,213 @@ test('getMetrics backfills every category, and generates a since stamp, from a b fsSync.rmSync(tmpDir, {recursive: true}); } }); + +// TESTS FOR orderJob AND awaitJob (GitHub issue #111) + +test('orderJob enqueues a new-page order directly into the job queue', async () => { + const {result, description, url, jobID} = await orderJob( + {description: 'Ordered Page', url: 'https://example.com/ordered'}, 'because accessibility matters here' + ) as any; + assert.equal(result, 'ok'); + assert.equal(description, 'Ordered Page'); + assert.equal(url, 'https://example.com/ordered'); + assert.match(jobID, /^\d{6}T\d{4}-[a-z0-9]{3}$/); + const queuedPath = path.join(jobsPath(), 'queue', `${jobID}.json`); + try { + const job = JSON.parse(await fs.readFile(queuedPath, 'utf8')); + assert.equal(job.target.what, 'Ordered Page'); + assert.equal(job.target.url, 'https://example.com/ordered'); + assert.equal(job.sources.reason, 'because accessibility matters here'); + // No pending-approval inbox entry is created; the job goes straight to the queue. + const testRequests = JSON.parse(await fs.readFile(testRequestsPath(), 'utf8')); + assert.equal(testRequests['https://example.com/ordered'], undefined); + } + finally { + await fs.unlink(queuedPath); + } +}); + +test('orderJob returns a retest error when a report already exists for the ordered page', async () => { + const {result} = await orderJob( + {description: 'Mixed Outcomes Page', url: 'https://example.com/mixed'}, 'because accessibility matters here' + ); + assert.equal(result, 'retest'); +}); + +test('orderJob returns "description" for a page matching a claimed job by description', async () => { + const claimedPath = path.join(jobsPath(), 'claimed', 'clm.json'); + await fs.writeFile(claimedPath, getJSON({ + target: {what: 'Claimed Order Page', url: 'https://example.com/claimed-order-job'} + })); + try { + const {result} = await orderJob( + {description: 'Claimed Order Page', url: 'https://example.com/some-other-order-url'}, + 'because accessibility matters here' + ); + assert.equal(result, 'description'); + } + finally { + await fs.unlink(claimedPath); + } +}); + +test('orderJob returns "url" for a page matching a queued job by URL', async () => { + const queuedPath = path.join(jobsPath(), 'queue', 'que.json'); + await fs.writeFile(queuedPath, getJSON({ + target: {what: 'Some Queued Order Page', url: 'https://example.com/queued-order-url'} + })); + try { + const {result} = await orderJob( + {description: 'A Different Order Page', url: 'https://example.com/queued-order-url'}, + 'because accessibility matters here' + ); + assert.equal(result, 'url'); + } + finally { + await fs.unlink(queuedPath); + } +}); + +test('orderJob throws when a job file in a category directory is defective', async () => { + const claimedPath = path.join(jobsPath(), 'claimed', 'bad.json'); + await fs.writeFile(claimedPath, 'not valid json'); + try { + await assert.rejects( + orderJob({description: 'Any Order Page', url: 'https://example.com/any-order'}, 'because accessibility matters here'), + /Failed to order test/ + ); + } + finally { + await fs.unlink(claimedPath); + } +}); + +test('orderJob returns "nonreport" when a retest order cites a report that does not exist', async () => { + const {result} = await orderJob({timeStamp: '999999T9999', jobID: 'xxx'}, 'because accessibility matters here'); + assert.equal(result, 'nonreport'); +}); + +test('orderJob returns "superseded" when a retest order cites a superseded report', async () => { + // 260101T0000-mix is an earlier report of "Mixed Outcomes Page" than 260202T0000-new, + // so it is superseded. + const {result} = await orderJob( + {timeStamp: '260101T0000', jobID: 'mix'}, 'because accessibility matters here' + ); + assert.equal(result, 'superseded'); +}); + +test('orderJob enqueues a retest order for the latest report of a page', async () => { + const {result, description, url, jobID} = await orderJob( + {timeStamp: '260202T0000', jobID: 'new'}, 'because accessibility matters here' + ) as any; + assert.equal(result, 'ok'); + assert.equal(description, 'Mixed Outcomes Page'); + assert.equal(url, 'https://example.com/mixed'); + const queuedPath = path.join(jobsPath(), 'queue', `${jobID}.json`); + try { + const job = JSON.parse(await fs.readFile(queuedPath, 'utf8')); + assert.equal(job.target.what, 'Mixed Outcomes Page'); + } + finally { + await fs.unlink(queuedPath); + } +}); + +test('orderJob honors JOB_QUEUE_MAX when the cap is reached', async () => { + process.env.JOB_QUEUE_MAX = '1'; + const fillerPath = path.join(jobsPath(), 'queue', 'fil.json'); + await fs.writeFile(fillerPath, getJSON({target: {what: 'Filler Queued Page', url: 'https://example.com/filler-queued'}})); + try { + const {result} = await orderJob( + {description: 'One Too Many Order Page', url: 'https://example.com/one-too-many-order'}, + 'because accessibility matters here' + ); + assert.equal(result, 'queueFull'); + } + finally { + delete process.env.JOB_QUEUE_MAX; + await fs.unlink(fillerPath); + } +}); + +test('orderJob never returns queueFull when JOB_QUEUE_MAX is 0 (no limit)', async () => { + process.env.JOB_QUEUE_MAX = '0'; + const fillerPath = path.join(jobsPath(), 'queue', 'fi2.json'); + await fs.writeFile(fillerPath, getJSON({target: {what: 'Filler Queued Page Two', url: 'https://example.com/filler-queued-2'}})); + try { + const {result, jobID} = await orderJob( + {description: 'One More Order Page', url: 'https://example.com/one-more-order'}, + 'because accessibility matters here' + ) as any; + assert.equal(result, 'ok'); + await fs.unlink(path.join(jobsPath(), 'queue', `${jobID}.json`)); + } + finally { + delete process.env.JOB_QUEUE_MAX; + await fs.unlink(fillerPath); + } +}); + +test('awaitJob returns "completed" immediately when a report already exists', async () => { + const outcome = await awaitJob('260101T0000', 'mix'); + assert.equal(outcome, 'completed'); +}); + +test('awaitJob returns "notFound" for an identifier naming neither a report nor a pending job', async () => { + const outcome = await awaitJob('999999T9999', 'xxx'); + assert.equal(outcome, 'notFound'); +}); + +test('awaitJob returns "failed" when the job has moved to the failed directory', async () => { + const failedPath = path.join(jobsPath(), 'failed', '260101T1111-flj.json'); + await fs.mkdir(path.dirname(failedPath), {recursive: true}); + await fs.writeFile(failedPath, getJSON({target: {what: 'Failed Await Page', url: 'https://example.com/failed-await'}})); + try { + const outcome = await awaitJob('260101T1111', 'flj'); + assert.equal(outcome, 'failed'); + } + finally { + await fs.unlink(failedPath); + } +}); + +test('awaitJob polls until a queued job completes, then returns "completed"', async () => { + process.env.AWAIT_JOB_POLL_MS = '20'; + process.env.AWAIT_JOB_TIMEOUT_MS = '2000'; + const jobName = '260101T2222-plj'; + const queuedPath = path.join(jobsPath(), 'queue', `${jobName}.json`); + const reportPath = path.join(reportsPath(), `${jobName}.json`); + await fs.writeFile(queuedPath, getJSON({target: {what: 'Polled Page', url: 'https://example.com/polled'}})); + // Simulate the job completing shortly after the wait begins. + setTimeout(async () => { + await fs.unlink(queuedPath).catch(() => {}); + await fs.writeFile(reportPath, getJSON({id: jobName, target: {what: 'Polled Page', url: 'https://example.com/polled'}})); + }, 50); + try { + const outcome = await awaitJob('260101T2222', 'plj'); + assert.equal(outcome, 'completed'); + } + finally { + delete process.env.AWAIT_JOB_POLL_MS; + delete process.env.AWAIT_JOB_TIMEOUT_MS; + await fs.unlink(queuedPath).catch(() => {}); + await fs.unlink(reportPath).catch(() => {}); + } +}); + +test('awaitJob returns "timedOut" when a claimed job neither completes nor fails before the deadline', async () => { + process.env.AWAIT_JOB_POLL_MS = '10'; + process.env.AWAIT_JOB_TIMEOUT_MS = '50'; + const jobName = '260101T3333-toj'; + const claimedPath = path.join(jobsPath(), 'claimed', `${jobName}.json`); + await fs.writeFile(claimedPath, getJSON({target: {what: 'Stuck Page', url: 'https://example.com/stuck'}})); + try { + const outcome = await awaitJob('260101T3333', 'toj'); + assert.equal(outcome, 'timedOut'); + } + finally { + delete process.env.AWAIT_JOB_POLL_MS; + delete process.env.AWAIT_JOB_TIMEOUT_MS; + await fs.unlink(claimedPath); + } +}); diff --git a/util.ts b/util.ts index a8d800b7..a92623af 100644 --- a/util.ts +++ b/util.ts @@ -1228,6 +1228,169 @@ export const processTestRequest = ( throw new Error('Failed to process test request', {cause: error}); } }); +// Maximum number of jobs allowed in the job queue (db/jobs/queue) at once. This queue is +// shared by every path that enqueues a job: manual approval of a pending test/retest +// request (web/enqueue), and any tool (such as orderTest) that enqueues directly. The cap +// therefore protects the queue itself, not any one source of jobs; a value of 0 means no +// limit. Configurable via JOB_QUEUE_MAX. +const jobQueueMax = () => getEnvMax('JOB_QUEUE_MAX', 20); +// Result of orderJob. description and url are absent only when the result is +// 'nonreport' (a retest order citing a report that does not exist), matching +// ProcessRequestResult's own discriminated-union shape. There is no pending approval +// inbox to compare against, so 'duplicate' (an identical pending request) never applies. +// jobID (the enqueued job's own identifier, timeStamp-jobID) is present only on 'ok'. +export type OrderJobResult += {result: 'nonreport'} +| {result: 'ok'; description: string; url: string; jobID: string} +| {result: Exclude; description: string; url: string}; +// Orders a test or retest as a transaction: validates the target is not already queued, +// claimed, or (for a new-page order) reported, then enqueues a job directly into +// db/jobs/queue, bypassing the pending-approval inbox (testRequests.json) that +// processTestRequest writes to. This is the auto-approval GitHub issue #111 asks for: a +// lightweight gate (the same claimed/queued dedup check processTestRequest already +// performs, plus a cap on the queue itself) substitutes for a maintainer's manual +// review. Runs under testRequestsLock, the same lock processTestRequest uses, so an +// orderJob call and a processTestRequest call can never race to enqueue duplicate jobs +// for the same page. The caller's stated reason is recorded on the job (sources.reason), +// which survives into the eventual report exactly as sources.worker already does: no +// approval rule consults it yet, but accumulating real callers' stated reasons is +// intended to inform the design of more sophisticated, reason-aware approval rules in a +// later iteration. +export const orderJob = ( + target: TestRequestTarget, reason: string +): Promise => testRequestsLock(async (): Promise => { + let description: string; + let url: string; + // Whether the cited report (for a retest order) has been superseded by a later one. + let superseded = false; + // If the target identifies a report to retest: + if ('timeStamp' in target) { + const {timeStamp, jobID} = target; + // Get the cited report. + const extract = await getReportExtract(timeStamp, jobID); + // If it does not exist: + if ('error' in extract) { + // Return this. + return {result: 'nonreport'}; + } + ({description, url} = extract); + // Get whether it has been superseded by a later report. + const reportExtracts = await getReportExtracts(); + superseded = reportExtracts.some( + ex => ex.timeStamp === timeStamp && ex.jobID === jobID && ex.superseded + ); + } + // Otherwise, i.e. if the target is a page to test for the first time: + else { + ({description, url} = target); + } + try { + // Get any property the page shares with a claimed job. + const claimedJobProperty = await getApprovedJobProperty(description, url, 'claimed'); + // Get any property the page shares with a queued job. + const queueJobProperty = await getApprovedJobProperty(description, url, 'queue'); + // If the page shares a property with a claimed or queued job: + if (claimedJobProperty || queueJobProperty) { + // Return the property. + return { + result: (claimedJobProperty || queueJobProperty) as 'description' | 'url', + description, + url + }; + } + // Otherwise, if the order is to retest a page that has been superseded: + if ('timeStamp' in target && superseded) { + // Return this. + return {result: 'superseded', description, url}; + } + // Otherwise, if the order is to test a new page for which a report already exists: + if (!('timeStamp' in target)) { + const reportExtracts = await getReportExtracts(); + if (reportExtracts.some(report => report.description === description && report.url === url)) { + // Return this. + return {result: 'retest', description, url}; + } + } + // Otherwise, if the job queue is already full (a cap of 0 means no limit, so the + // queue is never full): + const queueMax = jobQueueMax(); + const {queue: queuedJobNames} = await getJobNames(); + if (queueMax > 0 && queuedJobNames.length >= queueMax) { + // Return this. + return {result: 'queueFull', description, url}; + } + // The page is eligible, so build a job from the template. + const jobTemplateJSON = await fs.readFile(path.join(import.meta.dirname, 'job.json'), 'utf8'); + const job = JSON.parse(jobTemplateJSON); + const nowStamp = getNowStamp(); + const jobIDSuffix = getRandomString(3); + const jobName = `${nowStamp}-${jobIDSuffix}`; + job.id = jobName; + job.creationTimeStamp = nowStamp; + job.executionTimeStamp = nowStamp; + job.target.what = description; + job.target.url = url; + job.sources.reason = reason; + // Save the job in the queue. + await fs.writeFile(path.join(jobsPath(), 'queue', `${jobName}.json`), getJSON(job)); + console.log(`Test ordered for ${description} as job ${jobName}`); + // Return success, with the job identifier so the caller can report it. + return {result: 'ok', description, url, jobID: jobName}; + // If an error occurred: + } catch(error) { + // Throw it. + throw new Error('Failed to order test', {cause: error}); + } +}); +// Milliseconds between polls in awaitJob, and the maximum total milliseconds awaitJob +// waits before giving up. Both are read at call time (not cached), via environment +// variables, so tests can shrink them well below the real ~2-to-4-minute wait a +// production caller experiences; AWAIT_JOB_TIMEOUT_MS deliberately exceeds the +// documented "occasionally up to 4 minutes" a caller is told to expect, leaving margin +// so a job that is genuinely still running is not mistaken for one that never will +// finish. +const awaitJobPollMs = () => getEnvMax('AWAIT_JOB_POLL_MS', 2000); +const awaitJobTimeoutMs = () => getEnvMax('AWAIT_JOB_TIMEOUT_MS', 5 * 60 * 1000); +// Result of awaitJob. +export type AwaitJobResult = 'completed' | 'failed' | 'timedOut' | 'notFound'; +// Waits for a job ordered via orderJob (or, in principle, any job in db/jobs/queue or +// db/jobs/claimed identified the same way) to leave the queue/claimed pipeline, by +// polling for the outcome Kilotest already records for it: a completed report at +// timeStamp-jobID.json (see index.ts's worker/report handler, which is the sole writer +// of that file), or the job's own file having moved to db/jobs/failed (see +// index.ts's processJobRequest and worker/report handler, which are the only +// code that reclassifies a job as failed). Returns 'notFound' immediately, without +// polling, if the identifier names neither a completed report nor a job anywhere in the +// queue/claimed/failed pipeline, since there is then nothing to wait for. No progress +// notifications are emitted during the wait; this is a pure bounded poll. +export const awaitJob = async (timeStamp: string, jobID: string): Promise => { + const jobFileName = `${timeStamp}-${jobID}.json`; + const pollMs = awaitJobPollMs(); + const timeoutMs = awaitJobTimeoutMs(); + const deadline = Date.now() + timeoutMs; + // Returns the job's outcome if it is now determinable, or null if it is still pending. + const checkOutcome = async (): Promise => { + if (await getReportStats(timeStamp, jobID)) { + return 'completed'; + } + const jobNames = await getJobNames(); + if (jobNames.failed.includes(jobFileName)) { + return 'failed'; + } + if (jobNames.queue.includes(jobFileName) || jobNames.claimed.includes(jobFileName)) { + return null; + } + // The identifier names neither a report nor a job anywhere in the pipeline. + return 'notFound'; + }; + // If the job is not yet determinable, poll until it is or the deadline passes. + let outcome = await checkOutcome(); + while (outcome === null && Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, pollMs)); + outcome = await checkOutcome(); + } + return outcome ?? 'timedOut'; +}; // METRICS FUNCTIONS From 315eca2155725a04491da03dbd322268c08aad8f Mon Sep 17 00:00:00 2001 From: Jonathan Pool Date: Mon, 28 Sep 2026 03:15:04 -0700 Subject: [PATCH 2/5] debug --- api/awaitTest.test.ts | 16 ++++++++++++---- api/awaitTest.ts | 27 +++++++++++++++++++-------- api/schemas.ts | 22 ++++++++++++++++------ mcp.ts | 4 ++-- memory/plan_issue_111_instant_test.md | 18 ++++++++++++++++-- openapi.yaml | 8 +++++--- 6 files changed, 70 insertions(+), 25 deletions(-) diff --git a/api/awaitTest.test.ts b/api/awaitTest.test.ts index b8122627..f7090deb 100644 --- a/api/awaitTest.test.ts +++ b/api/awaitTest.test.ts @@ -95,12 +95,20 @@ test('awaitTest returns "timedOut" when a claimed job neither completes nor fail } }); -test('awaitTest response has no web UI equivalents for this request or its ancestor', async () => { +test('awaitTest response has no web UI equivalents for this request or its ancestors', async () => { const body = await response(['260101T0000', 'mix']); const similarWeb = body['URLs of similar requests for web users'] as any; assert.equal(similarWeb['this request'], null); assert.equal(similarWeb['closest ancestor request'], null); - const ancestor = (body['this request'] as any)['closest ancestor request']; - assert.equal(ancestor['tool name'], 'orderTest'); - assert.equal(ancestor.method, 'POST'); +}); + +test('awaitTest reports both orderTest and orderRetest as possible ancestors', async () => { + const body = await response(['260101T0000', 'mix']); + const ancestors = (body['this request'] as any)['closest ancestor request']; + assert.equal(ancestors.length, 2); + const names = ancestors.map((a: any) => a['tool name']).sort(); + assert.deepEqual(names, ['orderRetest', 'orderTest']); + for (const ancestor of ancestors) { + assert.equal(ancestor.method, 'POST'); + } }); diff --git a/api/awaitTest.ts b/api/awaitTest.ts index bbe10182..4395247e 100644 --- a/api/awaitTest.ts +++ b/api/awaitTest.ts @@ -1,6 +1,6 @@ /* awaitTest.ts - Waits for a job ordered via orderTest to complete, and returns its outcome. + Waits for a job ordered via orderTest or orderRetest to complete, and returns its outcome. */ // IMPORTS @@ -75,15 +75,26 @@ export const response = async (args: string[]) => { 'tool collection': getToolsFacts(), 'tool name': 'awaitTest', 'this request': { - description: 'Wait for a job ordered via the orderTest tool to complete. The timeStamp and jobID parameters are the report identifier that orderTest returned. This call blocks until the report is ready, the job fails, or a maximum wait time elapses.', + description: 'Wait for a job ordered via the orderTest or orderRetest tool to complete. The timeStamp and jobID parameters are the report identifier that orderTest or orderRetest returned. This call blocks until the report is ready, the job fails, or a maximum wait time elapses.', method: 'POST', URL: `${thisHost}/api/awaitTest/${timeStamp}/${jobID}`, - 'closest ancestor request': { - 'tool name': 'orderTest', - description: 'Process my order to test a page about which no report is available yet, immediately and without manual approval.', - method: 'POST', - URL: `${thisHost}/api/orderTest` - } + // awaitTest's own parameters do not say whether the job was ordered via orderTest + // or orderRetest, so both are reported as possible ancestors, rather than + // arbitrarily naming just one. + 'closest ancestor request': [ + { + 'tool name': 'orderTest', + description: 'Process my order to test a page about which no report is available yet, immediately and without manual approval.', + method: 'POST', + URL: `${thisHost}/api/orderTest` + }, + { + 'tool name': 'orderRetest', + description: 'Process my order to retest a page, immediately and without manual approval.', + method: 'POST', + URL: `${thisHost}/api/orderRetest` + } + ] }, 'URLs of similar requests for web users': { 'this request': null, diff --git a/api/schemas.ts b/api/schemas.ts index c8baa672..02c59943 100644 --- a/api/schemas.ts +++ b/api/schemas.ts @@ -115,7 +115,8 @@ const toolsFactsSchema = z.object({ // Identifies a related request (e.g., the closest ancestor request in the drill-down // hierarchy). method is usually GET (every drill-down tool's ancestor is a read), except -// for awaitTest, whose only ancestor, orderTest, is a POST (it creates a job). +// for orderTest/orderRetest, whose own ancestor is a GET, and awaitTest, whose ancestors +// (orderTest and orderRetest) are both POST (they create a job). const requestReferenceSchema = z.object({ 'tool name': z.string(), description: z.string(), @@ -124,23 +125,28 @@ const requestReferenceSchema = z.object({ }).meta({id: 'RequestReference'}); // GET endpoints have no 'body'; POST endpoints get a request-specific body schema. -const thisRequestSchema = (method: string, bodySchema?: ZodTypeAny) => z.object({ +// ancestorSchema overrides the default single-ancestor shape for a tool whose closest +// ancestor is not unique: awaitTest can be reached from either orderTest or orderRetest, +// and the identifier alone does not say which, so it reports both as an array rather +// than arbitrarily picking one. +const thisRequestSchema = (method: string, bodySchema?: ZodTypeAny, ancestorSchema: ZodTypeAny = requestReferenceSchema.nullable()) => z.object({ description: z.string(), method: z.literal(method), URL: z.string(), ...(bodySchema ? {body: bodySchema} : {}), - 'closest ancestor request': requestReferenceSchema.nullable() + 'closest ancestor request': ancestorSchema }); const envelope = ( method: string, responseContentSchema: T, bodySchema?: ZodTypeAny, - similarWebSchema: ZodTypeAny = similarWebRequestsSchema + similarWebSchema: ZodTypeAny = similarWebRequestsSchema, + ancestorSchema?: ZodTypeAny ) => z.object({ 'tool collection': toolsFactsSchema, 'tool name': z.string(), - 'this request': thisRequestSchema(method, bodySchema), + 'this request': thisRequestSchema(method, bodySchema, ancestorSchema), 'URLs of similar requests for web users': similarWebSchema, 'response metadata': responseMetadataSchema, 'response content': responseContentSchema @@ -418,5 +424,9 @@ export const awaitTestResponseSchema = envelope( z.object({ 'this request': z.null(), 'closest ancestor request': z.null() - }) + }), + // awaitTest can be reached from either orderTest or orderRetest, and its own + // parameters (timeStamp/jobID) do not say which one produced them, so both possible + // ancestors are reported, rather than arbitrarily naming just one as "the" ancestor. + z.array(requestReferenceSchema).length(2) ); diff --git a/mcp.ts b/mcp.ts index 63ec0a71..bdeb0404 100644 --- a/mcp.ts +++ b/mcp.ts @@ -304,11 +304,11 @@ export const createMCPServer = (): McpServer => { server.registerTool( 'awaitTest', { - description: toolDoc('Wait for a job ordered via the orderTest tool to complete, using the report identifier orderTest returned. Blocks until the report is ready, the job fails, or a maximum wait time elapses; sends no interim notices.'), + description: toolDoc('Wait for a job ordered via the orderTest or orderRetest tool to complete, using the report identifier either tool returned. Blocks until the report is ready, the job fails, or a maximum wait time elapses; sends no interim notices.'), inputSchema: awaitTestSchema, outputSchema: awaitTestResponseSchema, annotations: { - title: toolDoc('Wait for a job ordered via the orderTest tool to complete.'), + title: toolDoc('Wait for a job ordered via the orderTest or orderRetest tool to complete.'), readOnlyHint: true, idempotentHint: false, destructiveHint: false, diff --git a/memory/plan_issue_111_instant_test.md b/memory/plan_issue_111_instant_test.md index b8edb538..d9e04c1a 100644 --- a/memory/plan_issue_111_instant_test.md +++ b/memory/plan_issue_111_instant_test.md @@ -74,8 +74,22 @@ it enqueues synchronously and returns the job's own identifier immediately. "web users will not be offered an order option." The *ancestor* request's web URL (e.g. `listReports.html`, `listIssues.html`) is still populated where one exists. - `requestReferenceSchema` (`api/schemas.ts`) had to be widened from `method: - z.literal('GET')` to `z.enum(['GET', 'POST'])`, since `awaitTest`'s only ancestor, - `orderTest`/`orderRetest`, is a POST, unlike every other tool's GET ancestor. + z.literal('GET')` to `z.enum(['GET', 'POST'])`, since `awaitTest`'s ancestors, + `orderTest`/`orderRetest`, are POSTs, unlike every other tool's GET ancestor. +- **`awaitTest` has two possible ancestors, not one**: a bug in the first cut had + `awaitTest.ts` unconditionally naming `orderTest` as `'closest ancestor request'`, + wrong for any identifier that actually came from `orderRetest`, and `awaitTest` + cannot tell which one produced a given `timeStamp`/`jobID`, nor does it need to (its + poll logic is origin-agnostic, per the earlier finding above). Considered and + rejected: adding an `orderType: 'test' | 'retest'` parameter so `awaitTest` could + report a single correct ancestor; rejected because it would force every caller to + track and pass through a fact `awaitTest`'s own behavior never uses, reintroducing + exactly the origin-coupling the generic design deliberately avoided. Adopted instead: + `'closest ancestor request'` reports both `orderTest` and `orderRetest` as an array, + only for `awaitTest` (every other tool keeps a single nullable ancestor object). + Implemented by giving `thisRequestSchema`/`envelope` (`api/schemas.ts`) an optional + `ancestorSchema` override parameter, mirroring the existing `similarWebSchema` + override pattern, rather than widening the shared schema for every tool. - The caller's stated `reason` is persisted onto the job itself (`job.sources.reason`), surviving into the eventual report exactly as `sources.worker` already does. No approval rule consults it yet; the user's rationale for keeping the field even though diff --git a/openapi.yaml b/openapi.yaml index 44b1fd38..df5e1f02 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1538,9 +1538,11 @@ paths: URL: type: string closest ancestor request: - anyOf: - - $ref: '#/components/schemas/RequestReference' - - type: 'null' + minItems: 2 + maxItems: 2 + type: array + items: + $ref: '#/components/schemas/RequestReference' required: - description - method From e69c479a0bec552c982e215ff9ffb3e29c6ca17a Mon Sep 17 00:00:00 2001 From: Jonathan Pool Date: Mon, 28 Sep 2026 04:02:06 -0700 Subject: [PATCH 3/5] debug --- api/awaitTest.ts | 10 +++++----- env.example | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/api/awaitTest.ts b/api/awaitTest.ts index 4395247e..053730eb 100644 --- a/api/awaitTest.ts +++ b/api/awaitTest.ts @@ -27,7 +27,7 @@ export const response = async (args: string[]) => { // If the timestamp or job ID is not syntactically valid: if (!isTimeStamp(timeStamp) || !isJobID(jobID)) { requestDetails = { - error: 'wait invalid: the report timestamp or job identifier is malformed' + error: 'wait invalid: the report timeStamp or jobID is malformed' }; } // Otherwise, i.e. if the identifier is syntactically valid: @@ -53,14 +53,14 @@ export const response = async (args: string[]) => { else if (outcome === 'failed') { requestDisposition = { outcome: 'failed', - 'what happened': 'The job failed and no report will be produced.', + 'what happened': 'The job failed, so no report will be produced.', 'how you can check for completion': 'Not applicable.' }; } else { requestDisposition = { outcome: 'timedOut', - 'what happened': 'The job has not completed within the maximum wait time, but it may still be running.', + 'what happened': 'The job was not completed within the maximum wait time, but it may still be running.', 'how you can check for completion': `Call the listIssues tool with timeStamp "${timeStamp}" and jobID "${jobID}" after a few minutes to check whether the report is ready.` }; } @@ -84,13 +84,13 @@ export const response = async (args: string[]) => { 'closest ancestor request': [ { 'tool name': 'orderTest', - description: 'Process my order to test a page about which no report is available yet, immediately and without manual approval.', + description: 'Fulfill or reject my order to test a page about which no report is available yet.', method: 'POST', URL: `${thisHost}/api/orderTest` }, { 'tool name': 'orderRetest', - description: 'Process my order to retest a page, immediately and without manual approval.', + description: 'Fulfill or reject my order to retest a page.', method: 'POST', URL: `${thisHost}/api/orderRetest` } diff --git a/env.example b/env.example index e79e498f..f59506c4 100644 --- a/env.example +++ b/env.example @@ -48,10 +48,21 @@ TUTORIAL_AI_COMMENTS_MAX=__placeholder__ # Maximum number of feature requests held via the MCP server's requestFeature tool. FEATURE_REQUESTS_MAX=__placeholder__ # Maximum number of jobs allowed in the job queue (db/jobs/queue) at once. Shared by every -# path that enqueues a job: manual approval of a pending test/retest request, and the -# orderTest MCP tool, which enqueues directly. +# path that enqueues a job: manager approval of a pending test/retest request, and the +# orderTest/orderRetest MCP tools, which enqueue directly. JOB_QUEUE_MAX=__placeholder__ +# Timing for the awaitTest MCP tool's wait on a job ordered via orderTest/orderRetest. +# Defaults: 2000ms poll interval, 300000ms (5 minutes) maximum wait. The maximum wait +# deliberately exceeds the "occasionally up to 4 minutes" callers are told to expect, to +# leave margin so a job that is genuinely still running is not mistaken for one that +# never will finish. Lower values are useful for local testing. + +# Milliseconds between polls while awaitTest waits for a job to complete. +AWAIT_JOB_POLL_MS=__placeholder__ +# Maximum milliseconds awaitTest waits before giving up. +AWAIT_JOB_TIMEOUT_MS=__placeholder__ + # Alert configuration: MANAGER_EMAIL=__placeholder__ From 0d57e6605ae1a119ade43c8f77cbb9a59961bd81 Mon Sep 17 00:00:00 2001 From: Jonathan Pool Date: Mon, 28 Sep 2026 10:53:36 -0700 Subject: [PATCH 4/5] add secrets file to untracked files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 61be5ddd..47ee66f6 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ coverage/ # Secrets and environment-specific configurations .env +.env-prod .secrets From cf0a78cea85e43a0a4611df952d6b18a355e8d93 Mon Sep 17 00:00:00 2001 From: Jonathan Pool Date: Mon, 28 Sep 2026 10:54:35 -0700 Subject: [PATCH 5/5] update dependencies --- package-lock.json | 138 +++++++++++++++++++++++----------------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/package-lock.json b/package-lock.json index 78eae757..d4ac0257 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@jrpool/kilotest", - "version": "55.0.0", + "version": "56.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@jrpool/kilotest", - "version": "55.0.0", + "version": "56.0.0", "license": "MIT", "dependencies": { "@modelcontextprotocol/sdk": "*", @@ -1115,9 +1115,9 @@ "license": "ISC" }, "node_modules/@redocly/cli": { - "version": "2.54.3", - "resolved": "https://registry.npmjs.org/@redocly/cli/-/cli-2.54.3.tgz", - "integrity": "sha512-tgjCjKm9NjcbN0N04dylwUw8eZzematq07JHm+dVyPkAfA/eZEN57En7jpUYrPvwOb53MJLfDB/0qJ5iz/4eHQ==", + "version": "2.55.0", + "resolved": "https://registry.npmjs.org/@redocly/cli/-/cli-2.55.0.tgz", + "integrity": "sha512-YKDNfgv7hHsbWeudSLw8CQumX8LZeztdBht82yfFZLIg62BL6srKBzM0SIG+Ct+cnkHfkM0jzJ7OSP0I62W53Q==", "dev": true, "license": "MIT", "bin": { @@ -2789,17 +2789,17 @@ } }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.70.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.70.1.tgz", - "integrity": "sha512-nDNrUQ/4ruSNYbu749TRY7cfrzPtoLHEXSNBI8aaNY32LlZCajixqRf3FqcKC4p5Cam4VOHYx/t+i5+nKXvrqA==", + "version": "8.71.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.71.0.tgz", + "integrity": "sha512-pqcS9c1HxZTHt7End4nXqd0s5lJrrFzrgCkKFJrsbUnaL6M3+6oBFZaslg6Gjsl3argl2DDRFROnXARaZ2e4Nw==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.70.1", - "@typescript-eslint/type-utils": "8.70.1", - "@typescript-eslint/utils": "8.70.1", - "@typescript-eslint/visitor-keys": "8.70.1", + "@typescript-eslint/scope-manager": "8.71.0", + "@typescript-eslint/type-utils": "8.71.0", + "@typescript-eslint/utils": "8.71.0", + "@typescript-eslint/visitor-keys": "8.71.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" @@ -2812,7 +2812,7 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.70.1", + "@typescript-eslint/parser": "^8.71.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } @@ -2828,16 +2828,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.70.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.70.1.tgz", - "integrity": "sha512-nO974WLllwhSFWQXnMLj6nDGa8f0khKEz1JzpPJ1u7Vm/4X1X6ZHajpoknU4bb41vJyMB0HHVyS2GqdhWfIXZw==", + "version": "8.71.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.71.0.tgz", + "integrity": "sha512-CG4nPk1f2zc8yw4pALqHsFYH2hdo+h1T9daSp21+Hnxi9LOE3GT9hAfTKJCBXVNM2GmYs1eMEP615wPoeOgk3A==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.70.1", - "@typescript-eslint/types": "8.70.1", - "@typescript-eslint/typescript-estree": "8.70.1", - "@typescript-eslint/visitor-keys": "8.70.1", + "@typescript-eslint/scope-manager": "8.71.0", + "@typescript-eslint/types": "8.71.0", + "@typescript-eslint/typescript-estree": "8.71.0", + "@typescript-eslint/visitor-keys": "8.71.0", "debug": "^4.4.3" }, "engines": { @@ -2853,14 +2853,14 @@ } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.70.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.70.1.tgz", - "integrity": "sha512-62xOgboPfwc3/IgPSX/W6oQR3ZbF04194FPGUGH8HL8iLFHbt/456/8Ph1wLNUgVF+s94FlHoipBsz+v7+LMnA==", + "version": "8.71.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.71.0.tgz", + "integrity": "sha512-aABjw5rjBacYONVPaPiWOCjJu0vEF4a25iQuodlmQYL1trtLZ0X/y+2Vzl3BKI1odM4LnwLE1oUDXYp1wzx1TQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.70.1", - "@typescript-eslint/types": "^8.70.1", + "@typescript-eslint/tsconfig-utils": "^8.71.0", + "@typescript-eslint/types": "^8.71.0", "debug": "^4.4.3" }, "engines": { @@ -2875,14 +2875,14 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.70.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.70.1.tgz", - "integrity": "sha512-Pa0EeSeAusQc1WbjQMac+YfenewYTBu0KjgYvkUKwhXaHUKbFog23Dm/rp0DX/6tyYOQ3Xl1a+3EcFNZynGHCw==", + "version": "8.71.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.71.0.tgz", + "integrity": "sha512-gWF0BhUcnjZxSpLE8ngS/59n2SB0J3YqRxvX1+2aoRJk9hNtHSLOV+TcarFiOr5ipXm3yc1QrI4c9YZc8zyCxw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.70.1", - "@typescript-eslint/visitor-keys": "8.70.1" + "@typescript-eslint/types": "8.71.0", + "@typescript-eslint/visitor-keys": "8.71.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2893,9 +2893,9 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.70.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.70.1.tgz", - "integrity": "sha512-jumze1fPI+sDOaM2TWGQdn39PDxTr7TZGeuyLkAbNyx2vtMT3uRnVKChN0hfht5V2TugphJzF6bYXvBcE09qqg==", + "version": "8.71.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.71.0.tgz", + "integrity": "sha512-Z1UlWHADEK2Mlb9NpWfDeSjqoZ5EyrOv4R3eQpbkzqn/EwaIdOpXXupEA1+0ZIOSJSZZDBHG0BrQyN8zUG6Pwg==", "dev": true, "license": "MIT", "engines": { @@ -2910,15 +2910,15 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.70.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.70.1.tgz", - "integrity": "sha512-7zKTnyvaVWqzLZHPFQtX1hVHqgkMC+WebPWakNCSyrQVbIP1AM0L0TlBZtACldIRb6PptI8Odk+jyZ5kP3B1VA==", + "version": "8.71.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.71.0.tgz", + "integrity": "sha512-i8uO1qbdxeKgRnS5sCRt6On3/nfo2d2DwQe3Yvjx543zLy7r8ySqRuPPiIIXAhS03U0v5NfAFx+rUgxFzKKwNw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.70.1", - "@typescript-eslint/typescript-estree": "8.70.1", - "@typescript-eslint/utils": "8.70.1", + "@typescript-eslint/types": "8.71.0", + "@typescript-eslint/typescript-estree": "8.71.0", + "@typescript-eslint/utils": "8.71.0", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, @@ -2935,9 +2935,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "8.70.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.70.1.tgz", - "integrity": "sha512-Dm1ypdhhrGCTyyehxElhgJ6kgk8MVCv5qXdoOVqPr1uqk42jX8KjrZqhROvdShczA8qrDoYiOWn1ykWlx2k81Q==", + "version": "8.71.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.71.0.tgz", + "integrity": "sha512-cJ4OoxPGWvFnBTnSZyaU+qJzGTqPTGJY+gDchj6cRyLRdmIdt4rcsE4twj+zPfrNiWuVi38wijHzShL++Z9atQ==", "dev": true, "license": "MIT", "engines": { @@ -2949,16 +2949,16 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.70.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.70.1.tgz", - "integrity": "sha512-TU8PwyGN0PQJUcE96mw8eCQ44SmxGdQlJmlWakHaHQ15eIuuvye5yNtmh/i6oS88jzXVQB71xdNkbkB/fMwL0g==", + "version": "8.71.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.71.0.tgz", + "integrity": "sha512-PEEF4G5sLLWAS5BpPrUvms4ySZkiBQQZM4z+3ReI46axK5Vqr/vXBQatJQIZZOYdGyPUAKTtsrWzpqKuU+3DEw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.70.1", - "@typescript-eslint/tsconfig-utils": "8.70.1", - "@typescript-eslint/types": "8.70.1", - "@typescript-eslint/visitor-keys": "8.70.1", + "@typescript-eslint/project-service": "8.71.0", + "@typescript-eslint/tsconfig-utils": "8.71.0", + "@typescript-eslint/types": "8.71.0", + "@typescript-eslint/visitor-keys": "8.71.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", @@ -2977,16 +2977,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.70.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.70.1.tgz", - "integrity": "sha512-Esgul8MsnKnRLdYU2Eb2cRV9bS5HJYtKj1ByJnOzzG2M58DGdSUQ1jUuILxipqcpB2h9WLrbD5GijIWUjX/Tqw==", + "version": "8.71.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.71.0.tgz", + "integrity": "sha512-pKR/tEMVrXZG23UFKUn5BQf3zfmfk7KQceI2cGzywZ5nxM5Eu3hEJU1utjWzydtzBbcJAQhHN8iPCxobHpPcZQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.70.1", - "@typescript-eslint/types": "8.70.1", - "@typescript-eslint/typescript-estree": "8.70.1" + "@typescript-eslint/scope-manager": "8.71.0", + "@typescript-eslint/types": "8.71.0", + "@typescript-eslint/typescript-estree": "8.71.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3001,13 +3001,13 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.70.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.70.1.tgz", - "integrity": "sha512-Vwj9lUIW5Xq3wQ9w6gv3R86g1hMK8f2zNOdGTAgeXUMMXFK78G9ruCjjqutHMNJc0+CH7LYRnHeUB9IT8wFmcw==", + "version": "8.71.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.71.0.tgz", + "integrity": "sha512-8eQ9R218XORK+KLosnf4bu/QsUXvUyVwTbArg7/0NMB1Pu87OJKvj4nhFblkYE8gQV73mW1dx1ptlPCkwRGa7A==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.70.1", + "@typescript-eslint/types": "8.71.0", "eslint-visitor-keys": "^5.0.0" }, "engines": { @@ -5525,9 +5525,9 @@ } }, "node_modules/hono": { - "version": "4.13.9", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.9.tgz", - "integrity": "sha512-7dMkQmZoC4E6F7AtaQSPhlWAdnBti+j7rreMZl8QB4jFiEhP9TWbGWUMi8WYzBCgmgulxuvLQupKqo+Co6Omyg==", + "version": "4.13.10", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.10.tgz", + "integrity": "sha512-dQuLsa5oO+47QVMVMaaD9cIv8ctmVtK1iRvwWngkfloFJMeFeuoUFDswIqZGxmGX3hrRzREgEArjkK9OgsQEhA==", "license": "MIT", "engines": { "node": ">=16.9.0" @@ -9503,16 +9503,16 @@ } }, "node_modules/typescript-eslint": { - "version": "8.70.1", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.70.1.tgz", - "integrity": "sha512-AcWG7KDjZ2THNXsgwttMaGmzVi0VFRlFYfqFHYQRbDpF3owuYbuiL8c7UUrd2k8s3PoSfIQrWfrGXfcElrWLYA==", + "version": "8.71.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.71.0.tgz", + "integrity": "sha512-fBdHYiqQ14RW6mOMXD14Svn82ZsCYAoQSzGRzyEjR59S5A2Krh/l7fGTOQ7iCr8gGy/mHVXtEF7s5fgjEdV0Pw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.70.1", - "@typescript-eslint/parser": "8.70.1", - "@typescript-eslint/typescript-estree": "8.70.1", - "@typescript-eslint/utils": "8.70.1" + "@typescript-eslint/eslint-plugin": "8.71.0", + "@typescript-eslint/parser": "8.71.0", + "@typescript-eslint/typescript-estree": "8.71.0", + "@typescript-eslint/utils": "8.71.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0"