From a449810386fb7e3bd3e2fe951bb72b3569a8e9c0 Mon Sep 17 00:00:00 2001 From: ayushsingh82 Date: Mon, 17 Aug 2026 13:36:29 +0530 Subject: [PATCH] fix: report site/adapter-source command errors cleanly instead of crashing webcmd site * (notes, endpoints, field-maps, fixtures, samples) and adapter source get/put/path had no error handling, so any failure surfaced as a raw unhandled-rejection stack trace and always exited 1, discarding the intended CliError exit code (e.g. a missing fixture should exit 66, a bad flag 2). Wrap these actions the same way the rest of the CLI reports errors. Also adds e2e coverage for the local-mode portion of the manual QA checklist in #311 (override/source workflow, site memory, browser init/verify flags, mutable input-output files). --- src/cli.test.ts | 38 +- src/cli.ts | 26 +- src/site-memory/commands.ts | 50 ++- tests/e2e/adapter-authoring-parity.test.ts | 417 +++++++++++++++++++++ vitest.config.ts | 1 + 5 files changed, 504 insertions(+), 28 deletions(-) create mode 100644 tests/e2e/adapter-authoring-parity.test.ts diff --git a/src/cli.test.ts b/src/cli.test.ts index b4831fca..35760902 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -160,8 +160,16 @@ describe('site-memory and local adapter authoring', () => { }); it('rejects unresolved local adapter source paths', async () => { - await expect(createProgram('', '').parseAsync(['node', 'webcmd', 'adapter', 'path', 'missing/search'])) - .rejects.toThrow(/Adapter source is unavailable/); + const stderrSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const previousExitCode = process.exitCode; + try { + await createProgram('', '').parseAsync(['node', 'webcmd', 'adapter', 'path', 'missing/search']); + expect(stderrSpy.mock.calls.flat().join('\n')).toContain('Adapter source is unavailable'); + expect(process.exitCode).toBe(2); + } finally { + process.exitCode = previousExitCode; + stderrSpy.mockRestore(); + } }); it('rejects local adapter source writes while directing users to the source path', async () => { @@ -173,13 +181,22 @@ describe('site-memory and local adapter authoring', () => { getRegistry().set(key, { site: 'local-source', name: 'search', access: 'read', description: 'local source', args: [], source, } as never); + const stderrSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const previousExitCode = process.exitCode; try { - await expect(createProgram('', '').parseAsync(['node', 'webcmd', 'adapter', 'source', 'get', key, '--output', output])) - .rejects.toThrow(`webcmd adapter path ${key}`); - await expect(createProgram('', '').parseAsync(['node', 'webcmd', 'adapter', 'source', 'put', key, output])) - .rejects.toThrow(`webcmd adapter path ${key}`); + await createProgram('', '').parseAsync(['node', 'webcmd', 'adapter', 'source', 'get', key, '--output', output]); + expect(stderrSpy.mock.calls.flat().join('\n')).toContain(`webcmd adapter path ${key}`); + expect(process.exitCode).toBe(2); + + stderrSpy.mockClear(); + process.exitCode = previousExitCode; + await createProgram('', '').parseAsync(['node', 'webcmd', 'adapter', 'source', 'put', key, output]); + expect(stderrSpy.mock.calls.flat().join('\n')).toContain(`webcmd adapter path ${key}`); + expect(process.exitCode).toBe(2); } finally { getRegistry().delete(key); + process.exitCode = previousExitCode; + stderrSpy.mockRestore(); } }); @@ -191,11 +208,16 @@ describe('site-memory and local adapter authoring', () => { getRegistry().set(key, { site: 'stale-source', name: 'search', access: 'read', description: 'stale', args: [], source, } as never); + const stderrSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const previousExitCode = process.exitCode; try { - await expect(createProgram('', '').parseAsync(['node', 'webcmd', 'adapter', 'path', key])) - .rejects.toThrow(/Adapter source is unavailable/); + await createProgram('', '').parseAsync(['node', 'webcmd', 'adapter', 'path', key]); + expect(stderrSpy.mock.calls.flat().join('\n')).toContain('Adapter source is unavailable'); + expect(process.exitCode).toBe(2); } finally { getRegistry().delete(key); + process.exitCode = previousExitCode; + stderrSpy.mockRestore(); } }); }); diff --git a/src/cli.ts b/src/cli.ts index c7b2a09b..154e65a7 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1760,15 +1760,33 @@ cli({ return source; }; const reportLocalAdapterPath = (commandKey: string): void => console.log(localAdapterPath(commandKey)); + // Commander swallows a thrown error from a sync action as a raw stack trace + // and always exits 1, discarding a CliError's intended exit code (e.g. + // ArgumentError → 2). Route these through the same reporting the rest of + // the CLI uses so adapter source errors are clean and actionable. + const runAdapterSourceAction = (fn: () => void): void => { + try { + fn(); + } catch (err) { + console.error(`Error: ${getErrorMessage(err)}`); + process.exitCode = err instanceof CliError ? err.exitCode : EXIT_CODES.GENERIC_ERROR; + } + }; const adapterSourceCmd = adapterCmd.command('source').description('Inspect local adapter source paths; hosted mode reads or writes source'); adapterSourceCmd.command('get').description('Print local source path; --output is hosted-only').argument('').option('-o, --output ').action((commandKey: string, options: { output?: string }) => { - if (options.output) throw new ArgumentError(`Local adapter source get does not support --output. Use webcmd adapter path ${commandKey} and edit that file.`); - reportLocalAdapterPath(commandKey); + runAdapterSourceAction(() => { + if (options.output) throw new ArgumentError(`Local adapter source get does not support --output. Use webcmd adapter path ${commandKey} and edit that file.`); + reportLocalAdapterPath(commandKey); + }); }); adapterSourceCmd.command('put').description('Hosted-only source write; local users edit the adapter path').argument('').argument('').action((commandKey: string) => { - throw new ArgumentError(`Local adapter source put is unavailable. Use webcmd adapter path ${commandKey} and edit that file.`); + runAdapterSourceAction(() => { + throw new ArgumentError(`Local adapter source put is unavailable. Use webcmd adapter path ${commandKey} and edit that file.`); + }); + }); + adapterCmd.command('path').argument('').action((commandKey: string) => { + runAdapterSourceAction(() => reportLocalAdapterPath(commandKey)); }); - adapterCmd.command('path').argument('').action((commandKey: string) => reportLocalAdapterPath(commandKey)); // ── Built-in: browser profile selection ────────────────────────────────── const profileCmd = program.command('profile').description('Manage webcmd browser runtime profiles'); diff --git a/src/site-memory/commands.ts b/src/site-memory/commands.ts index bda01617..5945e59b 100644 --- a/src/site-memory/commands.ts +++ b/src/site-memory/commands.ts @@ -1,6 +1,6 @@ import { readFile, writeFile } from 'node:fs/promises'; import type { Command } from 'commander'; -import { ArgumentError, CliError, EXIT_CODES } from '../errors.js'; +import { ArgumentError, CliError, EXIT_CODES, getErrorMessage } from '../errors.js'; import { render as renderOutput } from '../output.js'; import { writeToStream } from '../stream-write.js'; import { @@ -33,51 +33,69 @@ export interface SiteMemoryBackend { sample(site: string, command: string, body: string): Promise; } +/** + * Commander doesn't format or set an exit code for a rejected async action — + * it surfaces as a raw unhandled-rejection stack trace and exit code 1, + * discarding CliError's intended exit code (e.g. SITE_MEMORY_NOT_FOUND → 66). + * Every action below goes through this so `site` errors look and behave like + * the rest of the CLI's error output. + */ +function wrapAction(fn: (...args: A) => Promise | void): (...args: A) => Promise { + return async (...args: A) => { + try { + await fn(...args); + } catch (err) { + console.error(`Error: ${getErrorMessage(err)}`); + process.exitCode = err instanceof CliError ? err.exitCode : EXIT_CODES.GENERIC_ERROR; + } + }; +} + export function registerSiteCommands(root: Command, backend: SiteMemoryBackend, stdout?: NodeJS.WritableStream): void { const site = root.command('site').description('Read and write site memory'); const memory = site.command('memory').description('Inspect site memory'); - memory.command('show').argument('').option('--kind ').option('-o, --output ').action(async (name, opts: { kind?: string; output?: string }) => { + memory.command('show').argument('').option('--kind ').option('-o, --output ').action(wrapAction(async (name, opts: { kind?: string; output?: string }) => { const result = await backend.show(name, parseKind(opts.kind)); if (opts.output) return writeFile(opts.output, `${JSON.stringify(result, null, 2)}\n`); await renderOutput(result, { fmt: 'json', stdout }); - }); - memory.command('list').argument('').option('-o, --output ').action(async (name, opts: { output?: string }) => { + })); + memory.command('list').argument('').option('-o, --output ').action(wrapAction(async (name, opts: { output?: string }) => { const result = await backend.list(name); if (opts.output) return writeFile(opts.output, `${JSON.stringify(result, null, 2)}\n`); await renderOutput(result, { fmt: 'table', fmtExplicit: true, columns: ['path', 'updatedAt', 'byteSize', 'sha256'], stdout }); - }); + })); site.command('note').command('add').argument('').requiredOption('--text ').option('--author ') - .action((name, opts: { text: string; author?: string }) => backend.note(name, opts.text, opts.author)); + .action(wrapAction((name, opts: { text: string; author?: string }) => backend.note(name, opts.text, opts.author))); const endpoint = site.command('endpoint').description('Maintain verified endpoints'); endpoint.command('set').argument('').argument('').requiredOption('--url ').requiredOption('--method ') .option('--params ').option('--rows-path ').option('--fields ').option('--notes ') - .action((siteName, name, opts: { url: string; method: string; params?: string; rowsPath?: string; fields?: string; notes?: string }) => backend.endpoint(siteName, name, { + .action(wrapAction((siteName, name, opts: { url: string; method: string; params?: string; rowsPath?: string; fields?: string; notes?: string }) => backend.endpoint(siteName, name, { url: opts.url, method: opts.method, ...(opts.params ? { params: parseJsonObject(opts.params) } : {}), ...(opts.rowsPath ? { rowsPath: opts.rowsPath } : {}), ...(opts.fields ? { sampleFields: opts.fields.split(',').map(value => value.trim()).filter(Boolean) } : {}), ...(opts.notes ? { notes: opts.notes } : {}), - })); - endpoint.command('stale').argument('').argument('').action((siteName, name) => backend.stale(siteName, name)); + }))); + endpoint.command('stale').argument('').argument('').action(wrapAction((siteName, name) => backend.stale(siteName, name))); site.command('field-map').command('add').argument('').argument('').requiredOption('--meaning ').requiredOption('--source ').option('--force') - .action((siteName, key, opts: { meaning: string; source: string; force?: boolean }) => backend.fieldMap(siteName, key, opts.meaning, opts.source, opts.force === true)); + .action(wrapAction((siteName, key, opts: { meaning: string; source: string; force?: boolean }) => backend.fieldMap(siteName, key, opts.meaning, opts.source, opts.force === true))); const fixture = site.command('fixture').description('Read and write verify fixtures'); - fixture.command('get').argument('').option('--output ').action(async (key, opts: { output?: string }) => { + fixture.command('get').argument('').option('--output ').action(wrapAction(async (key, opts: { output?: string }) => { const { site: siteName, command } = parseSiteCommand(key); const body = await backend.fixture(siteName, command); if (body === null) throw new CliError('SITE_MEMORY_NOT_FOUND', `Verify fixture ${key} was not found.`, undefined, EXIT_CODES.EMPTY_RESULT); if (opts.output) await writeFile(opts.output, body); else if (stdout) await writeToStream(stdout, body); else process.stdout.write(body); - }); - fixture.command('put').argument('').argument('').action(async (key, file) => { + })); + fixture.command('put').argument('').argument('').action(wrapAction(async (key, file) => { const { site: siteName, command } = parseSiteCommand(key); await backend.putFixture(siteName, command, await readFile(file, 'utf8')); - }); - site.command('sample').command('add').argument('').argument('').action(async (key, file) => { + })); + site.command('sample').command('add').argument('').argument('').action(wrapAction(async (key, file) => { const { site: siteName, command } = parseSiteCommand(key); await backend.sample(siteName, command, await readFile(file, 'utf8')); - }); + })); } export function createLocalSiteMemoryBackend(options: LocalStoreOptions = {}): SiteMemoryBackend { diff --git a/tests/e2e/adapter-authoring-parity.test.ts b/tests/e2e/adapter-authoring-parity.test.ts new file mode 100644 index 00000000..8a3ec7cc --- /dev/null +++ b/tests/e2e/adapter-authoring-parity.test.ts @@ -0,0 +1,417 @@ +/** + * E2E coverage for the local-mode portion of the manual QA checklist in + * https://github.com/agentrhq/webcmd/issues/311 ("validate adapter + * authoring and override parity in local and hosted modes"). + * + * Issue #311 is a manual test plan spanning local mode AND a hosted Cloud + * deployment (API keys, two tenants, hosted browser infra, workspace + * isolation, marketplace metadata). This file automates everything in that + * checklist that is testable purely in local mode: + * 1. Mode boundaries — local commands need no Cloud auth + * 2. Plugin override / adapter source get/put/path/reset + * 3. Site memory: notes, endpoints, field maps, fixtures, samples + * 4. `browser init` / `browser verify` and its flags + * 5. Mutable input-output files, local mode + * 8. Regression checks that apply locally + * Sections 6 (workspace/user isolation) and 7 (hosted marketplace) require a + * live WebCMD Cloud deployment and are out of scope here. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { runCli, parseJsonOutput, installFixturePlugin } from './helpers.js'; + +const TEST_HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'webcmd-adapter-parity-e2e-')); +const FIXTURE_SITE = 'dictionary'; +const FIXTURE_ENV = { HOME: TEST_HOME, USERPROFILE: TEST_HOME }; + +function run(args: string[], opts: { timeout?: number } = {}) { + return runCli(args, { ...opts, env: FIXTURE_ENV }); +} + +describe('adapter authoring & override parity (local mode) — issue #311', () => { + beforeAll(() => { + installFixturePlugin(TEST_HOME, FIXTURE_SITE); + }); + + afterAll(() => { + fs.rmSync(TEST_HOME, { recursive: true, force: true }); + }); + + // ── 1. Mode boundaries: local mode needs no Cloud auth ────────────────── + describe('1. mode boundaries', () => { + it('list works with no hosted credentials in the environment', async () => { + expect(FIXTURE_ENV).not.toHaveProperty('WEBCMD_API_KEY'); + const { stdout, code } = await run(['list', '-f', 'json']); + expect(code).toBe(0); + const data = parseJsonOutput(stdout); + expect(Array.isArray(data)).toBe(true); + expect(data.length).toBeGreaterThan(0); + }); + + it('plugin list works with no hosted credentials in the environment', async () => { + const { stdout, code } = await run(['plugin', 'list', '-f', 'json']); + expect(code).toBe(0); + const data = parseJsonOutput(stdout); + expect(data.some((p: any) => p.name === FIXTURE_SITE)).toBe(true); + }); + }); + + // ── 2. Plugin override and adapter source behavior ─────────────────────── + describe('2. plugin override and adapter source', () => { + const COMMAND = `${FIXTURE_SITE}/search`; + + it('override creates a local editable adapter that status/path/source get agree on', async () => { + const overridePath = path.join(TEST_HOME, '.webcmd', 'clis', FIXTURE_SITE, 'search.js'); + expect(fs.existsSync(overridePath)).toBe(false); + + const overrideResult = await run(['adapter', 'override', COMMAND]); + expect(overrideResult.code).toBe(0); + expect(fs.existsSync(overridePath)).toBe(true); + + const status = await run(['adapter', 'status', '-f', 'json']); + expect(status.code).toBe(0); + const entries = parseJsonOutput(status.stdout); + const entry = entries.find((e: any) => e.command === COMMAND); + expect(entry).toMatchObject({ kind: 'override', plugin: FIXTURE_SITE, orphaned: false }); + + const adapterPath = await run(['adapter', 'path', COMMAND]); + expect(adapterPath.code).toBe(0); + expect(adapterPath.stdout.trim()).toBe(overridePath); + + const sourceGet = await run(['adapter', 'source', 'get', COMMAND]); + expect(sourceGet.code).toBe(0); + expect(sourceGet.stdout.trim()).toBe(overridePath); + }); + + it('executing the command uses the override, not the plugin copy', async () => { + const overridePath = path.join(TEST_HOME, '.webcmd', 'clis', FIXTURE_SITE, 'search.js'); + const original = fs.readFileSync(overridePath, 'utf8'); + fs.writeFileSync(overridePath, original.replace(/description:\s*'[^']*'/, "description: 'PARITY_OVERRIDE_MARKER'")); + + const list = await run(['list', '-f', 'json']); + expect(list.code).toBe(0); + const data = parseJsonOutput(list.stdout); + const entry = data.find((c: any) => c.command === COMMAND); + expect(entry?.description).toBe('PARITY_OVERRIDE_MARKER'); + }); + + it('rejects local adapter source get --output without writing anything', async () => { + const destination = path.join(TEST_HOME, 'should-not-exist.js'); + const { stderr, code } = await run(['adapter', 'source', 'get', COMMAND, '--output', destination]); + expect(code).toBe(2); + expect(stderr).toContain(`webcmd adapter path ${COMMAND}`); + expect(fs.existsSync(destination)).toBe(false); + }); + + it('rejects local adapter source put without modifying the adapter', async () => { + const overridePath = path.join(TEST_HOME, '.webcmd', 'clis', FIXTURE_SITE, 'search.js'); + const before = fs.readFileSync(overridePath, 'utf8'); + const scratch = path.join(TEST_HOME, 'scratch-source.js'); + fs.writeFileSync(scratch, '// attempted overwrite\n'); + + const { stderr, code } = await run(['adapter', 'source', 'put', COMMAND, scratch]); + expect(code).toBe(2); + expect(stderr).toContain(`webcmd adapter path ${COMMAND}`); + expect(fs.readFileSync(overridePath, 'utf8')).toBe(before); + }); + + it('reset removes the override and restores plugin provenance', async () => { + const overridePath = path.join(TEST_HOME, '.webcmd', 'clis', FIXTURE_SITE, 'search.js'); + const { stdout, code } = await run(['adapter', 'reset', FIXTURE_SITE]); + expect(code).toBe(0); + expect(stdout).toContain('Removed local adapter override'); + expect(fs.existsSync(overridePath)).toBe(false); + + const status = await run(['adapter', 'status', '-f', 'json']); + const entries = parseJsonOutput(status.stdout); + expect(entries.find((e: any) => e.command === COMMAND)).toBeUndefined(); + + const list = await run(['list', '-f', 'json']); + const data = parseJsonOutput(list.stdout); + expect(data.find((c: any) => c.command === COMMAND)?.description).not.toBe('PARITY_OVERRIDE_MARKER'); + }); + + it('adapter path fails clearly for an unregistered command', async () => { + const { stderr, code } = await run(['adapter', 'path', `${FIXTURE_SITE}/does-not-exist`]); + expect(code).toBe(2); + expect(stderr).toContain('Adapter source is unavailable'); + }); + }); + + // ── 3. Site memory: notes, endpoints, field maps, fixtures, samples ───── + describe('3. site memory', () => { + const SITE = 'parity-site'; + const COMMAND = `${SITE}/search`; + + it('note add / memory show / memory list round-trip', async () => { + const note = await run(['site', 'note', 'add', SITE, '--text', 'Manual parity note', '--author', 'e2e-tester']); + expect(note.code).toBe(0); + + const show = await run(['site', 'memory', 'show', SITE, '--kind', 'notes']); + expect(show.code).toBe(0); + const body = parseJsonOutput(show.stdout); + expect(body[0].body).toContain('Manual parity note'); + + const list = await run(['site', 'memory', 'list', SITE]); + expect(list.code).toBe(0); + expect(list.stdout).toContain('notes.md'); + }); + + it('endpoint set / stale round-trip', async () => { + const set = await run([ + 'site', 'endpoint', 'set', SITE, 'search-api', + '--url', 'https://example.com/api/search', '--method', 'GET', + '--params', '{"q":"test"}', '--rows-path', 'items', '--fields', 'title,url', + ]); + expect(set.code).toBe(0); + + let show = await run(['site', 'memory', 'show', SITE, '--kind', 'endpoints']); + let body = parseJsonOutput(show.stdout); + let endpoints = JSON.parse(body[0].body); + expect(endpoints['search-api']).toMatchObject({ url: 'https://example.com/api/search', method: 'GET' }); + expect(endpoints['search-api'].stale).not.toBe(true); + + const stale = await run(['site', 'endpoint', 'stale', SITE, 'search-api']); + expect(stale.code).toBe(0); + + show = await run(['site', 'memory', 'show', SITE, '--kind', 'endpoints']); + body = parseJsonOutput(show.stdout); + endpoints = JSON.parse(body[0].body); + expect(endpoints['search-api'].stale).toBe(true); + }); + + it('marking an unknown endpoint stale fails cleanly instead of crashing', async () => { + const { stderr, code } = await run(['site', 'endpoint', 'stale', SITE, 'does-not-exist']); + expect(code).not.toBe(0); + expect(stderr).toContain('Error:'); + expect(stderr).not.toContain('at Command'); // no raw Node stack trace + }); + + it('field-map add rejects a duplicate key without --force, accepts it with --force', async () => { + const first = await run(['site', 'field-map', 'add', SITE, 'items[].title', '--meaning', 'Result title', '--source', 'manual-test']); + expect(first.code).toBe(0); + + const duplicate = await run(['site', 'field-map', 'add', SITE, 'items[].title', '--meaning', 'dup', '--source', 'manual-test']); + expect(duplicate.code).not.toBe(0); + expect(duplicate.stderr).toContain('already exists'); + expect(duplicate.stderr).not.toContain('at Command'); + + const forced = await run(['site', 'field-map', 'add', SITE, 'items[].title', '--meaning', 'Result title v2', '--source', 'manual-test', '--force']); + expect(forced.code).toBe(0); + + const show = await run(['site', 'memory', 'show', SITE, '--kind', 'field-map']); + const body = parseJsonOutput(show.stdout); + const mapping = JSON.parse(body[0].body); + expect(mapping['items[].title'].meaning).toBe('Result title v2'); + }); + + it('fixture put/get round-trips and rejects invalid fixtures without disturbing the valid one', async () => { + const goodFixture = path.join(TEST_HOME, 'good-fixture.json'); + const badJsonFixture = path.join(TEST_HOME, 'bad-fixture.json'); + const badRangeFixture = path.join(TEST_HOME, 'bad-range-fixture.json'); + fs.writeFileSync(goodFixture, JSON.stringify({ + args: { q: 'agent' }, + expect: { columns: ['title', 'url'], notEmpty: ['title'], rowCount: { min: 1 } }, + })); + fs.writeFileSync(badJsonFixture, 'not json'); + fs.writeFileSync(badRangeFixture, JSON.stringify({ expect: { rowCount: { min: 5, max: 1 } } })); + + const put = await run(['site', 'fixture', 'put', COMMAND, goodFixture]); + expect(put.code).toBe(0); + + const roundtripOut = path.join(TEST_HOME, 'roundtrip.json'); + const get = await run(['site', 'fixture', 'get', COMMAND, '--output', roundtripOut]); + expect(get.code).toBe(0); + expect(JSON.parse(fs.readFileSync(roundtripOut, 'utf8'))).toEqual(JSON.parse(fs.readFileSync(goodFixture, 'utf8'))); + + const putBadJson = await run(['site', 'fixture', 'put', COMMAND, badJsonFixture]); + expect(putBadJson.code).not.toBe(0); + expect(putBadJson.stderr).toContain('valid JSON'); + + const putBadRange = await run(['site', 'fixture', 'put', COMMAND, badRangeFixture]); + expect(putBadRange.code).not.toBe(0); + + // Previous valid fixture must survive both rejected writes. + const getAfter = await run(['site', 'fixture', 'get', COMMAND]); + expect(getAfter.code).toBe(0); + expect(JSON.parse(getAfter.stdout)).toEqual(JSON.parse(fs.readFileSync(goodFixture, 'utf8'))); + }); + + it('fixture get on a missing fixture fails cleanly with an empty-result exit code', async () => { + const { stderr, code } = await run(['site', 'fixture', 'get', `${SITE}/never-written`]); + expect(code).toBe(66); + expect(stderr).toContain('was not found'); + expect(stderr).not.toContain('at Command'); + }); + + it('sample add stores a response sample under fixtures/', async () => { + const sampleFile = path.join(TEST_HOME, 'sample.json'); + fs.writeFileSync(sampleFile, JSON.stringify({ items: [{ title: 'x', url: 'y' }] })); + const { code } = await run(['site', 'sample', 'add', COMMAND, sampleFile]); + expect(code).toBe(0); + + const list = await run(['site', 'memory', 'list', SITE]); + expect(list.code).toBe(0); + expect(list.stdout).toMatch(/fixtures\/search-\d+-/); + }); + + it('rejects a site name that attempts to escape the storage root', async () => { + const { stderr, code } = await run(['site', 'note', 'add', '../escape-attempt', '--text', 'x']); + expect(code).not.toBe(0); + expect(stderr).toContain('Invalid site memory site'); + expect(stderr).not.toContain('at Command'); + expect(fs.existsSync(path.join(TEST_HOME, '.webcmd', 'sites', '..', 'escape-attempt'))).toBe(false); + }); + }); + + // ── 4. Adapter authoring: browser init and verify ──────────────────────── + describe('4. browser init and verify', () => { + const NAME = 'parityverify/rows'; + const scaffoldPath = path.join(TEST_HOME, '.webcmd', 'clis', 'parityverify', 'rows.js'); + + it('init scaffolds an adapter and is idempotent', async () => { + expect(fs.existsSync(scaffoldPath)).toBe(false); + const first = await run(['browser', 'init', NAME]); + expect(first.code).toBe(0); + expect(first.stdout).toContain('Created:'); + expect(fs.existsSync(scaffoldPath)).toBe(true); + + const second = await run(['browser', 'init', NAME]); + expect(second.code).toBe(0); + expect(second.stdout).toContain('already exists'); + + const adapterPath = await run(['adapter', 'path', NAME]); + expect(adapterPath.stdout.trim()).toBe(scaffoldPath); + }, 20_000); + + it('verify runs the scaffold, reports no fixture, and warns (not fails) on missing memory', async () => { + const { stdout, code } = await run(['browser', 'verify', NAME, '--no-fixture'], { timeout: 20_000 }); + expect(code).toBe(0); + expect(stdout).toContain('Adapter runs'); + expect(stdout).toContain('Memory: missing endpoints.json, notes.md'); + }, 20_000); + + it('--strict-memory turns the missing-memory warning into a failure', async () => { + const { code } = await run(['browser', 'verify', NAME, '--no-fixture', '--strict-memory'], { timeout: 20_000 }); + expect(code).not.toBe(0); + }, 20_000); + + it('--write-fixture seeds a fixture, --update-fixture is required to overwrite it', async () => { + const fixturePath = path.join(TEST_HOME, '.webcmd', 'sites', 'parityverify', 'verify', 'rows.json'); + const first = await run(['browser', 'verify', NAME, '--write-fixture'], { timeout: 20_000 }); + expect(first.code).toBe(0); + expect(first.stdout).toContain('Wrote fixture'); + expect(fs.existsSync(fixturePath)).toBe(true); + + const second = await run(['browser', 'verify', NAME, '--write-fixture'], { timeout: 20_000 }); + expect(second.code).toBe(0); + expect(second.stdout).toContain('already exists'); + expect(second.stdout).toContain('--update-fixture'); + + const third = await run(['browser', 'verify', NAME, '--update-fixture'], { timeout: 20_000 }); + expect(third.code).toBe(0); + expect(third.stdout).toContain('Updated fixture'); + }, 30_000); + + it('rejects a non-positive --max-top-level-keys client-side', async () => { + const { stderr, code } = await run(['browser', 'verify', NAME, '--no-fixture', '--max-top-level-keys', '0']); + expect(code).toBe(2); + expect(stderr).toContain('--max-top-level-keys must be a positive integer'); + }); + + it('an invalid --trace mode fails the run instead of silently succeeding', async () => { + const { code } = await run(['browser', 'verify', NAME, '--no-fixture', '--trace', 'bogus'], { timeout: 20_000 }); + expect(code).not.toBe(0); + }, 20_000); + + it('--seed-args seeds adapter args when no fixture is used', async () => { + const { code } = await run(['browser', 'verify', NAME, '--no-fixture', '--seed-args', '{"limit":2}'], { timeout: 20_000 }); + expect(code).toBe(0); + }, 20_000); + }); + + // ── 5. Mutable input-output files, local mode ───────────────────────────── + describe('5. mutable input-output files', () => { + const filetestDir = path.join(TEST_HOME, '.webcmd', 'clis', 'filetest'); + + beforeAll(() => { + fs.mkdirSync(filetestDir, { recursive: true }); + fs.writeFileSync(path.join(filetestDir, 'resume.js'), ` +import { cli, Strategy } from '@agentrhq/webcmd/registry'; +import * as fs from 'node:fs'; + +cli({ + site: 'filetest', + name: 'resume', + description: 'mutable input-output file test adapter', + access: 'write', + example: 'webcmd filetest resume --resume-file /tmp/x.json', + domain: 'filetest', + strategy: Strategy.PUBLIC, + browser: false, + args: [ + { name: 'resume-file', type: 'string', required: false, file: { direction: 'input-output', pathKind: 'file', multiple: false, contentTypes: ['application/json'] }, help: 'Resume file' }, + ], + columns: ['status', 'count', 'path'], + func: async (kwargs) => { + const filePath = kwargs['resume-file']; + let state = { count: 0 }; + if (filePath && fs.existsSync(filePath)) { + state = JSON.parse(fs.readFileSync(filePath, 'utf8')); + } + state.count += 1; + if (filePath) fs.writeFileSync(filePath, JSON.stringify(state)); + return [{ status: 'ok', count: state.count, path: filePath ?? null }]; + }, +}); +`); + }); + + it('a missing mutable file is created by the command', async () => { + const resumeFile = path.join(TEST_HOME, 'resume-missing.json'); + expect(fs.existsSync(resumeFile)).toBe(false); + + const { stdout, code } = await run(['filetest', 'resume', '--resume-file', resumeFile, '-f', 'json']); + expect(code).toBe(0); + const rows = parseJsonOutput(stdout); + expect(rows[0]).toMatchObject({ status: 'ok', count: 1, path: resumeFile }); + expect(JSON.parse(fs.readFileSync(resumeFile, 'utf8'))).toEqual({ count: 1 }); + }); + + it('local mode passes the same path directly for an existing mutable file, and state persists across runs', async () => { + const resumeFile = path.join(TEST_HOME, 'resume-existing.json'); + fs.writeFileSync(resumeFile, JSON.stringify({ count: 5 })); + + const { stdout, code } = await run(['filetest', 'resume', '--resume-file', resumeFile, '-f', 'json']); + expect(code).toBe(0); + const rows = parseJsonOutput(stdout); + expect(rows[0]).toMatchObject({ status: 'ok', count: 6, path: resumeFile }); + expect(JSON.parse(fs.readFileSync(resumeFile, 'utf8'))).toEqual({ count: 6 }); + }); + + it('an ordinary command with no file argument still works unaffected', async () => { + const { code } = await run(['filetest', 'resume', '-f', 'json']); + expect(code).toBe(0); + }); + }); + + // ── 8. Regression checks (local-testable subset) ───────────────────────── + describe('8. regressions', () => { + it('unsupported output format on a built-in command produces a clean usage error', async () => { + const { stderr, code } = await run(['adapter', 'status', '-f', 'xml']); + expect(code).toBe(2); + expect(stderr).toContain('Unknown output format "xml"'); + }); + + it('site memory writes cannot escape their storage root via a crafted command key', async () => { + const fixtureFile = path.join(TEST_HOME, 'escape-fixture.json'); + fs.writeFileSync(fixtureFile, JSON.stringify({ expect: {} })); + const { stderr, code } = await run(['site', 'fixture', 'put', 'sitex/../../etc/passwd', fixtureFile]); + expect(code).not.toBe(0); + expect(stderr).toContain('site/command format'); + }); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 806d64f2..7a75094a 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -53,6 +53,7 @@ export default defineConfig({ 'tests/e2e/management.test.ts', 'tests/e2e/output-formats.test.ts', 'tests/e2e/plugin-management.test.ts', + 'tests/e2e/adapter-authoring-parity.test.ts', 'tests/e2e/article-download-pipeline.test.ts', 'tests/e2e/cloak-runtime.test.ts', 'tests/e2e/cloak-session-concurrency.test.ts',