diff --git a/docs/cli-reference.mdx b/docs/cli-reference.mdx index 0c3a1c6d..70358747 100644 --- a/docs/cli-reference.mdx +++ b/docs/cli-reference.mdx @@ -178,6 +178,16 @@ Each keeps its human-readable report as the `table` rendering, which stays the d `profile list` returns one row per profile with `contextId`, `alias`, `default`, `connected`, and `runtimeVersion`, covering both connected profiles and saved aliases that are not currently connected. If the daemon is unreachable or stale, `profile list -f json`/`-f yaml` fails with a `DAEMON_UNAVAILABLE` error (exit 1) and a restart hint instead of returning `[]` — an empty list and an unreadable runtime are different facts. +### Adapter verification reports + +`browser verify` accepts `-f/--format`. Its default `table` rendering is the human progress report; any other format returns the verification result as data instead: + +```bash +webcmd browser verify hn/top -f json +``` + +The report carries `ok`, `site`, `command`, `rowCount`, a `fixture` block (`path`, `exists`, `action`), and a `memory` block mirroring the site-memory check. Failures replace the prose with structured detail: `shapeFailures` for row-shape violations, `matchFailures` for fixture mismatches, and an `error` object with a `code` (`ADAPTER_NOT_FOUND`, `ADAPTER_EXEC_FAILED`, `ADAPTER_OUTPUT_NOT_JSON`) when the adapter could not be run or read. The exit code is unchanged in every case, so `-f json` is safe to add to an existing verification step. + ## Global Flags | Flag / Env | Purpose | diff --git a/skill-src/webcmd-usage/SKILL.src.md b/skill-src/webcmd-usage/SKILL.src.md index c243786b..2654e3c7 100644 --- a/skill-src/webcmd-usage/SKILL.src.md +++ b/skill-src/webcmd-usage/SKILL.src.md @@ -201,9 +201,11 @@ Scaffolding and checks: webcmd browser init / webcmd validate [target] webcmd verify [target] [--smoke] -webcmd browser verify / +webcmd browser verify / [-f json] ``` +`browser verify -f json` returns the verification result as data rather than the progress report: `ok`, `site`, `command`, `rowCount`, a `fixture` block, and a `memory` block, plus `shapeFailures`, `matchFailures`, or an `error.code` when it fails. Parse that instead of scraping the ✓/✗ text. + Adapters import only `@agentrhq/webcmd/registry` and `@agentrhq/webcmd/errors`. `columns` must align one-to-one, in name and order, with returned row object keys. See `webcmd-adapter-author`. ## Plugins diff --git a/skills/webcmd-usage/SKILL.md b/skills/webcmd-usage/SKILL.md index 3a2e7034..56173691 100644 --- a/skills/webcmd-usage/SKILL.md +++ b/skills/webcmd-usage/SKILL.md @@ -201,9 +201,11 @@ Scaffolding and checks: webcmd browser init / webcmd validate [target] webcmd verify [target] [--smoke] -webcmd browser verify / +webcmd browser verify / [-f json] ``` +`browser verify -f json` returns the verification result as data rather than the progress report: `ok`, `site`, `command`, `rowCount`, a `fixture` block, and a `memory` block, plus `shapeFailures`, `matchFailures`, or an `error.code` when it fails. Parse that instead of scraping the ✓/✗ text. + Adapters import only `@agentrhq/webcmd/registry` and `@agentrhq/webcmd/errors`. `columns` must align one-to-one, in name and order, with returned row object keys. See `webcmd-adapter-author`. ## Plugins diff --git a/src/cli.test.ts b/src/cli.test.ts index 4b8c2546..9f29e711 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -1677,6 +1677,158 @@ describe('browser verify', () => { fs.rmSync(fakeHome, { recursive: true, force: true }); } }); + + describe('structured output', () => { + const consoleLogSpy = vi.mocked(console.log); + + /** + * Runs `browser verify` against a throwaway adapter under a temp HOME and + * returns whatever reached stdout. Every structured-output case needs the + * same scaffolding; only the adapter rows and argv differ. + */ + const runVerify = async ( + slug: string, + adapterOutput: string, + argv: string[], + seedFixture?: unknown, + ): Promise => { + const originalHome = process.env.HOME; + const originalUserProfile = process.env.USERPROFILE; + const fakeHome = fs.mkdtempSync(path.join(os.tmpdir(), `webcmd-browser-verify-${slug}-`)); + process.env.HOME = fakeHome; + process.env.USERPROFILE = fakeHome; + mockExecFileSync.mockReturnValue(adapterOutput); + consoleLogSpy.mockClear(); + + try { + const adapterDir = path.join(fakeHome, '.webcmd', 'clis', 'hn'); + fs.mkdirSync(adapterDir, { recursive: true }); + fs.writeFileSync(path.join(adapterDir, 'top.js'), 'export default {};\n', 'utf-8'); + + if (seedFixture !== undefined) { + const verifyDir = path.join(fakeHome, '.webcmd', 'sites', 'hn', 'verify'); + fs.mkdirSync(verifyDir, { recursive: true }); + fs.writeFileSync(path.join(verifyDir, 'top.json'), JSON.stringify(seedFixture), 'utf-8'); + } + + await createProgram('', '').parseAsync(['node', 'webcmd', '--session', 'test', 'browser', 'verify', 'hn/top', ...argv]); + return consoleLogSpy.mock.calls.map((args) => args.join(' ')).join('\n'); + } finally { + consoleLogSpy.mockClear(); + if (originalHome === undefined) delete process.env.HOME; + else process.env.HOME = originalHome; + if (originalUserProfile === undefined) delete process.env.USERPROFILE; + else process.env.USERPROFILE = originalUserProfile; + fs.rmSync(fakeHome, { recursive: true, force: true }); + } + }; + + it('reports a fixture-less pass as a structured report with no prose', async () => { + const stdout = await runVerify('json-pass', JSON.stringify([{ title: 'ok' }]), ['--no-fixture', '-f', 'json']); + + expect(process.exitCode).toBeUndefined(); + const report = JSON.parse(stdout); + expect(report).toMatchObject({ + ok: true, + site: 'hn', + command: 'top', + rowCount: 1, + fixture: { exists: false, action: 'none' }, + }); + expect(report.memory).toMatchObject({ ok: false }); + // The emoji progress report is the table rendering; it must not leak into JSON. + expect(stdout).not.toContain('🔍'); + expect(stdout).not.toContain('Verifying'); + }); + + it('reports row-shape violations as structured failures', async () => { + const stdout = await runVerify( + 'json-shape', + JSON.stringify([{ title: 'ok', author: { user_id: 'u1' } }]), + ['--no-fixture', '-f', 'json'], + ); + + expect(process.exitCode).toBe(1); + const report = JSON.parse(stdout); + expect(report.ok).toBe(false); + expect(report.rowCount).toBe(1); + expect(report.shapeFailures).toEqual( + expect.arrayContaining([expect.objectContaining({ rule: 'shapeNestedId' })]), + ); + expect(stdout).not.toContain('violates row shape conventions'); + }); + + it('reports fixture mismatches as structured failures', async () => { + const stdout = await runVerify( + 'json-mismatch', + JSON.stringify([{ title: 'actual' }]), + ['-f', 'json'], + { expect: { columns: ['title'], rowCount: { min: 5 } } }, + ); + + expect(process.exitCode).toBe(1); + const report = JSON.parse(stdout); + expect(report.ok).toBe(false); + expect(report.fixture).toMatchObject({ exists: true }); + expect(report.matchFailures).toEqual( + expect.arrayContaining([expect.objectContaining({ rule: 'rowCount' })]), + ); + }); + + it('reports a missing adapter as a structured error', async () => { + const originalHome = process.env.HOME; + const originalUserProfile = process.env.USERPROFILE; + const fakeHome = fs.mkdtempSync(path.join(os.tmpdir(), 'webcmd-browser-verify-json-missing-')); + process.env.HOME = fakeHome; + process.env.USERPROFILE = fakeHome; + consoleLogSpy.mockClear(); + + try { + await createProgram('', '').parseAsync(['node', 'webcmd', '--session', 'test', 'browser', 'verify', 'hn/top', '-f', 'json']); + + expect(process.exitCode).toBe(1); + const report = JSON.parse(consoleLogSpy.mock.calls.map((args) => args.join(' ')).join('\n')); + expect(report).toMatchObject({ + ok: false, + site: 'hn', + command: 'top', + error: { code: 'ADAPTER_NOT_FOUND' }, + }); + } finally { + consoleLogSpy.mockClear(); + if (originalHome === undefined) delete process.env.HOME; + else process.env.HOME = originalHome; + if (originalUserProfile === undefined) delete process.env.USERPROFILE; + else process.env.USERPROFILE = originalUserProfile; + fs.rmSync(fakeHome, { recursive: true, force: true }); + } + }); + + it('rejects an unsupported format before running the adapter', async () => { + const originalHome = process.env.HOME; + const originalUserProfile = process.env.USERPROFILE; + const fakeHome = fs.mkdtempSync(path.join(os.tmpdir(), 'webcmd-browser-verify-json-badfmt-')); + process.env.HOME = fakeHome; + process.env.USERPROFILE = fakeHome; + + try { + const adapterDir = path.join(fakeHome, '.webcmd', 'clis', 'hn'); + fs.mkdirSync(adapterDir, { recursive: true }); + fs.writeFileSync(path.join(adapterDir, 'top.js'), 'export default {};\n', 'utf-8'); + + await createProgram('', '').parseAsync(['node', 'webcmd', '--session', 'test', 'browser', 'verify', 'hn/top', '--no-fixture', '-f', 'xml']); + + expect(process.exitCode).toBe(2); + expect(mockExecFileSync).not.toHaveBeenCalled(); + } finally { + if (originalHome === undefined) delete process.env.HOME; + else process.env.HOME = originalHome; + if (originalUserProfile === undefined) delete process.env.USERPROFILE; + else process.env.USERPROFILE = originalUserProfile; + fs.rmSync(fakeHome, { recursive: true, force: true }); + } + }); + }); }); describe('profile list', () => { diff --git a/src/cli.ts b/src/cli.ts index da953ffa..b86967b7 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -993,7 +993,7 @@ cli({ // ── Verify (test adapter) ── - browser.command('verify') + const browserVerifyCmd = browser.command('verify') .argument('', 'Adapter name in site/command format (e.g. hn/top)') .option('--write-fixture', 'Write a starter fixture to ~/.webcmd/sites//verify/.json if none exists') .option('--update-fixture', 'Overwrite an existing fixture with one derived from current output') @@ -1002,8 +1002,17 @@ cli({ .option('--seed-args ', 'Seed args when no fixture exists; use JSON array/object for multiple args or flags') .option('--trace ', 'Trace capture for the adapter subprocess: off, on, retain-on-failure', 'off') .option('--max-top-level-keys ', 'Override the row-shape top-level key cap (default: 12) for adapters whose rows are wide by design') - .description('Execute an adapter and validate output; uses fixture at ~/.webcmd/sites//verify/.json when present') - .action(async (name: string, opts: { fixture?: boolean; writeFixture?: boolean; updateFixture?: boolean; strictMemory?: boolean; seedArgs?: string; trace?: string; maxTopLevelKeys?: string } = {}) => { + .option('-f, --format ', OUTPUT_FORMAT_HELP, 'table') + .description('Execute an adapter and validate output; uses fixture at ~/.webcmd/sites//verify/.json when present'); + browserVerifyCmd.action(async (name: string, opts: { fixture?: boolean; writeFixture?: boolean; updateFixture?: boolean; strictMemory?: boolean; seedArgs?: string; trace?: string; maxTopLevelKeys?: string; format?: string } = {}) => { + const fmt = resolveOutputFormat(opts.format); + if (fmt === null) return; + const fmtExplicit = browserVerifyCmd.getOptionValueSource('format') === 'cli'; + const asTable = fmt === 'table'; + // Prose-only progress/detail lines. The structured report below carries the + // same facts as data; -f json/yaml callers get the report, not this text. + const notice = (...lines: string[]) => { if (asTable) for (const line of lines) console.log(line); }; + try { const parts = name.split('/'); if (parts.length !== 2) { console.error('Name must be site/command format'); process.exitCode = EXIT_CODES.USAGE_ERROR; return; } @@ -1028,17 +1037,19 @@ cli({ const { loadFixture, writeFixture, deriveFixture, validateRows, validateRowShape, fixturePath, expandFixtureArgs, parseSeedArgs } = await import('./browser/verify-fixture.js'); const filePath = path.join(os.homedir(), '.webcmd', 'clis', site, `${command}.js`); if (!fs.existsSync(filePath)) { - console.error(`Adapter not found: ${filePath}`); - console.error(`Run "webcmd browser init ${name}" to create it.`); + const message = `Adapter not found: ${filePath}`; + const hint = `Run "webcmd browser init ${name}" to create it.`; + if (asTable) { console.error(message); console.error(hint); } + else await renderOutput({ ok: false, site, command, error: { code: 'ADAPTER_NOT_FOUND', message, hint } }, { fmt, fmtExplicit }); process.exitCode = EXIT_CODES.GENERIC_ERROR; return; } - console.log(`🔍 Verifying ${name}...\n`); - console.log(` Loading: ${filePath}`); + notice(`🔍 Verifying ${name}...\n`, ` Loading: ${filePath}`); const useFixture = opts.fixture !== false; let fixture = useFixture ? loadFixture(site, command) : null; + const declaredFixturePath = fixturePath(site, command); // Build adapter args: fixture.args override the legacy --limit 3 heuristic. // - object form { "limit": 3 } → `--limit 3` @@ -1068,91 +1079,146 @@ cli({ ...(invocation.shell ? { shell: true } : {}), }); } catch (err) { - console.log(` Executing: webcmd ${site} ${command} ${argDisplay}\n`); + notice(` Executing: webcmd ${site} ${command} ${argDisplay}\n`); const execErr = err as { stdout?: string | Buffer; stderr?: string | Buffer }; - if (execErr.stdout) console.log(String(execErr.stdout)); - if (execErr.stderr) console.error(String(execErr.stderr).slice(0, 500)); - console.log(`\n ✗ Adapter failed. Fix the code and try again.`); + if (asTable) { + if (execErr.stdout) console.log(String(execErr.stdout)); + if (execErr.stderr) console.error(String(execErr.stderr).slice(0, 500)); + console.log(`\n ✗ Adapter failed. Fix the code and try again.`); + } else { + await renderOutput({ + ok: false, + site, + command, + error: { + code: 'ADAPTER_EXEC_FAILED', + message: 'Adapter failed to execute.', + ...(execErr.stderr ? { stderr: String(execErr.stderr).slice(0, 500) } : {}), + }, + }, { fmt, fmtExplicit }); + } process.exitCode = EXIT_CODES.GENERIC_ERROR; return; } - console.log(` Executing: webcmd ${site} ${command} ${argDisplay}\n`); + notice(` Executing: webcmd ${site} ${command} ${argDisplay}\n`); let rows: Record[]; try { rows = normalizeVerifyRows(JSON.parse(rawJson)); } catch { - console.log(rawJson); - console.log('\n ✗ Could not parse adapter output as JSON. Is `--format json` broken?'); + if (asTable) { + console.log(rawJson); + console.log('\n ✗ Could not parse adapter output as JSON. Is `--format json` broken?'); + } else { + await renderOutput({ + ok: false, + site, + command, + error: { code: 'ADAPTER_OUTPUT_NOT_JSON', message: 'Could not parse adapter output as JSON.' }, + }, { fmt, fmtExplicit }); + } process.exitCode = EXIT_CODES.GENERIC_ERROR; return; } - console.log(renderVerifyPreview(rows)); - console.log(`\n → ${rows.length} row${rows.length === 1 ? '' : 's'}`); + notice(renderVerifyPreview(rows), `\n → ${rows.length} row${rows.length === 1 ? '' : 's'}`); const shapeFailures = validateRowShape(rows, { maxTopLevelKeys }); if (shapeFailures.length > 0) { - console.log(`\n ✗ Adapter output violates row shape conventions:`); - for (const f of shapeFailures.slice(0, 20)) { - const where = f.rowIndex !== undefined ? `row[${f.rowIndex}] ` : ''; - console.log(` - [${f.rule}] ${where}${f.detail}`); - } - if (shapeFailures.length > 20) { - console.log(` ... and ${shapeFailures.length - 20} more failure(s)`); + if (asTable) { + console.log(`\n ✗ Adapter output violates row shape conventions:`); + for (const f of shapeFailures.slice(0, 20)) { + const where = f.rowIndex !== undefined ? `row[${f.rowIndex}] ` : ''; + console.log(` - [${f.rule}] ${where}${f.detail}`); + } + if (shapeFailures.length > 20) { + console.log(` ... and ${shapeFailures.length - 20} more failure(s)`); + } + console.log(`\n Keep rows agent-native: <=${maxTopLevelKeys ?? 12} top-level keys, nesting depth <=1, and id-shaped fields at top level.`); + console.log(` If this adapter's rows are wide by design, rerun with --max-top-level-keys .`); + } else { + await renderOutput({ ok: false, site, command, rowCount: rows.length, shapeFailures }, { fmt, fmtExplicit }); } - console.log(`\n Keep rows agent-native: <=${maxTopLevelKeys ?? 12} top-level keys, nesting depth <=1, and id-shaped fields at top level.`); - console.log(` If this adapter's rows are wide by design, rerun with --max-top-level-keys .`); process.exitCode = EXIT_CODES.GENERIC_ERROR; return; } // ── Fixture handling ─────────────────────────────────────────── + let fixtureAction: 'none' | 'written' | 'updated' | 'skipped-exists' = 'none'; if (opts.writeFixture || opts.updateFixture) { if (fixture && !opts.updateFixture) { - console.log(`\n Fixture already exists at ${fixturePath(site, command)}.`); - console.log(` Use --update-fixture to overwrite.`); + notice(`\n Fixture already exists at ${declaredFixturePath}.`, ` Use --update-fixture to overwrite.`); + fixtureAction = 'skipped-exists'; } else { const fixtureArgs = explicitArgs !== undefined ? explicitArgs : (hasLimitArg ? { limit: 3 } : undefined); const derived = deriveFixture(rows, fixtureArgs); const p = writeFixture(site, command, derived); - console.log(`\n ${fixture ? '↻ Updated' : '✎ Wrote'} fixture: ${p}`); - console.log(` Review and hand-tune the derived expectations (add patterns / notEmpty, tighten rowCount).`); + notice(`\n ${fixture ? '↻ Updated' : '✎ Wrote'} fixture: ${p}`, ` Review and hand-tune the derived expectations (add patterns / notEmpty, tighten rowCount).`); + fixtureAction = fixture ? 'updated' : 'written'; fixture = derived; } } if (!fixture) { - console.log(`\n ✓ Adapter runs. (No fixture at ${fixturePath(site, command)} — consider --write-fixture to seed one.)`); + notice(`\n ✓ Adapter runs. (No fixture at ${declaredFixturePath} — consider --write-fixture to seed one.)`); const memoryReport = checkSiteMemory(site); - printSiteMemoryReport(memoryReport, opts.strictMemory); - if (!memoryReport.ok && opts.strictMemory) { - process.exitCode = EXIT_CODES.GENERIC_ERROR; + if (asTable) printSiteMemoryReport(memoryReport, opts.strictMemory); + const ok = !(!memoryReport.ok && opts.strictMemory); + if (!asTable) { + await renderOutput({ + ok, + site, + command, + rowCount: rows.length, + fixture: { path: declaredFixturePath, exists: false, action: fixtureAction }, + memory: memoryReport, + }, { fmt, fmtExplicit }); } + if (!ok) process.exitCode = EXIT_CODES.GENERIC_ERROR; return; } const failures = validateRows(rows, fixture); if (failures.length === 0) { - console.log(`\n ✓ Adapter matches fixture (${fixturePath(site, command)}).`); + notice(`\n ✓ Adapter matches fixture (${declaredFixturePath}).`); const memoryReport = checkSiteMemory(site); - printSiteMemoryReport(memoryReport, opts.strictMemory); - if (!memoryReport.ok && opts.strictMemory) { - process.exitCode = EXIT_CODES.GENERIC_ERROR; + if (asTable) printSiteMemoryReport(memoryReport, opts.strictMemory); + const ok = !(!memoryReport.ok && opts.strictMemory); + if (!asTable) { + await renderOutput({ + ok, + site, + command, + rowCount: rows.length, + fixture: { path: declaredFixturePath, exists: true, action: fixtureAction }, + memory: memoryReport, + }, { fmt, fmtExplicit }); } + if (!ok) process.exitCode = EXIT_CODES.GENERIC_ERROR; return; } - console.log(`\n ✗ Adapter output does not match fixture:`); - for (const f of failures.slice(0, 20)) { - const where = f.rowIndex !== undefined ? `row[${f.rowIndex}] ` : ''; - console.log(` - [${f.rule}] ${where}${f.detail}`); - } - if (failures.length > 20) { - console.log(` ... and ${failures.length - 20} more failure(s)`); + if (asTable) { + console.log(`\n ✗ Adapter output does not match fixture:`); + for (const f of failures.slice(0, 20)) { + const where = f.rowIndex !== undefined ? `row[${f.rowIndex}] ` : ''; + console.log(` - [${f.rule}] ${where}${f.detail}`); + } + if (failures.length > 20) { + console.log(` ... and ${failures.length - 20} more failure(s)`); + } + } else { + await renderOutput({ + ok: false, + site, + command, + rowCount: rows.length, + fixture: { path: declaredFixturePath, exists: true, action: fixtureAction }, + matchFailures: failures, + }, { fmt, fmtExplicit }); } process.exitCode = EXIT_CODES.GENERIC_ERROR; } catch (err) {