diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index ed7f4d5..2697ae8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -29,7 +29,7 @@ body: attributes: label: Workflow and CLI version description: Record the workflowVersion from run-state.json and the installed skill version when known. - placeholder: workflow 1.1, skill 1.0.0 + placeholder: workflow 1.1, skill 1.1.0 validations: required: false - type: dropdown diff --git a/AGENTS.md b/AGENTS.md index 7f74780..511f697 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,8 +28,12 @@ When documentation and runtime behavior disagree, verify the CLI and tests, then - `validate` distinguishes valid-incomplete, valid-complete, and invalid state. Structural validation is not semantic assurance. - Stage-gate approval requires at least 8/10, no zero dimension, and zero blockers. - A requested revision must retain its ID and be resolved by an upstream artifact change or an explicit no-change disposition. +- `package.json` is the single source of truth for the release version. `SKILL.md` metadata, `CITATION.cff`, the README version badge, and a dated `CHANGELOG.md` release section must match it; `npm run validate` fails on any partial bump. +- Do not hardcode the release version anywhere else. Scripts that need it must read it from `package.json` or the shipped `SKILL.md` frontmatter. - The CLI is single-writer. Do not introduce concurrent state mutation without a designed locking/generation protocol and cross-platform tests. - Preserve atomic state replacement, managed-path containment, symlink rejection, non-overwrite behavior, and explicit exit classes. +- Preflight the complete initialization layout before the first scaffold write; a late path conflict must leave earlier missing files untouched. +- Keep `currentPhase` equal to the latest non-pending phase and keep stage-gate score, dimensions, blockers, and decision metadata consistent with the gate status. - Never convert skipped or zero-execution host checks into a pass. ## Implementation Style @@ -39,6 +43,7 @@ When documentation and runtime behavior disagree, verify the CLI and tests, then - Validate arguments and current state before filesystem mutation. - Use workspace-relative managed paths; reject traversal, symlinks, type conflicts, and unsafe slugs. - Preserve the previous valid `run-state.json` if a write fails. +- Repository validators and test fixtures must not follow symlinked directories or copy ignored local workflow/secrets into disposable repository clones. - Keep exit codes stable: `0` success, `2` usage, `3` workflow/inconclusive state, `4` filesystem or safety failure. - Add focused regression tests for every reproduced defect and verify rejected mutations leave state/artifacts unchanged. - Do not weaken tests, hide expected failures, or rewrite unrelated user changes to make a gate pass. @@ -51,6 +56,7 @@ Update all affected surfaces when behavior changes: - Required artifact heading or field: asset template, CLI contract table, benchmark fixture, tests, example run, and traceability guidance. - Host claim: README matrix, `references/compatibility.md`, and a dated eval with the exact host/version/check executed. - Package/release process: `RELEASING.md`, repository validator, changelog, and public wording. +- Release version: `package.json` first, then `skills/agentic-rd-skill/SKILL.md` metadata, `CITATION.cff`, the README badge, and a dated `CHANGELOG.md` section. Confirm with `npm run validate` rather than by inspection. - Security boundary: `SECURITY.md`, `quality-and-safety.md`, tests, and threat notes in the change description. Do not copy current package bytes or benchmark point values into undated README prose. Keep point-in-time measurements in dated eval files and make `npm run benchmark` the current source. diff --git a/CHANGELOG.md b/CHANGELOG.md index 8be1cc7..682c225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,18 @@ ## [Unreleased] +## [1.1.0] - 2026-08-01 + +Workflow contract 1.1 is now released. Existing workflow 1.0 state remains readable and there is no breaking change to the v1 artifact layout or CLI surface. + ### Added - Added workflow contract 1.1 with minimum phase headings, stable finding-coverage tables, safe multi-artifact scaffolding, configurable human-review metadata, and explicit valid-incomplete versus valid-complete output. - Added durable revision IDs, upstream-artifact fingerprints, and explicit no-change dispositions for stage-gate reapproval. - Added deterministic coverage for zero-host smoke behavior and the newly reproduced setup, revision-bypass, and invalid-state mutation paths. - Added repository-wide `AGENTS.md` instructions and a maintainer release-integrity checklist. +- Added a repository-validated release-version invariant: `package.json` is the single source of truth and `npm run validate` now fails when `SKILL.md` metadata, `CITATION.cff`, the README version badge, or a dated `CHANGELOG.md` release section disagrees with it. +- Added a regression test that mutates a copied repository tree to prove the release-version invariant reports drift on every checked surface. ### Changed @@ -22,6 +28,17 @@ - Prevented stage-gate reapproval from silently bypassing a requested revision. - Prevented `advance` and `finalize` from mutating state that fails full workflow validation. - Clarified protected tag/version selection versus GitHub immutable-release guarantees. +- Fixed the model-backed host smoke asserting a hardcoded `1.0.0` activation marker; it now reads the shipped `SKILL.md` metadata version, so a release bump can no longer make the activation check assert a version the package does not declare. +- Fixed the repository validator's Markdown walk descending into the local `.venv` and treating the ignored local `project-brief.md` dogfood artifact as tracked source, which made link validation depend on the contributor's local environment. A directory whose name ends in `.md` is also no longer misread as a file. +- Preflighted every managed initialization destination before the first write so a late file/directory conflict cannot leave partial scaffolding behind. +- Rejected contradictory `currentPhase` and stage-gate decision metadata before a state-changing command can persist a newly invalid workflow. +- Stopped repository Markdown validation from following symlinked directories and restricted disposable test copies to project-owned roots instead of ignored local files. +- Reported malformed `package.json` content as a validation failure instead of crashing the repository validator. + +### Known Issues + +- Symlink-rejection tests are skipped with `EPERM` on Windows hosts that cannot create symlinks; the Linux and macOS CI jobs remain the authoritative coverage for those paths. +- `npm run smoke:hosts` remains environment-dependent and exits 3 with `inconclusive` when no eligible host CLI is installed. The host activation matrix is still the dated 2026-07-18 snapshot and was not rerun for this release. ## [1.0.0] - 2026-07-31 diff --git a/CITATION.cff b/CITATION.cff index 5f3b256..919362f 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -5,6 +5,6 @@ authors: - family-names: Gasperini given-names: Michael url: "https://github.com/TheStreamCode/agentic-rd-skill" -version: "1.0.0" -date-released: 2026-07-31 +version: "1.1.0" +date-released: 2026-08-01 license: MIT diff --git a/README.md b/README.md index 6e820f8..ba7dac2 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![CI](https://github.com/TheStreamCode/agentic-rd-skill/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/TheStreamCode/agentic-rd-skill/actions/workflows/ci.yml) [![Agent Skills](https://img.shields.io/badge/Agent%20Skills-open%20standard-0969da)](https://agentskills.io/specification) -[![Version](https://img.shields.io/badge/version-1.0.0-2ea44f)](CHANGELOG.md) +[![Version](https://img.shields.io/badge/version-1.1.0-2ea44f)](CHANGELOG.md) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) Agentic R&D Skill is an open-source, portable multi-agent research and development workflow for Codex, Claude Code, GitHub Copilot, Gemini CLI, and OpenCode. It turns complex technical, product, business, feasibility, strategy, and investigation briefs into traceable evidence, executable plans, reviewed results, and gated final reports—without requiring a provider-specific LLM SDK. @@ -153,7 +153,7 @@ work/ The final output can be created only after a stage-gate score of at least 8/10, no zero-scored dimension, and no blocker. The CLI deliberately refuses v0.3 workspaces rather than guessing at a migration. -New runs use workflow contract 1.1: setup stays in progress until the filled brief and run log pass their minimum artifact contracts. Material findings receive stable IDs and coverage tables. Existing workflow 1.0 state remains readable. +New runs use workflow contract 1.1, released in skill version 1.1.0: setup stays in progress until the filled brief and run log pass their minimum artifact contracts. Material findings receive stable IDs and coverage tables. Existing workflow 1.0 state remains readable. Common commands: @@ -198,13 +198,17 @@ gh skill publish --dry-run [`evals/manifest.json`](evals/manifest.json) defines repeatable activation, safety, failure, gating, and efficiency scenarios. Metrics are recorded only when the host exposes them. This project does not claim cost, speed, or quality improvements without a measured comparison. +The dated [`v1.1.0 security review`](evals/security-review-v1.1.0.md) records the filesystem, state-integrity, secret-handling, and release-surface checks performed for this release, including residual single-writer and host-verification limitations. + The local benchmark uses a completed standard-profile workspace with four evidence artifacts, four execution artifacts, two result artifacts, and about 126 KiB of artifact data. It measures fresh-process `init`, `status`, and `validate` latency and enforces deliberately broad regression budgets; it is a CLI guardrail, not a claim about model response time or research quality. See the [v1.0.0 dogfood](evals/dogfood-v1.0.0.md) for the current real-case workflow evaluation and [`evals/usability-review.md`](evals/usability-review.md) for the earlier UX snapshot. Run `npm run benchmark` for the current package size and local timing snapshot. CI enforces broad limits of 256 KiB for the installable package and p95 limits of 750 ms for `init`, 500 ms for `status`, and 750 ms for `validate`. These guardrails cover local workflow bookkeeping, not model latency, token cost, or research quality; avoiding copied point-in-time numbers prevents README drift. ## Versioning -Version 1.0 introduced the breaking artifact layout and state contract. Unreleased workflow contract 1.1 adds truthful setup state, minimum artifact headings, revision history, and pre-mutation validation while retaining read compatibility with v1.0 state. The CLI intentionally does not migrate v0.3 runs. +The project follows semantic versioning. Version 1.0 introduced the breaking artifact layout and state contract. Version 1.1 released workflow contract 1.1, which adds truthful setup state, minimum artifact headings, revision history, and pre-mutation validation while retaining read compatibility with v1.0 state. The CLI intentionally does not migrate v0.3 runs. + +`package.json` is the single source of truth for the release version. `npm run validate` fails when the skill metadata, citation file, README badge, or dated changelog section disagrees with it, so a partial version bump cannot reach a tag. ## Frequently Asked Questions @@ -234,7 +238,7 @@ The stage gate scores alignment, evidence quality, execution correctness, risk a ## Maintainer, Citation, and Support -Agentic R&D Skill is maintained by [Michael Gasperini (Mikesoft)](https://mikesoft.it) through [TheStreamCode](https://github.com/TheStreamCode). Deterministic workflow/package checks were last rerun on July 31, 2026; the host activation matrix is a separate dated snapshot documented in the compatibility reference. Maintainers should follow [RELEASING.md](RELEASING.md) so release integrity and evidence dates are verified independently. +Agentic R&D Skill is maintained by [Michael Gasperini (Mikesoft)](https://mikesoft.it) through [TheStreamCode](https://github.com/TheStreamCode). Deterministic workflow/package checks were last rerun on August 1, 2026; the host activation matrix is a separate dated snapshot documented in the compatibility reference. Maintainers should follow [RELEASING.md](RELEASING.md) so release integrity and evidence dates are verified independently. For academic or published use, cite the project using [`CITATION.cff`](CITATION.cff). Contributions are welcome through the [contribution guide](CONTRIBUTING.md), and security issues should follow the [security policy](SECURITY.md). diff --git a/RELEASING.md b/RELEASING.md index e2fa35e..671d9aa 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -5,7 +5,7 @@ Use this checklist for every stable release. A passing local suite does not auth ## Prepare 1. Start from a clean `main` checkout synchronized with `origin/main`. -2. Align the intended version in `package.json`, `CITATION.cff`, `skills/agentic-rd-skill/SKILL.md`, README badge, and `CHANGELOG.md`. +2. Set the intended version in `package.json`, then align `skills/agentic-rd-skill/SKILL.md` metadata, `CITATION.cff`, the README version badge, and a dated `CHANGELOG.md` release section. `npm run validate` enforces this alignment and fails on a partial bump, so run it before continuing rather than checking each file by hand. 3. Review public host/version evidence separately from deterministic CLI evidence. Date every activation snapshot and keep untested hosts labeled as documentation-only. 4. Run: diff --git a/SECURITY.md b/SECURITY.md index 190cdbc..560b22e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -16,6 +16,8 @@ Use GitHub private vulnerability reporting or a security advisory for sensitive - Web pages, repositories, documents, logs, issues, and tool output are untrusted data and may contain prompt injection. - The portable skill does not pre-approve tools. Host permission, sandbox, workspace trust, and organizational policy remain authoritative. - The CLI refuses symlinked managed paths, unknown flags, incompatible state, out-of-order phases, mutation from globally invalid state, and finalization before approval. +- Initialization validates every managed destination before its first scaffold write, so a late path conflict does not leave partial workflow files behind. +- Repository validation does not follow symlinked documentation directories, and disposable validation fixtures copy only project-owned roots rather than ignored local workflow or secret files. - Workflow 1.1 verifies minimum artifact headings and revision fingerprints. These are integrity and traceability controls, not proof that Markdown claims are true or independently reviewed. - Revision fingerprints show whether upstream artifact bytes changed after a request; they are not signatures, provenance attestations, or protection against a malicious workspace owner. - The state CLI assumes one writer. Concurrent commands in the same workspace are outside the supported threat model until locking or generation checks are designed and tested. diff --git a/evals/security-review-v1.1.0.md b/evals/security-review-v1.1.0.md new file mode 100644 index 0000000..62be3c3 --- /dev/null +++ b/evals/security-review-v1.1.0.md @@ -0,0 +1,50 @@ +# v1.1.0 Security Review + +Snapshot: 2026-08-01 on Windows 11 with Node.js 24.18.0, npm 11.16.0, Python 3.12.10, and GitHub CLI 2.96.0. This is a repository-grounded maintainer review, not a third-party audit or a guarantee of semantic correctness. + +## Executive Summary + +No critical or high-severity vulnerability was identified in the dependency-free skill package or local workflow CLI. Four defense-in-depth findings were corrected before release: atomic initialization preflight, stricter workflow-state coherence, non-following repository document traversal, and narrower disposable test copies. No tracked credential pattern was found, and the installable skill remained free of runtime dependencies. + +## Scope And Method + +- Reviewed the installable package, CLI state transitions, managed-path operations, repository validators, benchmarks, host-smoke runner, tests, GitHub Actions, release metadata, and public documentation. +- Examined process execution, filesystem writes, symlink handling, path containment, temporary-directory cleanup, environment inheritance, release version synchronization, and package contents. +- Ran syntax checks, repository validation, deterministic tests, realistic benchmarks, the hash-pinned Agent Skills reference validator, GitHub skill publish dry-run, tracked-secret pattern checks, and Git integrity checks. +- Used GitHub API evidence for repository visibility, secret scanning, push protection, pull-request checks, CodeQL, releases, and branch rules. + +## Medium Findings + +### SEC-01: Late initialization conflicts could leave partial scaffolding + +Impact: a repair-style `init` could recreate an earlier missing artifact before discovering that a later managed destination had the wrong file type. + +Resolution: `skills/agentic-rd-skill/scripts/rd.mjs:386-400` now dry-runs every managed destination before the first write. A regression test proves a conflicting run-log path leaves the missing brief and existing state unchanged. + +### SEC-02: Repository Markdown traversal followed directory symlinks + +Impact: a local or malicious checkout could make validation leave the repository root, inspect unintended Markdown, or recurse through a symlink cycle. + +Resolution: `scripts/validate-repo.mjs:70-106` now uses non-following file checks and `Dirent` traversal. Symlinked directories are not visited; required symlinked files do not satisfy the regular-file contract. + +## Low Findings + +### SEC-03: Disposable repository fixtures copied broad local content + +The release-version regression fixture copied nearly the whole working directory except a short denylist. `tests/package.test.mjs:11-41` now allowlists project-owned roots, excluding ignored root-level workflow artifacts, environment files, logs, and unrelated local material. + +### SEC-04: Malformed package metadata could abort validation + +`scripts/validate-repo.mjs:181-205` now converts JSON parse failure into an explicit repository-validation finding. The validator continues reporting the fault without an uncaught exception. + +## Residual Risks And Limitations + +- The workflow state machine remains intentionally single-writer. Concurrent state-changing CLI processes are unsupported until a locking or generation protocol is designed and tested. +- Structural heading, coverage, fingerprint, and gate checks do not establish that artifact claims are true, unbiased, or independently reviewed. +- Windows could not create symlinks in the local test environment (`EPERM`); Linux and macOS CI remain the authoritative regression coverage for symlink paths. +- Host discovery was inconclusive because GitHub Copilot CLI and OpenCode were not installed. No model-backed host smoke was run, and the 2026-07-18 activation matrix remains historical evidence. +- Child host processes inherit the invoking environment so authenticated CLIs can operate. Model-backed smokes remain opt-in and require explicit credential and budget authorization. + +## Verification Status + +The complete local gate, official Agent Skills validator, and GitHub publish dry-run passed after these changes. Remote CI and CodeQL evidence must be rechecked on the final commit; release, tag, and installation verification remain separate publication gates. diff --git a/package.json b/package.json index 6c6562e..06ee31f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agentic-rd-skill", - "version": "1.0.0", + "version": "1.1.0", "description": "A portable Agent Skill for evidence-aware, stateful multi-agent research and development workflows.", "private": true, "repository": { diff --git a/scripts/smoke-hosts.mjs b/scripts/smoke-hosts.mjs index d1fd0aa..f210e56 100644 --- a/scripts/smoke-hosts.mjs +++ b/scripts/smoke-hosts.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node -import { cpSync, mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import process from 'node:process'; @@ -11,6 +11,21 @@ const scriptPath = fileURLToPath(import.meta.url); const repositoryRoot = path.resolve(path.dirname(scriptPath), '..'); const skillSource = path.join(repositoryRoot, 'skills', 'agentic-rd-skill'); const runModelSmokes = process.argv.includes('--model-smokes'); + +// Read the shipped skill version instead of hardcoding it, so a release bump cannot +// leave the activation smoke asserting a version the package no longer declares. +function skillVersion() { + const frontmatter = readFileSync(path.join(skillSource, 'SKILL.md'), 'utf8') + .replaceAll('\r\n', '\n') + .match(/^---\n([\s\S]*?)\n---\n/); + const version = frontmatter?.[1].match(/^\s{2}version:\s*"(\d+\.\d+\.\d+)"\s*$/m)?.[1]; + if (!version) { + console.error('Cannot read metadata.version from skills/agentic-rd-skill/SKILL.md'); + process.exit(2); + } + return version; +} + const hostOptionIndex = process.argv.indexOf('--host'); const selectedHost = hostOptionIndex === -1 ? null : process.argv[hostOptionIndex + 1]; const validHosts = new Set(['codex', 'claude', 'copilot', 'opencode']); @@ -99,9 +114,11 @@ try { } if (runModelSmokes) { - const expected = /DISCOVERED\s+agentic-rd-skill\s+1\.0\.0/i; + const version = skillVersion(); + const expected = new RegExp(`DISCOVERED\\s+agentic-rd-skill\\s+${version.replaceAll('.', '\\.')}`, 'i'); const genericPrompt = - 'Activate the workspace skill named agentic-rd-skill. Do not create or edit files, run scripts, or research the web. Read only the activated skill metadata and return exactly: DISCOVERED agentic-rd-skill 1.0.0'; + 'Activate the workspace skill named agentic-rd-skill. Do not create or edit files, run scripts, or research the web. ' + + `Read only the activated skill metadata and return exactly: DISCOVERED agentic-rd-skill ${version}`; if (includesHost('codex') && commandAvailable('codex')) { record( diff --git a/scripts/validate-repo.mjs b/scripts/validate-repo.mjs index cd80215..b68180e 100644 --- a/scripts/validate-repo.mjs +++ b/scripts/validate-repo.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node -import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; +import { existsSync, lstatSync, readFileSync, readdirSync } from 'node:fs'; import path from 'node:path'; import process from 'node:process'; import { fileURLToPath } from 'node:url'; @@ -53,6 +53,7 @@ const REQUIRED_ROOT_FILES = [ '.github/workflows/ci.yml', 'evals/manifest.json', 'evals/dogfood-v1.0.0.md', + 'evals/security-review-v1.1.0.md', 'evals/usability-review.md', 'requirements-validation.txt', 'scripts/benchmark.mjs', @@ -66,6 +67,14 @@ function readText(filePath) { return readFileSync(filePath, 'utf8').replaceAll('\r\n', '\n'); } +function isRegularFile(filePath) { + try { + return lstatSync(filePath).isFile(); + } catch { + return false; + } +} + function parseScalarFrontmatter(text) { const match = text.match(/^---\n([\s\S]*?)\n---\n/); if (!match) return null; @@ -80,39 +89,50 @@ function parseScalarFrontmatter(text) { function markdownFiles(root) { const files = []; - const ignoredDirectories = new Set(['.git', 'node_modules', 'work']); + // `.venv` is the documented local validator environment and `work/`/`project-brief.md` + // are ignored local dogfood artifacts; neither is tracked product source. + const ignoredDirectories = new Set(['.git', '.venv', 'coverage', 'node_modules', 'work']); + const ignoredFiles = new Set([path.join(root, 'project-brief.md')]); const visit = (directory) => { - for (const [name, entry] of Object.entries(importDirectory(directory))) { - if (entry.directory && !ignoredDirectories.has(name)) visit(entry.path); - else if (entry.path.toLowerCase().endsWith('.md')) files.push(entry.path); + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + if (!ignoredDirectories.has(entry.name)) visit(entryPath); + } else if (entry.isFile() && entryPath.toLowerCase().endsWith('.md') && !ignoredFiles.has(entryPath)) { + files.push(entryPath); + } } }; visit(root); return files; } -function importDirectory(directory) { - const entries = {}; - for (const name of readdirSync(directory)) { - const entryPath = path.join(directory, name); - const stats = statSync(entryPath); - entries[name] = { path: entryPath, directory: stats.isDirectory() }; +function releaseVersion(root) { + const packagePath = path.join(root, 'package.json'); + if (!isRegularFile(packagePath)) return null; + try { + const version = JSON.parse(readText(packagePath)).version; + return typeof version === 'string' && /^\d+\.\d+\.\d+$/.test(version) ? version : null; + } catch { + return null; } - return entries; } export function validateRepository(root = repositoryRoot) { const failures = []; const skillRoot = path.join(root, 'skills', 'agentic-rd-skill'); const fail = (message) => failures.push(message); + // package.json is the single source of truth for the release version; every other + // surface listed in RELEASING.md is checked against it so a partial bump fails locally. + const expectedVersion = releaseVersion(root); for (const relativePath of REQUIRED_ROOT_FILES) { const fullPath = path.join(root, relativePath); - if (!existsSync(fullPath) || !statSync(fullPath).isFile()) fail(`Missing repository file: ${relativePath}`); + if (!isRegularFile(fullPath)) fail(`Missing repository file: ${relativePath}`); } for (const relativePath of REQUIRED_SKILL_FILES) { const fullPath = path.join(skillRoot, relativePath); - if (!existsSync(fullPath) || !statSync(fullPath).isFile()) fail(`Missing skill file: ${relativePath}`); + if (!isRegularFile(fullPath)) fail(`Missing skill file: ${relativePath}`); } for (const legacyPath of [ @@ -124,7 +144,7 @@ export function validateRepository(root = repositoryRoot) { } const skillPath = path.join(skillRoot, 'SKILL.md'); - if (existsSync(skillPath)) { + if (isRegularFile(skillPath)) { const skillText = readText(skillPath); const frontmatter = parseScalarFrontmatter(skillText); if (!frontmatter) { @@ -136,7 +156,12 @@ export function validateRepository(root = repositoryRoot) { if (!values.compatibility || values.compatibility.length > 500) fail('SKILL.md compatibility must be 1-500 characters'); if (values.license !== 'MIT') fail('SKILL.md license must be MIT'); if (/^allowed-tools\s*:/m.test(raw)) fail('Portable SKILL.md must not pre-approve host-specific tools'); - if (!/^\s{2}version:\s*"1\.0\.0"\s*$/m.test(raw)) fail('SKILL.md metadata version must be 1.0.0'); + const skillVersion = raw.match(/^\s{2}version:\s*"(\d+\.\d+\.\d+)"\s*$/m)?.[1] ?? null; + if (skillVersion === null) { + fail('SKILL.md metadata version must be a quoted x.y.z string'); + } else if (expectedVersion && skillVersion !== expectedVersion) { + fail(`SKILL.md metadata version ${skillVersion} must match package.json version ${expectedVersion}`); + } } if (skillText.split('\n').length > 500) fail('SKILL.md must remain under 500 lines'); for (const requiredReference of [ @@ -152,27 +177,34 @@ export function validateRepository(root = repositoryRoot) { } const packagePath = path.join(root, 'package.json'); - if (existsSync(packagePath)) { - const packageJson = JSON.parse(readText(packagePath)); - if (packageJson.version !== '1.0.0') fail('package.json version must be 1.0.0'); - if (packageJson.engines?.node !== '>=20') fail('package.json must require Node.js >=20'); - if (packageJson.scripts?.test !== 'node --test') fail('package.json test script must run node --test'); - if (packageJson.scripts?.check !== 'npm run validate && npm test && npm run benchmark') { - fail('package.json check script must run validation, tests, and benchmark'); - } - if (packageJson.scripts?.benchmark !== 'node scripts/benchmark.mjs') { - fail('package.json benchmark script must run scripts/benchmark.mjs'); + if (isRegularFile(packagePath)) { + let packageJson; + try { + packageJson = JSON.parse(readText(packagePath)); + } catch (error) { + fail(`Invalid package.json: ${error.message}`); } - const keywords = packageJson.keywords ?? []; - if (new Set(keywords).size !== keywords.length) fail('package.json keywords must be unique'); - for (const keyword of ['agent-skills', 'ai-research', 'research-agent', 'codex', 'claude-code']) { - if (!keywords.includes(keyword)) fail(`package.json keywords must include ${keyword}`); + if (packageJson) { + if (expectedVersion === null) fail('package.json version must be a semantic x.y.z string'); + if (packageJson.engines?.node !== '>=20') fail('package.json must require Node.js >=20'); + if (packageJson.scripts?.test !== 'node --test') fail('package.json test script must run node --test'); + if (packageJson.scripts?.check !== 'npm run validate && npm test && npm run benchmark') { + fail('package.json check script must run validation, tests, and benchmark'); + } + if (packageJson.scripts?.benchmark !== 'node scripts/benchmark.mjs') { + fail('package.json benchmark script must run scripts/benchmark.mjs'); + } + const keywords = packageJson.keywords ?? []; + if (new Set(keywords).size !== keywords.length) fail('package.json keywords must be unique'); + for (const keyword of ['agent-skills', 'ai-research', 'research-agent', 'codex', 'claude-code']) { + if (!keywords.includes(keyword)) fail(`package.json keywords must include ${keyword}`); + } } } for (const workflowPath of ['.github/workflows/ci.yml']) { const fullPath = path.join(root, workflowPath); - if (!existsSync(fullPath)) continue; + if (!isRegularFile(fullPath)) continue; const workflow = readText(fullPath); for (const match of workflow.matchAll(/^\s*-?\s*uses:\s*[^@\s]+@([^\s#]+)/gm)) { if (!/^[0-9a-f]{40}$/.test(match[1])) { @@ -182,15 +214,36 @@ export function validateRepository(root = repositoryRoot) { } const citationPath = path.join(root, 'CITATION.cff'); - if (existsSync(citationPath) && !/^version:\s*"1\.0\.0"\s*$/m.test(readText(citationPath))) { - fail('CITATION.cff version must be 1.0.0'); + if (isRegularFile(citationPath) && expectedVersion) { + const citationVersion = readText(citationPath).match(/^version:\s*"(\d+\.\d+\.\d+)"\s*$/m)?.[1] ?? null; + if (citationVersion !== expectedVersion) { + fail(`CITATION.cff version ${citationVersion ?? ''} must match package.json version ${expectedVersion}`); + } + } + + const readmePath = path.join(root, 'README.md'); + if (isRegularFile(readmePath) && expectedVersion) { + const badgeVersion = readText(readmePath) + .match(/!\[Version\]\(https:\/\/img\.shields\.io\/badge\/version-(\d+\.\d+\.\d+)-/)?.[1] ?? null; + if (badgeVersion !== expectedVersion) { + fail(`README version badge ${badgeVersion ?? ''} must match package.json version ${expectedVersion}`); + } + } + + const changelogPath = path.join(root, 'CHANGELOG.md'); + if (isRegularFile(changelogPath) && expectedVersion) { + const escapedVersion = expectedVersion.replaceAll('.', '\\.'); + const released = new RegExp(`^## \\[?${escapedVersion}\\]? - \\d{4}-\\d{2}-\\d{2}[ \\t]*$`, 'm'); + if (!released.test(readText(changelogPath))) { + fail(`CHANGELOG.md must contain a dated "## [${expectedVersion}] - YYYY-MM-DD" section`); + } } const rootLicensePath = path.join(root, 'LICENSE'); const skillLicensePath = path.join(skillRoot, 'LICENSE'); if ( - existsSync(rootLicensePath) - && existsSync(skillLicensePath) + isRegularFile(rootLicensePath) + && isRegularFile(skillLicensePath) && readText(rootLicensePath) !== readText(skillLicensePath) ) { fail('Bundled skill LICENSE must match the repository LICENSE'); @@ -210,7 +263,7 @@ export function validateRepository(root = repositoryRoot) { } const evalPath = path.join(root, 'evals', 'manifest.json'); - if (existsSync(evalPath)) { + if (isRegularFile(evalPath)) { try { const manifest = JSON.parse(readText(evalPath)); if (manifest.schemaVersion !== 1) fail('eval manifest schemaVersion must be 1'); diff --git a/skills/agentic-rd-skill/SKILL.md b/skills/agentic-rd-skill/SKILL.md index adefc99..eb30235 100644 --- a/skills/agentic-rd-skill/SKILL.md +++ b/skills/agentic-rd-skill/SKILL.md @@ -5,7 +5,7 @@ license: MIT compatibility: Requires filesystem read/write access. The optional workflow CLI requires Node.js 20+. Web access and native subagents are optional; the workflow has offline and single-agent fallbacks. metadata: author: Michael Gasperini - version: "1.0.0" + version: "1.1.0" --- # Agentic R&D Workflow diff --git a/skills/agentic-rd-skill/scripts/rd.mjs b/skills/agentic-rd-skill/scripts/rd.mjs index 0f635de..8b17013 100644 --- a/skills/agentic-rd-skill/scripts/rd.mjs +++ b/skills/agentic-rd-skill/scripts/rd.mjs @@ -383,15 +383,22 @@ function commandInit(tokens) { stateCreated = true; } - const results = [ - ['project-brief.md', copyTemplateIfMissing(workspace, 'project-brief.md', 'project-brief.md', dryRun)], - ['work/', ensureManagedDirectory(workspace, 'work', dryRun)], - ['work/00-run-log.md', copyTemplateIfMissing(workspace, 'run-log.md', 'work/00-run-log.md', dryRun)], - ['work/01-evidence/', ensureManagedDirectory(workspace, 'work/01-evidence', dryRun)], - ['work/03-execution/', ensureManagedDirectory(workspace, 'work/03-execution', dryRun)], - ['work/04-results/', ensureManagedDirectory(workspace, 'work/04-results', dryRun)] + const plannedResults = [ + ['project-brief.md', (preview) => copyTemplateIfMissing(workspace, 'project-brief.md', 'project-brief.md', preview)], + ['work/', (preview) => ensureManagedDirectory(workspace, 'work', preview)], + ['work/00-run-log.md', (preview) => copyTemplateIfMissing(workspace, 'run-log.md', 'work/00-run-log.md', preview)], + ['work/01-evidence/', (preview) => ensureManagedDirectory(workspace, 'work/01-evidence', preview)], + ['work/03-execution/', (preview) => ensureManagedDirectory(workspace, 'work/03-execution', preview)], + ['work/04-results/', (preview) => ensureManagedDirectory(workspace, 'work/04-results', preview)] ]; + // Validate every destination before the first write. A conflict in a later path + // must not leave a newly created brief or partial directory tree behind. + if (!dryRun) { + for (const [, apply] of plannedResults) apply(true); + } + const results = plannedResults.map(([item, apply]) => [item, apply(dryRun)]); + if (stateCreated && !dryRun) writeManagedJson(workspace, 'work/run-state.json', state); results.splice(2, 0, ['work/run-state.json', stateCreated]); @@ -828,22 +835,46 @@ function validateState(workspace, state) { if (state.finalStale && state.phases.stageGate === 'approved') { failures.push('stale final output requires a recorded review before finalization'); } - if (state.phases.stageGate === 'approved') { - const dimensions = state.stageGate.dimensions; - const invalidDimensions = !Array.isArray(dimensions) - || dimensions.length !== 5 - || dimensions.some((item) => !Number.isInteger(item) || item < 1 || item > 2) - || dimensions.reduce((total, item) => total + item, 0) !== state.stageGate.score; - if (state.stageGate.score < 8 || state.stageGate.blockers !== 0 || invalidDimensions) { - failures.push('Approved gate has an invalid score or blockers'); - } - } const expectedGateDecision = ['approved', 'needs_revision', 'blocked'].includes(state.phases.stageGate) ? state.phases.stageGate : null; if (state.stageGate.decision !== expectedGateDecision) { failures.push('Stage-gate decision metadata does not match its phase status'); } + const dimensions = state.stageGate.dimensions; + const score = state.stageGate.score; + const dimensionsInvalid = dimensions !== null && ( + dimensions.length !== 5 + || dimensions.some((item) => !Number.isInteger(item) || item < 0 || item > 2) + ); + if (score !== null && (score < 0 || score > 10)) failures.push('Stage-gate score must be between 0 and 10'); + if (dimensionsInvalid) failures.push('Stage-gate dimensions must contain five scores from 0 to 2'); + if ( + score !== null + && dimensions !== null + && !dimensionsInvalid + && dimensions.reduce((total, item) => total + item, 0) !== score + ) { + failures.push('Stage-gate dimensions must sum to its score'); + } + if (state.stageGate.blockers < 0) failures.push('Stage-gate blockers must be a non-negative integer'); + if ( + expectedGateDecision === null + && (score !== null || dimensions !== null || state.stageGate.blockers !== 0) + ) { + failures.push('Inactive stage gate cannot retain decision metadata'); + } + if (state.phases.stageGate === 'approved') { + const invalidApprovalDimensions = dimensionsInvalid + || dimensions === null + || dimensions.some((item) => item === 0); + if (score === null || score < 8 || state.stageGate.blockers !== 0 || invalidApprovalDimensions) { + failures.push('Approved gate has an invalid score or blockers'); + } + } + if (state.phases.stageGate === 'blocked' && state.stageGate.blockers < 1) { + failures.push('Blocked stage gate must record at least one blocker'); + } if (state.revisionRounds > state.budgets.maxRevisionRounds) { failures.push('Revision round limit exceeded'); } @@ -906,6 +937,10 @@ function validateState(workspace, state) { if (PHASES.includes(state.currentPhase) && state.phases[state.currentPhase] === 'pending') { failures.push('currentPhase cannot point to a pending phase'); } + const expectedCurrentPhase = [...PHASES].reverse().find((phase) => state.phases[phase] !== 'pending'); + if (expectedCurrentPhase && state.currentPhase !== expectedCurrentPhase) { + failures.push(`currentPhase must match the latest started phase (${expectedCurrentPhase})`); + } return [...new Set(failures)]; } diff --git a/tests/package.test.mjs b/tests/package.test.mjs index 223af63..d877885 100644 --- a/tests/package.test.mjs +++ b/tests/package.test.mjs @@ -1,12 +1,115 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import path from 'node:path'; +import { cpSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; import { validateRepository } from '../scripts/validate-repo.mjs'; const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const INCLUDED_ROOTS = new Set([ + '.editorconfig', + '.gitattributes', + '.github', + '.gitignore', + 'AGENTS.md', + 'assets', + 'CHANGELOG.md', + 'CITATION.cff', + 'CODE_OF_CONDUCT.md', + 'CONTRIBUTING.md', + 'evals', + 'LICENSE', + 'package.json', + 'README.md', + 'RELEASING.md', + 'requirements-validation.txt', + 'scripts', + 'SECURITY.md', + 'skills', + 'tests' +]); + +function copyRepository() { + const destination = mkdtempSync(path.join(tmpdir(), 'agentic-rd-repo-')); + cpSync(repositoryRoot, destination, { + recursive: true, + filter: (source) => { + const relative = path.relative(repositoryRoot, source); + return relative === '' || INCLUDED_ROOTS.has(relative.split(path.sep)[0]); + } + }); + return destination; +} test('repository package satisfies structural invariants', () => { assert.deepEqual(validateRepository(repositoryRoot), []); }); + +test('release version drift is reported on every synchronized surface', (t) => { + const root = copyRepository(); + t.after(() => rmSync(root, { recursive: true, force: true })); + + // The copy must be clean before drift is introduced, otherwise the assertions below + // could pass for an unrelated reason. + assert.deepEqual(validateRepository(root), []); + + const packagePath = path.join(root, 'package.json'); + const packageJson = JSON.parse(readFileSync(packagePath, 'utf8')); + packageJson.version = '9.9.9'; + writeFileSync(packagePath, `${JSON.stringify(packageJson, null, 2)}\n`); + + const failures = validateRepository(root); + for (const surface of ['SKILL.md metadata version', 'CITATION.cff version', 'README version badge']) { + assert.ok( + failures.some((failure) => failure.startsWith(surface) && failure.includes('9.9.9')), + `expected a drift failure for ${surface}, received: ${failures.join(' | ')}` + ); + } + assert.ok( + failures.some((failure) => failure.includes('CHANGELOG.md must contain a dated')), + `expected a missing changelog release failure, received: ${failures.join(' | ')}` + ); +}); + +test('a non-semantic package version is rejected', (t) => { + const root = copyRepository(); + t.after(() => rmSync(root, { recursive: true, force: true })); + + const packagePath = path.join(root, 'package.json'); + const packageJson = JSON.parse(readFileSync(packagePath, 'utf8')); + packageJson.version = '1.1'; + writeFileSync(packagePath, `${JSON.stringify(packageJson, null, 2)}\n`); + + assert.ok(validateRepository(root).includes('package.json version must be a semantic x.y.z string')); +}); + +test('malformed package metadata is reported without crashing validation', (t) => { + const root = copyRepository(); + t.after(() => rmSync(root, { recursive: true, force: true })); + + writeFileSync(path.join(root, 'package.json'), '{ invalid json\n'); + + assert.ok(validateRepository(root).some((failure) => failure.startsWith('Invalid package.json:'))); +}); + +test('repository Markdown validation does not follow symlinked directories', (t) => { + const root = copyRepository(); + const outside = mkdtempSync(path.join(tmpdir(), 'agentic-rd-docs-outside-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + t.after(() => rmSync(outside, { recursive: true, force: true })); + writeFileSync(path.join(outside, 'external.md'), '[broken external link](missing.md)\n'); + + try { + symlinkSync(outside, path.join(root, 'linked-docs'), 'dir'); + } catch (error) { + if (error.code === 'EPERM' || error.code === 'EACCES' || error.code === 'UNKNOWN') { + t.skip(`symlink creation unavailable: ${error.code}`); + return; + } + throw error; + } + + assert.deepEqual(validateRepository(root), []); +}); diff --git a/tests/rd-cli.test.mjs b/tests/rd-cli.test.mjs index 1a87d32..371ee1a 100644 --- a/tests/rd-cli.test.mjs +++ b/tests/rd-cli.test.mjs @@ -138,6 +138,23 @@ test('init rejects file-path conflicts before creating workflow state', (t) => { assert.equal(existsSync(path.join(workspace, 'work', 'run-state.json')), false); }); +test('init preflights every managed path before writing any scaffolding', (t) => { + const workspace = workspaceFor(t, 'late-conflict-'); + assert.equal(runCli(['init', workspace]).status, 0); + const statePath = path.join(workspace, 'work', 'run-state.json'); + const stateBefore = readFileSync(statePath, 'utf8'); + rmSync(path.join(workspace, 'project-brief.md')); + rmSync(path.join(workspace, 'work', '00-run-log.md')); + mkdirSync(path.join(workspace, 'work', '00-run-log.md'), { recursive: true }); + + const result = runCli(['init', workspace]); + + assert.equal(result.status, 4); + assert.match(result.stderr, /Expected a regular file/); + assert.equal(existsSync(path.join(workspace, 'project-brief.md')), false); + assert.equal(readFileSync(statePath, 'utf8'), stateBefore); +}); + test('dry-run writes nothing', (t) => { const workspace = workspaceFor(t, 'dry-'); const result = runCli(['init', workspace, '--dry-run']); @@ -495,6 +512,24 @@ test('malformed revision history is reported as invalid instead of crashing stat assert.match(validation.stdout, /Revision history contains invalid metadata/); }); +test('state validation rejects contradictory current-phase and stage-gate metadata', (t) => { + const workspace = initializeFilledBrief(t); + assert.equal(runCli(['advance', workspace, '--phase', 'evidence', '--status', 'in_progress']).status, 0); + const statePath = path.join(workspace, 'work', 'run-state.json'); + const state = JSON.parse(readFileSync(statePath, 'utf8')); + state.currentPhase = 'setup'; + state.stageGate.score = 99; + writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\n`, 'utf8'); + + const status = runCli(['status', workspace, '--json']); + + assert.equal(status.status, 0, status.stderr); + const failures = JSON.parse(status.stdout).validation.failures.join('\n'); + assert.match(failures, /currentPhase must match the latest started phase/); + assert.match(failures, /Stage-gate score must be between 0 and 10/); + assert.match(failures, /Inactive stage gate cannot retain decision metadata/); +}); + test('unsafe integers and stage-gate-only options are rejected as usage errors', (t) => { const workspace = initializeFilledBrief(t); const unsafeInteger = runCli([