diff --git a/.development-kit/workspace.json b/.development-kit/workspace.json new file mode 100644 index 00000000..fcedf328 --- /dev/null +++ b/.development-kit/workspace.json @@ -0,0 +1,24 @@ +{ + "schemaVersion": 1, + "targets": { + "dkf-framework": { + "path": ".", + "kind": "development-framework", + "dependencies": [], + "commands": { + "test": "npm run release:validate", + "test:orchestration": "npm run orchestration-core:validate" + }, + "capabilities": [ + "dkf-control-plane", + "node-runtime", + "release-validation" + ], + "verification": [ + "test", + "test:orchestration" + ], + "adapter": null + } + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ff48e76..50c49c93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,8 @@ on: permissions: contents: read + pull-requests: read + issues: read jobs: validate: @@ -20,10 +22,10 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.0 - name: Set up Node.js - uses: actions/setup-node@v7 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 22 @@ -83,6 +85,31 @@ jobs: npm run token-efficiency:test npm run token:audit:check + - name: Run Scale/Context Phase 0-1 baseline + run: node scripts/phase01-scale-context-baseline.mjs + + - name: Validate Scale/Context v0.12 Phase 2 PLAN + run: node scripts/phase02-scale-context-plan-validate.mjs + + - name: Validate v0.12 Cost Observatory increment + run: npm run cost-observatory:test + + - name: Validate v0.12 Workspace Target Engine increment + run: npm run workspace-targets:test + + - name: Validate v0.12 Execution Capsules & Context Cache increment + run: npm run execution-capsule:test + + - name: Validate v0.12 State Engine V2 increment + run: npm run state-engine-v2:test + + - name: Validate cross-owner legacy restore metadata + if: ${{ matrix.os == 'ubuntu-latest' && github.event.pull_request.head.repo.full_name == github.repository }} + run: sudo "$(command -v node)" --test --test-name-pattern='T04-H30' scripts/state-engine-v2-migration.test.mjs + + - name: Benchmark v0.12 State Engine V2 append endurance + run: npm run state-engine-v2:benchmark + - name: Validate Design Authority governance run: npm run design-authority:validate @@ -91,3 +118,39 @@ jobs: - name: Validate exact release command run: npm run release:validate + + t04_acceptance: + name: T04 independently reviewed acceptance + if: ${{ github.event_name == 'pull_request' && github.head_ref == 'fix/v0.12-t04-state-engine-hardening' }} + needs: validate + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: read + pull-requests: read + steps: + - name: Check out validated source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.0 + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + + - name: Materialize source-bound T04 Development Contract + run: node scripts/phase06-state-engine-v2-contract.mjs + + - name: Verify independent reviews and compute deterministic T04 acceptance + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: node scripts/t04-hardening-acceptance-gate.mjs + + - name: Retain independent risk-3 acceptance evidence + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.0 + with: + name: t04-hardening-acceptance + path: .development-kit/runs/INC-DKF120-T04/T04-HARDENING-REVIEWED-20261009/ + retention-days: 30 + include-hidden-files: true + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 1737882c..7d942060 100644 --- a/.gitignore +++ b/.gitignore @@ -30,5 +30,7 @@ desktop.ini .development-kit/runs/ .development-kit/decisions/ .development-kit/secrets/ +.development-kit/telemetry/ +.development-kit/state/ freebuff-chat-*.md diff --git a/docs/03-reference/scripts/README.md b/docs/03-reference/scripts/README.md index 9f1eff0d..e0ee1127 100644 --- a/docs/03-reference/scripts/README.md +++ b/docs/03-reference/scripts/README.md @@ -22,3 +22,13 @@ Development Kit ships **9 primary non-test Node.js scripts** in `scripts/`, expo - `install-antigravity.mjs` — exits 1 when no target is found and none requested; 0 otherwise See [validation-architecture.md](../../04-architecture/validation-architecture.md) for how the scripts fit together. + +## Additional v0.12 Verification + +- **State Engine V2 endurance:** `npm run state-engine-v2:benchmark` — 300-transition disposable fixture on the Windows/Ubuntu CI matrix. See [state-engine-v2-benchmark.md](state-engine-v2-benchmark.md). The resulting timings are observations, not release performance acceptance. + +- **T04 risk-3 runtime acceptance:** `node scripts/t04-hardening-acceptance-gate.mjs` — GitHub Actions-only, after exact release gates; validates immutable reviewed source, creates persisted DKF verification/security/acceptance records and uploads CI evidence. See [t04-hardening-acceptance-gate.md](t04-hardening-acceptance-gate.md). + +- **T04 reviewer-thread safety:** `node --test scripts/t04-review-thread-blockers.test.mjs` — proves every unresolved code-review thread, including P2 and unclassified findings, blocks independent acceptance. See [t04-review-thread-blockers.md](t04-review-thread-blockers.md). + +- **Windows restore rollback:** `scripts/windows-durable-rename.ps1` — native Win32 write-through directory move used for recovery, with explicit platform guarantees and review limitations. See [windows-durable-rename.md](windows-durable-rename.md). diff --git a/docs/03-reference/scripts/phase01-scale-context-baseline.md b/docs/03-reference/scripts/phase01-scale-context-baseline.md new file mode 100644 index 00000000..b8c3a5c5 --- /dev/null +++ b/docs/03-reference/scripts/phase01-scale-context-baseline.md @@ -0,0 +1,51 @@ +# phase01-scale-context-baseline.mjs + +## Purpose + +`scripts/phase01-scale-context-baseline.mjs` is an analysis-only baseline runner for the Development Kit Scale, Context & Iteration initiative. + +It measures the current framework before implementation changes are made. The script does not modify DKF production runtime behaviour, change package versions, publish releases, or mutate repository state outside temporary test fixtures. + +## What it measures + +The runner records: + +- repository, test, and canonical instruction-file inventory; +- the existing deterministic token/context audit; +- representative single-package context packaging; +- representative multi-package context packaging; +- representative calculation-heavy context packaging; +- Autopilot state-file growth across 26 immutable revisions; +- current observability gaps for provider token usage, repository reads/scans, agent invocations, and host tool calls. + +## Representative fixtures + +The fixtures are synthetic and deterministic. They exercise the existing DKF runtime APIs rather than claiming to reproduce provider billing or a specific production repository exactly. + +The multi-package fixture contains web, mobile, shared, calculation, and database-style paths. The calculation-heavy fixture establishes a baseline before domain/numerical gate profiles exist. + +## Usage + +Print JSON to standard output: + +```bash +node scripts/phase01-scale-context-baseline.mjs +``` + +Optionally persist the JSON output: + +```bash +node scripts/phase01-scale-context-baseline.mjs --out=path/to/baseline.json +``` + +## Measurement rules + +The runner reuses `scripts/token-audit.mjs` and the existing `runtime/orchestration/token-efficiency.mjs` estimator. + +Estimated token values therefore remain deterministic `ceil(characters / 4)` approximations and must not be represented as provider billing-token counts. + +Metrics that the current runtime cannot observe are emitted as unavailable with a reason instead of being estimated or fabricated. + +## Scope + +This script belongs to Phase 0/1 evidence gathering only. Later Cost Observatory work may extend or replace parts of this runner after the baseline has been preserved. diff --git a/docs/03-reference/scripts/phase02-scale-context-plan-validate.md b/docs/03-reference/scripts/phase02-scale-context-plan-validate.md new file mode 100644 index 00000000..55e4dda9 --- /dev/null +++ b/docs/03-reference/scripts/phase02-scale-context-plan-validate.md @@ -0,0 +1,42 @@ +# phase02-scale-context-plan-validate.mjs + +## Purpose + +`scripts/phase02-scale-context-plan-validate.mjs` validates the machine-readable Development Kit v0.12.0 Scale, Context & Iteration PLAN using the existing deterministic `runtime/orchestration/plan-validator.mjs`. + +It is a Phase 2 planning validator. It does not implement v0.12 runtime functionality. + +## Authoritative PLAN model + +```text +docs/04-architecture/dkf-scale-context-iteration-v0.12-plan-model.json +``` + +## Validation performed + +The script checks: + +- declared task count; +- dependency references; +- dependency cycles; +- declared dependency graph versus computed graph; +- required logical resource ownership; +- duplicate resource ownership; +- acceptance-criterion coverage; +- configuration dependency validity. + +The acceptance-criterion authority comes from the reconciled v0.12 specification. + +## Usage + +```bash +node scripts/phase02-scale-context-plan-validate.mjs +``` + +The command prints the complete deterministic validation report as JSON and exits non-zero when the PLAN is invalid. + +## Scope + +This validator intentionally reuses the existing DKF PLAN validation engine instead of creating a second planning model. + +A green result proves structural PLAN consistency. It does not replace specification review, architecture review, implementation verification, or Product Owner release approval. diff --git a/docs/03-reference/scripts/phase03-cost-observatory-contract.md b/docs/03-reference/scripts/phase03-cost-observatory-contract.md new file mode 100644 index 00000000..d5802cf5 --- /dev/null +++ b/docs/03-reference/scripts/phase03-cost-observatory-contract.md @@ -0,0 +1,47 @@ +# phase03-cost-observatory-contract.mjs + +## Purpose + +`scripts/phase03-cost-observatory-contract.mjs` creates or reuses the Development Contract for **DKF120-T01 — Cost Observatory**. + +The contract is normal DKF runtime state and is persisted under: + +```text +.development-kit/contracts/INC-DKF120-T01/ +``` + +That directory remains git-ignored. The script itself is committed so contract construction is reproducible from the authoritative v0.12 sources. + +## Authoritative sources + +The contract binds: + +- the v0.12 Scale, Context & Iteration specification; +- the v0.12 technical design; +- the validated v0.12 implementation PLAN; +- the Phase 0/1 baseline evidence. + +Whole-file fingerprints remain authoritative even when the contract requests selected sections for context delivery. + +## Acceptance criteria + +The T01 contract owns: + +- `DKF-120-AC-001` +- `DKF-120-AC-002` +- `DKF-120-AC-003` +- `DKF-120-AC-033` + +## Usage + +```bash +node scripts/phase03-cost-observatory-contract.mjs +``` + +The script validates the resulting contract and immediately verifies that it is not stale. + +If an existing contract has become stale, normal DKF stale-contract handling applies; the script does not silently overwrite it. + +## Boundary + +This script authorises only T01 Cost Observatory work. It does not authorise T02 or later v0.12 increments. diff --git a/docs/03-reference/scripts/phase04-workspace-target-engine-contract.md b/docs/03-reference/scripts/phase04-workspace-target-engine-contract.md new file mode 100644 index 00000000..e5540058 --- /dev/null +++ b/docs/03-reference/scripts/phase04-workspace-target-engine-contract.md @@ -0,0 +1,60 @@ +# phase04-workspace-target-engine-contract.mjs + +## Purpose + +`scripts/phase04-workspace-target-engine-contract.mjs` creates or reuses the Development Contract for **DKF120-T02 — Workspace Target Engine**. + +The contract is persisted through the normal DKF runtime contract store under: + +```text +.development-kit/contracts/INC-DKF120-T02/ +``` + +That runtime directory remains git-ignored. + +## Bootstrap note + +T02 introduces Workspace Target Engine support itself. + +Therefore the T02 Development Contract is created **without** a workspace target binding. This is intentional bootstrap behaviour: target binding cannot be mandatory for the increment that first creates the target engine. + +After T02 acceptance, target-aware Development Contracts may bind: + +- primary target; +- affected targets; +- verification targets; +- the exact workspace-registry fingerprint. + +A later change to that registry then makes the target-aware contract stale. + +## Authoritative sources + +The contract binds: + +- the v0.12 Scale, Context & Iteration specification; +- the v0.12 technical design; +- the validated v0.12 implementation PLAN; +- accepted T01 evidence; +- the Phase 0/1 multi-package baseline. + +## Acceptance criteria + +The T02 contract owns: + +- `DKF-120-AC-004` +- `DKF-120-AC-005` +- `DKF-120-AC-006` +- `DKF-120-AC-007` +- `DKF-120-AC-039` + +## Usage + +```bash +node scripts/phase04-workspace-target-engine-contract.mjs +``` + +The script validates the contract, confirms its derived gates, and verifies that it is not stale. + +## Boundary + +This script authorises only T02 Workspace Target Engine work. It does not authorise T03 or later v0.12 increments. diff --git a/docs/03-reference/scripts/phase05-execution-capsule-contract.md b/docs/03-reference/scripts/phase05-execution-capsule-contract.md new file mode 100644 index 00000000..949286df --- /dev/null +++ b/docs/03-reference/scripts/phase05-execution-capsule-contract.md @@ -0,0 +1,77 @@ +# phase05-execution-capsule-contract.mjs + +## Purpose + +`scripts/phase05-execution-capsule-contract.mjs` creates or reuses the Development Contract for **DKF120-T03 — Execution Capsules & Context Cache**. + +The contract is persisted through the normal DKF contract store under: + +```text +.development-kit/contracts/INC-DKF120-T03/ +``` + +The runtime contract directory remains git-ignored. + +## Target binding + +T03 is the first Scale, Context & Iteration increment that is created after the Workspace Target Engine is available. + +Its Development Contract therefore binds: + +```text +primary: dkf-framework +affected: dkf-framework +verification: dkf-framework +``` + +and stores the exact fingerprint of: + +```text +.development-kit/workspace.json +``` + +If that registry changes after contract creation, the T03 contract becomes stale. + +## Risk and derived gates + +The validated PLAN classifies T03 as **High risk** because an incorrect cache hit could feed stale repository context into implementation or independent verification. + +Under the current DKF gate selector, risk level 3 derives: + +- specification verification; +- tests; +- code review; +- architecture review; +- security review; +- security control-domain review. + +No human consequential-action approval is derived at risk level 3. + +## Acceptance criteria + +The T03 contract owns: + +- `DKF-120-AC-008` +- `DKF-120-AC-009` +- `DKF-120-AC-010` +- `DKF-120-AC-011` +- `DKF-120-AC-035` + +## Usage + +```bash +node scripts/phase05-execution-capsule-contract.mjs +``` + +The script validates: + +- the Development Contract; +- target binding; +- derived gates; +- contract staleness. + +## Boundary + +This script authorises only T03. + +It does not authorise State Engine V2, Lifecycle Instances, discovery/re-entry, Gate Profiles, the Verification Extension SDK, Complexity Delta Guard, Control Center v0.12 integration, release publication, or a package-version change. diff --git a/docs/03-reference/scripts/phase06-state-engine-v2-contract.md b/docs/03-reference/scripts/phase06-state-engine-v2-contract.md new file mode 100644 index 00000000..baba7d8a --- /dev/null +++ b/docs/03-reference/scripts/phase06-state-engine-v2-contract.md @@ -0,0 +1,96 @@ +# phase06-state-engine-v2-contract.mjs + +## Purpose + +`scripts/phase06-state-engine-v2-contract.mjs` creates or reuses the Development Contract for **DKF120-T04 — State Engine V2**. + +The contract is persisted through the normal DKF contract store under: + +```text +.development-kit/contracts/INC-DKF120-T04/ +``` + +The runtime contract directory remains git-ignored. + +## Target binding + +T04 binds to the accepted Workspace Target Engine registry: + +```text +primary: dkf-framework +affected: dkf-framework +verification: dkf-framework +``` + +The exact `.development-kit/workspace.json` fingerprint is bound into the contract. Registry drift therefore makes the contract stale through normal DKF contract-staleness enforcement. + +## Risk and derived gates + +The validated PLAN classifies T04 as **High risk** because persistent-state migration and recovery are control-plane critical. + +The contract uses risk level 3 and derives: + +- specification verification; +- tests; +- migration verification; +- code review; +- architecture review; +- security review; +- security control-domain review. + +No consequential-action human approval is derived at risk level 3. + +## Acceptance criteria + +The T04 contract owns: + +- `DKF-120-AC-012` +- `DKF-120-AC-013` +- `DKF-120-AC-014` +- `DKF-120-AC-015` +- `DKF-120-AC-034` + +## Canonical authority rule + +T04 must preserve: + +```text +events.jsonl = canonical runtime history +snapshot.json = derived current state +index.db = disposable acceleration index +``` + +A binary or portable index can accelerate queries but may never become the only source of truth. + +## Migration rule + +Supported legacy migration must: + +1. validate legacy state before conversion; +2. preserve a recoverable legacy backup; +3. preserve legacy files until verified cutover; +4. reconstruct semantic state through canonical events; +5. verify semantic equivalence; +6. be idempotent; +7. fail closed on corruption or mismatch. + +T04 does not authorise deletion of legacy state. + +## Usage + +```bash +node scripts/phase06-state-engine-v2-contract.mjs +``` + +The script validates: + +- the Development Contract; +- target binding; +- derived gates; +- contract staleness. + +## Boundary + +This script authorises only T04. + +It does not authorise Lifecycle Instances, discovery/re-entry, Gate Profiles, the Verification Extension SDK, Complexity Delta Guard, Control Center v0.12 integration, release publication, or a package-version change. diff --git a/docs/03-reference/scripts/state-engine-v2-benchmark.md b/docs/03-reference/scripts/state-engine-v2-benchmark.md new file mode 100644 index 00000000..b9778862 --- /dev/null +++ b/docs/03-reference/scripts/state-engine-v2-benchmark.md @@ -0,0 +1,35 @@ +# state-engine-v2-benchmark.mjs + +## Purpose + +A reproducible, disposable **T04 State Engine V2 endurance fixture** that checks canonical event append and history reconstruction without modifying an existing project. + +The script creates a temporary project, appends a sequence of generic entity revisions, verifies that every event remains in its monotonic canonical hash chain, verifies the final entity state, then removes the temporary project. + +## Usage + +```bash +npm run state-engine-v2:benchmark +node scripts/state-engine-v2-benchmark.mjs --events 1000 +``` + +The CI matrix uses 300 events on **Ubuntu and Windows** to keep continuous verification bounded. The CLI accepts 1–5,000 events for targeted endurance measurements. + +## Reported observations + +The script reports `T04_ENDURANCE_EVIDENCE` JSON containing operation count, platform/Node version, append duration, read/integrity verification duration, final canonical ledger bytes and modeled disk-write savings against the previous full-ledger rewrite design. + +**Measurement boundary:** the historical rewrite comparison is a deterministic calculation from observed event-ledger lengths, not a measured execution of the old implementation. These are local deterministic fixtures, not user billing-token observations or release-wide performance claims. The implementation still revalidates/replays canonical history on reads and that CPU cost can increase with event count. + +## Authority and safety + +- `events.jsonl` remains canonical; snapshot/index are rebuildable. +- The pending journal permits deterministic repair of interrupted append writes. +- Event writes and preparation are confined under the process-aware project-local lock. +- History and final entity state must pass integrity validation. +- The fixture never modifies canonical repository release data. +- No v0.12 release approval is implied by passing this benchmark. + +## Scope + +This is T04 technical verification, not a replacement for T10 release-wide baseline comparison. diff --git a/docs/03-reference/scripts/t04-hardening-acceptance-gate.md b/docs/03-reference/scripts/t04-hardening-acceptance-gate.md new file mode 100644 index 00000000..f265ba7f --- /dev/null +++ b/docs/03-reference/scripts/t04-hardening-acceptance-gate.md @@ -0,0 +1,29 @@ +# t04-hardening-acceptance-gate.mjs + +## Purpose + +Runs the existing DKF risk-3 `decideAcceptance` engine for the T04 State Engine V2 hardening increment. It runs **only inside GitHub Actions** after Windows and Ubuntu validation and the exact `npm run release:validate` command have passed on the source being reviewed. It is an acceptance-evidence gate, not a substitute for independent review. + +## Usage + +```bash +node scripts/t04-hardening-acceptance-gate.mjs +``` + +The command requires trusted GitHub Actions repository, pull-request, workflow and job identities, an authenticated GitHub review-read token and three genuine external role-specific reviews of the **exact immutable source commit**. It cannot issue a local or implementation-authored PASS. + +## Evidence and failure conditions + +The gate loads the approved T04 Development Contract `INC-DKF120-T04`; derives verification for `AC-012`, `AC-013`, `AC-014`, `AC-015` and `AC-034`; verifies full source-tree provenance against the recorded reviewed commit; and checks authenticated GitHub code, architecture and security reviewer receipts. Review requests must bind precisely one role each. The final review receipt document is `docs/04-architecture/dkf-t04-independent-review-receipts.json`. + +It also checks **every review thread is resolved**. `scripts/t04-review-thread-blockers.mjs` rejects unresolved P1/P2 and lower-priority or unclassified threads alike, as the contract requires zero outstanding review comments. Regression tests are included in `npm run state-engine-v2:test`. + +The command records `verification.json`, `control-security.json` and `acceptance.json`, containing the source fingerprints, typed evidence, blocker/pending lists, review provenance and deterministic verdict in the T04 run directory. CI uploads the artifacts when available. Missing, edited, stale, unauthenticated or finding-bearing review evidence fails closed. + +**Passing CI alone is not acceptance.** The receipt file may remain `PENDING_INDEPENDENT_REVIEWS` until authentic reviews have completed; no reviewer receipts may be fabricated or self-certified. + +## Scope and release limitations + +The Product Owner deferred stronger protection against malicious concurrent same-account filesystem writers to future Issue [#72](https://github.com/eybersjp/development-kit/issues/72). T04 supports the explicitly documented trusted/exclusive local-runtime limitation, without claiming to prevent hostile same-UID path replacement. Restricted offline restoration has its own operator confirmation and recovery boundary. Security reviewers still assess all in-scope integrity and durability defects. + +Only after all exact-source tests, independent review, applicable approvals and the deterministic `ACCEPTED` state may PR #70 be promoted into its approved T04 base. This does not independently authorize merging the cumulative PR #69 into `main`, commencing T05, deleting retained legacy state, or publishing v0.12.0. diff --git a/docs/03-reference/scripts/t04-review-thread-blockers.md b/docs/03-reference/scripts/t04-review-thread-blockers.md new file mode 100644 index 00000000..2bfb3fa9 --- /dev/null +++ b/docs/03-reference/scripts/t04-review-thread-blockers.md @@ -0,0 +1,22 @@ +# t04-review-thread-blockers.mjs + +## Purpose + +Provides a pure, deterministic helper to identify unresolved GitHub pull-request review threads. The DKF120-T04 risk-3 independent acceptance gate must refuse approval while **any review thread remains unresolved**, including P0/P1/P2, unclassified findings and missing resolution status. Priority labels are not a sufficient substitute for the documented requirement of zero outstanding review comments. + +## Usage + +The helper is imported by `scripts/t04-hardening-acceptance-gate.mjs` and is not an independently runnable acceptance command. + +```js +import { unresolvedReviewThreads } from './t04-review-thread-blockers.mjs'; +const blockers = unresolvedReviewThreads(githubReviewThreads); +``` + +It returns every input thread without an explicit `isResolved: true` marker. Malformed top-level evidence fails closed. The module also provides `requireCanonicalReviewRequestCommit()` to enforce one uniquely anchored full source SHA in role-scoped review requests, and `parseReviewThreadPage()` to reject missing or malformed authenticated GraphQL thread pages before pagination. An authenticated reaction alone carries no source SHA; ambiguous or conflicting requests cannot establish it. No caller may use these functions to generate external reviewer PASS results or bypass authenticated source-bound review receipts. + +## Verification + +`node --test scripts/t04-review-thread-blockers.test.mjs` runs deterministic regressions covering unresolved P1/P2/unclassified comments, resolved threads, conflicting/duplicated commit SHAs, null GraphQL pages and malformed pagination evidence. It is included in `npm run state-engine-v2:test` and therefore the full `npm run release:validate` pipeline. + +This helper is part of the T04 hardening increment and does not authorize T05, a merge, or publication of v0.12.0. diff --git a/docs/03-reference/scripts/windows-durable-rename.md b/docs/03-reference/scripts/windows-durable-rename.md new file mode 100644 index 00000000..4a69668b --- /dev/null +++ b/docs/03-reference/scripts/windows-durable-rename.md @@ -0,0 +1,21 @@ +# windows-durable-rename.ps1 + +## Purpose + +Windows-only, source-controlled offline restore recovery helper for the T04 State Engine V2 legacy migration. It uses the native Unicode `MoveFileExW` API with `MOVEFILE_WRITE_THROUGH` to move a displaced recovery directory back into the intended target path. Node.js `fs.renameSync` does not expose this flag, and Node's portable directory `fsync` operation cannot establish the corresponding Windows namespace durability. + +## Execution + +The helper is **not** a user-facing command. `runtime/orchestration/state-engine-migration.mjs` calls it with `execFileSync` using an explicitly located, bundled Windows PowerShell executable and literal `-From` / `-To` arguments. There is no shell string interpolation or remotely downloaded script. + +The script invokes Win32 `MoveFileExW` and propagates native error codes. If PowerShell, `Add-Type`, the helper, or the native rename fails, the caller raises a `StateMigrationError` and **retains the restore journal** rather than treating an uncertain recovery as acknowledged. + +## Verification and limitations + +`T04-H48` exercises a real Windows child-process death after root displacement, ensures recovery does not use Node's ordinary rename, and verifies that the journal is removed only after the restored original becomes visible. `T04-H47` separately proves parent-directory `fsync` ordering on POSIX hosts. + +The Microsoft `MOVEFILE_WRITE_THROUGH` documentation says the move does not return until the file is moved on disk, while expressly describing a copy/delete flush guarantee. This is the selected **candidate native mechanism**, not independent proof of same-volume NTFS crash durability on every supported filesystem. Security/architecture reviewers must verify the supported Windows storage and transaction guarantees; this helper must not be represented as a universal power-loss guarantee based solely on a functional CI test. + +Official API references: [MoveFileExW](https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-movefileexw) and [Moving Directories](https://learn.microsoft.com/en-us/windows/win32/fileio/moving-directories). + +This helper does not authorize T04 acceptance, broader same-account process isolation, T05, merging or a v0.12.0 release. diff --git a/docs/04-architecture/dkf-cost-observatory-t01-review.md b/docs/04-architecture/dkf-cost-observatory-t01-review.md new file mode 100644 index 00000000..8cf28a3b --- /dev/null +++ b/docs/04-architecture/dkf-cost-observatory-t01-review.md @@ -0,0 +1,150 @@ +# DKF120-T01 — Cost Observatory Review + +**Review status:** PASS +**Date:** 7 October 2026 +**Branch:** `feature/v0.12-cost-observatory` +**Reviewed head:** `3b6164dc67a6ceb98ca43e971c2b4636a6c95aa3` + +## Scope reviewed + +Production/runtime change: + +- `runtime/orchestration/cost-observatory.mjs` +- `runtime/orchestration/index.mjs` +- `schemas/cost-record.schema.json` +- `.gitignore` + +Contract/test/integration change: + +- `scripts/phase03-cost-observatory-contract.mjs` +- `scripts/cost-observatory.test.mjs` +- `package.json` +- `.github/workflows/ci.yml` +- script reference documentation and documentation index. + +## Contract compliance + +The implementation remains inside DKF120-T01. + +It does not implement: + +- Workspace Target Engine; +- Execution Capsules/context caching; +- State Engine V2; +- Lifecycle Instances; +- discovery/re-entry; +- Gate Profiles; +- Verification Extension SDK; +- Complexity Delta Guard; +- Control Center v0.12 work. + +No application-specific or industry-specific business logic is present. + +## Architecture review + +### General-purpose core + +PASS. + +The runtime operates on generic operation identifiers, context text, optional host/provider usage counters, generic orchestration counters, timestamps, and optional lifecycle/contract/task/run references. + +There are no solar, energy, finance, healthcare, CAD, legal, ecommerce, or other domain semantics. + +### Host/provider independence + +PASS. + +Provider token values are optional numeric observations. No provider library, provider credential, API key, provider-specific object, or external service is required. + +### Observational boundary + +PASS. + +The Cost Observatory module is exported but is not consulted by: + +- `acceptance-engine.mjs`; +- `gate-selector.mjs`; +- Development Contract validation; +- lifecycle/autopilot state transitions. + +A regression test proves that writing a cost observation does not alter deterministic acceptance output other than the acceptance engine's own observation timestamp. + +### Persistence + +PASS for T01 scope. + +Cost observations use one append-only project-local JSONL ledger: + +```text +.development-kit/telemetry/cost-records.jsonl +``` + +The telemetry directory is git-ignored. + +This is telemetry persistence only. It does not pre-empt State Engine V2 canonical-state design. + +### Cross-platform measurement + +PASS. + +Logical text measurement normalises `CRLF` and legacy `CR` to `LF` before character/byte/token estimation. + +The historical `token:audit` implementation remains unchanged. + +## Review findings and corrections + +### REV-T01-001 — Test incorrectly compared volatile acceptance timestamps + +**Severity:** Minor test defect +**Status:** Corrected + +The first CI execution failed because the observational regression test deep-compared two acceptance records whose `createdAt` values were generated independently. + +Acceptance state/gates were identical. + +Correction: + +- preserve the assertion that both records are `ACCEPTED`; +- compare the remaining acceptance semantics with the expected observation timestamp normalised out. + +Correction commit: + +`869bde49c29495507d21cf6a1395dd4161c6f29c` + +### REV-T01-002 — Non-security architecture rule was classified as a security constraint + +**Severity:** Governance classification issue +**Status:** Corrected + +The original T01 contract task placed “does not require provider credentials” in `securityConstraints`. + +Under existing DKF gate-selection rules any non-empty security constraint correctly adds `security-reviewer` and the security control domain. That exceeded the validated T01 PLAN, whose required reviewer is `code-reviewer`. + +The Cost Observatory does not read, persist, or request credentials; the statement is an architectural boundary rather than a security-sensitive implementation requirement. + +Correction: + +- moved the no-credentials/no-fabrication rules to architecture constraints; +- left `securityConstraints` empty; +- added a contract-time assertion that derived T01 gates are exactly: + - verification: `specification`, `tests`; + - reviewer: `code-reviewer`; + - no specialist control domain; + - no human approval; + - no configuration-readiness dependency. + +Correction commit: + +`3b6164dc67a6ceb98ca43e971c2b4636a6c95aa3` + +## Outstanding findings + +Critical: **0** +Major: **0** +Unresolved minor: **0** + +## Review verdict + +**PASS** + +The T01 implementation is appropriately small, general-purpose, contract-bounded, and non-authoritative. diff --git a/docs/04-architecture/dkf-execution-capsule-t03-review.md b/docs/04-architecture/dkf-execution-capsule-t03-review.md new file mode 100644 index 00000000..9b4f3cf7 --- /dev/null +++ b/docs/04-architecture/dkf-execution-capsule-t03-review.md @@ -0,0 +1,284 @@ +# DKF120-T03 — Execution Capsules & Context Cache Review + +**Review status:** PASS +**Date:** 7 October 2026 +**Increment:** DKF120-T03 +**Branch:** `feature/v0.12-execution-capsules-context-cache` +**Reviewed implementation head:** `765a64a577238d73169fc9ae91bb8697a76ded6d` +**Implementation validation run:** `37672806212` + +## 1. Review gates + +The T03 Development Contract is risk level 3. + +Derived gates: + +- specification verification; +- tests; +- code-reviewer; +- architecture-reviewer; +- security-reviewer; +- security control domain. + +No human consequential-action approval was derived. + +All required review perspectives are recorded below. + +## 2. Code review + +**Verdict: PASS** + +The implementation adds: + +- `runtime/orchestration/context-cache.mjs`; +- `runtime/orchestration/execution-capsule.mjs`; +- capsule-aware integration in `context-package.mjs`; +- public orchestration exports; +- Execution Capsule and context-cache schemas; +- focused acceptance tests. + +The implementation remains within T03 scope. + +No T04+ production capability was introduced. + +### Code-review findings + +#### REV-T03-001 — Synthetic configuration timestamp caused permanent cache misses + +**Severity:** Major during implementation +**Status:** Corrected + +When no Configuration Readiness registry existed, `loadConfigurationRegistry()` returned an in-memory object containing a fresh `updatedAt` timestamp. + +Fingerprinting that synthetic object made identical cache inputs look different on every call. + +Correction: + +- cache invalidation now fingerprints the persisted Configuration Readiness registry file only when it exists; +- absence is represented by stable `null`; +- persisted registry changes still invalidate the cache. + +This converted repeated false misses into deterministic hits without weakening configuration invalidation. + +#### REV-T03-002 — Cache entry semantic fingerprint originally included creation time + +**Severity:** Major for determinism +**Status:** Corrected before acceptance + +`createdAt` was removed from semantic cache-entry fingerprinting. + +The timestamp remains audit metadata, while identical semantic inputs now produce the same cache-entry and capsule identity across rebuilds. + +#### REV-T03-003 — Changed files could have depended on caller relevance classification + +**Severity:** Major reliability risk +**Status:** Corrected + +Present changed files are now automatically unioned into the relevant-file invalidation set. + +A caller can no longer provide a changed-file delta while accidentally omitting the same present file from content-fingerprint invalidation. + +#### REV-T03-004 — Low-level cache I/O needed an explicit storage boundary + +**Severity:** Security hardening +**Status:** Corrected + +Cache load/write helpers now refuse paths outside: + +```text +.development-kit/cache/context +``` + +Relevant/changed file reads are project-root checked, symbolic-link content reads are rejected, and target roots are realpath checked before scanning. + +#### REV-T03-005 — Capsule creation needed to validate cache-entry integrity directly + +**Severity:** Security/reliability hardening +**Status:** Corrected + +`createExecutionCapsule()` now validates the supplied cache entry before using it. + +A forged or internally inconsistent cache entry cannot be converted into a valid capsule. + +## 3. Architecture review + +**Verdict: PASS** + +### 3.1 Existing context authority preserved + +T03 does **not** replace `buildContextPackage()`. + +The flow is: + +```text +Development Contract + + +Workspace target binding + + +Repository context cache + + +Relevant file/test fingerprints + + +Runtime/evidence references + ↓ +Execution Capsule + ↓ +buildContextPackage() + ↓ +fresh / independently rehydrated role context +``` + +Authoritative sources are still independently: + +1. located; +2. whole-file fingerprint checked; +3. section materialised; +4. inserted into the role context. + +The capsule does not carry authoritative source content. + +### 3.2 Cache remains acceleration-only + +The context cache is not canonical state. + +Deletion results in a safe cache miss. + +Corruption results in a safe cache miss and rebuild. + +An old capsule whose referenced cache entry disappears becomes stale and cannot be used until a new capsule is built. + +### 3.3 Target-aware invalidation + +T03 builds on the T02 target model. + +Cache inputs include: + +- Development Contract source fingerprint; +- Development Mode fingerprint; +- workspace registry fingerprint where target-aware; +- persisted Configuration Readiness fingerprint where present; +- selected target structural fingerprints; +- relevant file fingerprints; +- relevant test fingerprints; +- optional future gate-profile revision. + +Target structure facts contain path/structure information only. + +Content changes outside the declared relevant file/test set do not invalidate structural facts unless repository structure itself changes. + +### 3.4 Unrelated target isolation + +A file/content/structure change in an unselected unrelated target does not invalidate a target-scoped cache or capsule when the Development Contract/workspace authority is otherwise unchanged. + +This is the key scaling property required by the multi-target architecture. + +### 3.5 Fresh-role isolation preserved + +Implementation and verification contexts may receive the same deterministic capsule references, but: + +- verifier roles are still independently rehydrated; +- implementation output remains non-authoritative; +- the capsule is freshness checked before context construction; +- authoritative source reads still happen independently. + +## 4. Security review + +**Verdict: PASS** + +The review considered stale-context injection, path escape, cache tampering, symlink content access, cache poisoning, and authority escalation. + +Controls confirmed: + +- project-root path checks; +- context-cache-root write/read confinement; +- cache-entry content fingerprint validation; +- capsule semantic fingerprint validation; +- Development Contract identity/fingerprint binding; +- workspace registry binding; +- relevant file/test content fingerprints; +- automatic changed-file invalidation participation; +- corrupted cache fail-safe miss; +- missing cache stale-capsule rejection; +- non-authoritative runtime-fact marking; +- no copied secret values in cache invalidation; +- no cached summary becomes specification/business authority. + +No unresolved critical/major security finding remains. + +## 5. Security control-domain review + +**Verdict: PASS** + +The risk-3 security control domain is satisfied by deterministic rejection of untrusted/stale cache state before role context construction. + +The security boundary is fail closed for: + +- path escape; +- invalid cache-entry fingerprints; +- stale relevant files/tests; +- stale Development Contract authority; +- stale workspace registry; +- stale configuration registry; +- missing capsule cache entry; +- capsule/cache mismatch. + +A cache miss affects performance only; it cannot grant authority or bypass verification. + +## 6. Verification evidence + +Focused T03 suite on the reviewed implementation head: + +```text +tests: 19 +pass: 19 +fail: 0 +``` + +GitHub Actions run `37672806212`: + +- Ubuntu: PASS; +- Windows: PASS; +- focused T03 gate: PASS on both; +- exact `release:validate`: PASS on both. + +## 7. Measured context-economy evidence + +The focused two-target fixture recorded: + +| Measure | Baseline | Capsule/cache | Reduction | +|---|---:|---:|---:| +| Logical context bytes | 9,591 | 7,053 | 26.46% | +| Estimated context tokens | 2,398 | 1,764 | 26.44% | +| Target-orientation scans | 2 | 1 | 50.00% | +| Cache hit | 0 | 1 | — | +| Cache miss | 0 | 0 | — | + +These are T03 fixture results only. + +They are **not** a claim that the release-wide v0.12 performance objectives have already been reached. + +Release-wide targets remain owned by later comparative validation. + +## 8. Scope check + +T03 did not implement: + +- State Engine V2; +- Lifecycle Instances; +- discovery/re-entry; +- Gate Profiles; +- Verification Extension SDK; +- Complexity Delta Guard; +- Control Center v0.12 integration. + +## 9. Outstanding findings + +Critical: **0** +Major: **0** +Unresolved minor: **0** + +## 10. Review decision + +**PASS** + +DKF120-T03 is technically suitable for acceptance and progression to the next validated PLAN increment. diff --git a/docs/04-architecture/dkf-scale-context-iteration-phase01-baseline.md b/docs/04-architecture/dkf-scale-context-iteration-phase01-baseline.md new file mode 100644 index 00000000..ca8ee274 --- /dev/null +++ b/docs/04-architecture/dkf-scale-context-iteration-phase01-baseline.md @@ -0,0 +1,348 @@ +# DKF Scale, Context & Iteration — Phase 0/1 Baseline + +**Status:** Completed baseline evidence +**Date:** 7 October 2026 +**Repository:** `eybersjp/development-kit` +**Authoritative main baseline:** `62badbf7d0abee3344b1f5db2b2fbc876cf004e2` +**Analysis branch:** `analysis/scale-context-iteration-phase01` +**Baseline CI run:** GitHub Actions run `37618731275` +**Current framework version:** `0.11.2` + +## 1. Purpose + +This document records Phase 0 repository reconnaissance and Phase 1 measurements for the Scale, Context & Iteration initiative. + +No DKF production runtime behaviour, package version, release tag, publication path, safety gate, acceptance rule, or verification rule was changed to obtain this baseline. + +The baseline runner is `scripts/phase01-scale-context-baseline.mjs`. + +## 2. Phase 0 findings + +### 2.1 Release/version finding + +The approved working specification originally proposed `v0.11.0`. + +The live repository is already `0.11.2`, and `v0.11.0`, `v0.11.1`, and `v0.11.2` already exist in release history. + +The repository versioning policy defines backwards-compatible new runtime capabilities as a MINOR increment. + +**Recommended feature release target: `v0.12.0`.** + +No version change is made during Phase 0/1. + +### 2.2 Current repository baseline + +The authoritative main commit contained: + +| Measure | Main baseline | +|---|---:| +| Repository files | 793 | +| Repository bytes | 4,643,607 | +| Runtime files | 73 | +| Script files | 74 | +| Test files (`*.test.mjs`) | 62 | +| Agent files | 18 | +| Command files | 16 | +| Skill files | 64 | +| Schema files | 11 | +| Documentation files | 313 | + +The analysis branch adds only diagnostic/reference material required to execute this baseline. + +### 2.3 Existing token/context work is substantial + +The current framework already contains: + +- section-aware authoritative-source materialisation; +- whole-file fingerprint/staleness protection; +- role-specific context packages; +- advisory role context budgets; +- static instruction token budgets; +- fail-safe full-source fallback when a requested selector cannot be resolved; +- independent verification contexts; +- policy guidance for cache-first repository orientation. + +Therefore the new initiative must extend this implementation rather than recreate it. + +### 2.4 Current state persistence confirms the micro-file scaling concern + +Autopilot currently persists immutable workflow snapshots as: + +```text +.development-kit/autopilot/state/ + revision-000001.json + revision-000002.json + ... + current.json +``` + +Orchestration runs separately persist: + +```text +runs/// + manifest.json + state-revisions/00000001.json + state-revisions/00000002.json + ... + current-state.json + final-state.json +``` + +This gives strong auditability and recovery, but state-file count grows with state transitions at both workflow and run level. + +### 2.5 Context rehydration is already safe but remains expensive by design + +`buildContextPackage()`: + +- validates Development Contract staleness; +- verifies authoritative source fingerprints; +- materialises requested source sections; +- creates fresh/rehydrated role context; +- prevents implementation output from becoming authority. + +However, the current context metadata still declares `repositoryReRead: true`, and no first-class workspace-target/context-cache engine exists. + +### 2.6 Current gate selection is risk-aware but not domain-aware + +The gate selector currently understands: + +- specification verification; +- required verification declared by the contract; +- code review; +- architecture sensitivity; +- security sensitivity; +- UI/design sensitivity; +- configuration dependencies; +- human approvals. + +It does not yet provide first-class numerical/engineering/financial/domain verification profiles. + +### 2.7 Current lifecycle remains one canonical global sequence + +The current Autopilot state machine still governs: + +```text +UNDERSTAND -> DEFINE -> DESIGN -> PLAN -> IMPLEMENT -> VERIFY -> REVIEW -> SIMPLIFY -> COMPLETE +``` + +There is no first-class parent/child Lifecycle Instance or formal discovery-driven minimum re-entry model. + +## 3. Phase 1 execution evidence + +The baseline was executed through the repository's normal CI matrix using Node `v22.23.3`. + +Both platforms passed the complete CI matrix: + +| Platform | Result | +|---|---| +| Ubuntu | PASS | +| Windows | PASS | + +The same run also passed all existing validation before and after the new baseline step, including orchestration, evidence/control coverage, v0.9 reliability regressions, Live UI Preview, token/context efficiency, Design Authority, Autopilot, and exact release-command validation. + +## 4. Existing static token baseline + +The current estimator is `chars-div-4-v1`. These values are deterministic local approximations, not provider billing-token measurements. + +### Ubuntu + +| Measure | Current | Historical baseline | Reduction | Budget | +|---|---:|---:|---:|---:| +| Priority runtime skills | ~2,820 | 10,436 | 72.98% | <=3,300 | +| Implementation hot path | ~7,587 | 19,994 | 62.05% | <=9,000 | + +### Windows + +| Measure | Current | Historical baseline | Reduction | Budget | +|---|---:|---:|---:|---:| +| Priority runtime skills | ~2,901 | 10,436 | 72.20% | <=3,300 | +| Implementation hot path | ~7,747 | 19,994 | 61.25% | <=9,000 | + +Both platforms are inside the existing budgets. + +### Measurement-quality finding + +The Windows checkout contains CRLF line endings while Ubuntu uses LF. Because the current estimator counts raw characters, the same logical instruction set reports different values: + +- priority-runtime difference: 81 estimated tokens (~2.87%); +- implementation-hot-path difference: 160 estimated tokens (~2.11%); +- analysis-branch repository-byte difference: 86,297 bytes (~1.85%). + +This is not evidence of actual model-token drift. + +The future Cost Observatory should normalise line endings for cross-platform deterministic context-size comparisons while preserving the existing estimator's historical regression compatibility where required. + +## 5. Representative context fixtures + +### 5.1 Single-package fixture + +Synthetic workspace files: **32** + +| Measure | Result | +|---|---:| +| Raw authoritative-source tokens | 359 | +| Delivered authoritative-source tokens | 65 | +| Source reduction | 81.89% | +| Estimated complete context package | 1,166 | +| Role budget exceeded | No | +| Source delivery | Scoped | +| Selected verification | specification + tests | +| Selected reviewer | code-reviewer | + +### 5.2 Multi-package fixture + +Synthetic workspace files: **265** + +Represented topology: + +```text +apps/web +apps/mobile +packages/shared +packages/calculations +supabase/migrations +``` + +| Measure | Result | +|---|---:| +| Raw authoritative-source tokens | 1,405 | +| Delivered authoritative-source tokens | 76 | +| Source reduction | 94.59% | +| Estimated complete context package | 3,190 | +| Role budget exceeded | No | +| Source delivery | Scoped | +| Selected verification | specification + tests | +| Selected reviewer | code-reviewer | + +The source materialisation is efficient, but the current runtime has no first-class concept of `@web`, `@mobile`, `@database`, `@calculations`, target-specific command routing, or target dependency closure. + +### 5.3 Calculation-heavy fixture + +Synthetic workspace files: **150** + +| Measure | Result | +|---|---:| +| Raw authoritative-source tokens | 2,410 | +| Delivered authoritative-source tokens | 180 | +| Source reduction | 92.53% | +| Estimated complete context package | 2,129 | +| Role budget exceeded | No | +| Source delivery | Scoped | +| Selected verification | specification + tests | +| Selected reviewer | code-reviewer | + +This is the key domain-verification baseline. + +Despite the fixture being explicitly calculation-heavy, current gate selection still resolves only generic specification/tests plus code review. There is no automatic numerical tolerance, invariant, unit, golden-dataset, replay, or engineering sanity gate. + +## 6. Legacy state-growth fixture + +A real `saveStateRevision()` loop was executed for **26 revisions**. + +Measured result on both platforms: + +| Measure | Result | +|---|---:| +| Requested revisions | 26 | +| Immutable revision files | 26 | +| Pointer files | 1 | +| Total workflow state files | 27 | +| Total state bytes | 17,537 | +| Files per revision (including pointer amortisation) | ~1.038 | + +This fixture measures only the Autopilot workflow-level state store. + +It does **not** include the additional per-run orchestration revision chains, contracts, evidence records, IDEA journals/receipts, memory data, UI-preview state, or other project-local runtime state. + +Therefore it confirms the linear micro-file behaviour but should not be interpreted as the total state cost of a complex project. + +## 7. Observability gaps + +The current DKF runtime cannot centrally report the following as measured orchestration costs: + +| Metric | Current status | +|---|---| +| Provider billing/input/output tokens | Unavailable | +| Repository file-read count per run | Unavailable | +| Repository-orientation scan count per run | Unavailable | +| Host agent invocation count per run | Unavailable | +| Host tool-call count per run | Unavailable | + +The framework already measures estimated context size and static instruction size. + +It does **not** yet have the end-to-end Cost Observatory needed to prove file-read, reorientation, agent-call, tool-call, and provider-token improvements. + +These values must remain `null/unavailable` until instrumented. They must not be inferred. + +## 8. Baseline conclusions + +### Confirmed strengths + +1. Source section scoping is already highly effective in all three fixtures. +2. Whole-file source authority and independent verification remain intact. +3. Existing token budgets are comfortably green. +4. Existing control-plane regression coverage remains green on Ubuntu and Windows. +5. Documentation governance correctly blocked an undocumented diagnostic script during the first baseline attempt; the analysis work had to comply with the documentation contract rather than bypass it. + +### Confirmed gaps + +1. **State micro-file growth is real.** +2. **Workspace targets are not first-class.** +3. **Calculation/domain verification is not first-class.** +4. **Lifecycle nesting/re-entry is not first-class.** +5. **Runtime context costs beyond estimated package size are not centrally observable.** +6. **Repository context reuse is policy-level rather than a complete target-aware cache/invalidation engine.** +7. **Cross-platform static token measurements need newline normalisation for exact comparability.** + +## 9. Impact on the approved initiative + +The implementation sequence remains valid, but the repository evidence changes the emphasis. + +### Keep + +- Cost Observatory first; +- Workspace Target Engine; +- Execution Capsules/context cache; +- State Engine V2; +- Lifecycle Instances; +- Discovery/re-entry; +- Gate Profiles; +- Domain Verification SDK; +- Complexity Delta Guard; +- Control Center integration. + +### Do not duplicate + +Do not rebuild: + +- source section materialisation; +- token-profile generation; +- current static token audit; +- whole-file fingerprint protection; +- independent verification context; +- current risk/reviewer selection. + +Extend these components. + +## 10. Recommended release identity + +Subject to Product Owner approval and final PLAN validation: + +```text +Development Kit v0.12.0 +Scale, Context & Iteration +``` + +This is a backwards-compatible feature release under the repository's Semantic Versioning policy. + +## 11. Phase 0/1 verdict + +**PHASE 0: COMPLETE** + +Repository architecture, current version, state persistence, context packaging, gate selection, existing token hardening, CI/test surface, and relevant constraints have been inspected. + +**PHASE 1: COMPLETE** + +A reproducible analysis runner has executed against representative single-package, multi-package, calculation-heavy, and legacy-state fixtures on Ubuntu and Windows. Existing release validation remains green. + +The next permitted specification step is to reconcile the approved Scale, Context & Iteration specification against these findings and enter DEFINE/DESIGN/PLAN for the implementation release. Production implementation has not started. diff --git a/docs/04-architecture/dkf-scale-context-iteration-phase02-validation.md b/docs/04-architecture/dkf-scale-context-iteration-phase02-validation.md new file mode 100644 index 00000000..d5d31ec7 --- /dev/null +++ b/docs/04-architecture/dkf-scale-context-iteration-phase02-validation.md @@ -0,0 +1,156 @@ +# DKF Scale, Context & Iteration — Phase 2 Validation + +**Status:** COMPLETE +**Date:** 7 October 2026 +**Target release:** v0.12.0 +**Analysis branch:** `analysis/scale-context-iteration-phase01` +**Validated commit:** `21267e9144f3a31dc9ebd070e470af0ef7451832` +**Validation run:** GitHub Actions `37659800053` + +## 1. Phase 2 objective + +Reconcile the provisional Scale, Context & Iteration direction against the live v0.11.2 repository and Phase 0/1 evidence, then produce: + +- final v0.12 specification; +- technical design; +- machine-readable implementation PLAN; +- deterministic PLAN validation; +- no production implementation. + +## 2. General-purpose architecture lock + +The Phase 2 design explicitly locks DKF as a general-purpose framework. + +Core rules: + +- no application-specific business rules in DKF core; +- no industry-specific policy in DKF core; +- target IDs are arbitrary/project-defined; +- target routing is capability/configuration driven; +- project/domain verification is supplied through extensions/adapters/configuration; +- DKF core supplies generic verification primitives only; +- examples and fixtures do not become core policy. + +The earlier label **Domain Verification SDK** is replaced by **Verification Extension SDK**. + +## 3. Canonical Phase 2 artifacts + +- `dkf-scale-context-iteration-v0.12-specification.md` +- `dkf-scale-context-iteration-v0.12-technical-design.md` +- `dkf-scale-context-iteration-v0.12-implementation-plan.md` +- `dkf-scale-context-iteration-v0.12-plan-model.json` +- `scripts/phase02-scale-context-plan-validate.mjs` + +## 4. PLAN structure + +The validated implementation PLAN contains 10 increments: + +1. DKF120-T01 — Cost Observatory +2. DKF120-T02 — Workspace Target Engine +3. DKF120-T03 — Execution Capsules & Context Cache +4. DKF120-T04 — State Engine V2 +5. DKF120-T05 — Lifecycle Instances +6. DKF120-T06 — Discovery & Controlled Re-entry +7. DKF120-T07 — Generic Gate Profiles +8. DKF120-T08 — Verification Extension SDK +9. DKF120-T09 — Complexity Delta Guard +10. DKF120-T10 — Control Center + Comparative Release Validation + +The declared dependency graph is sequential and explicit: + +```text +T01 -> T02 -> T03 -> T04 -> T05 -> T06 -> T07 -> T08 -> T09 -> T10 +``` + +## 5. Deterministic PLAN validation + +The existing `runtime/orchestration/plan-validator.mjs` was used. + +Observed result: + +```json +{ + "planId": "DKF-V012-SCALE-CONTEXT-ITERATION", + "schemaVersion": "1.0.0", + "valid": true, + "computed": { + "taskCount": 10 + }, + "issues": [] +} +``` + +The validator confirmed: + +- task count = 10; +- every dependency points to a declared task; +- no dependency cycle; +- declared dependency diagram matches computed dependency graph; +- every required logical resource has an owner; +- no duplicate logical resource owners; +- every `DKF-120-AC-001` through `DKF-120-AC-040` acceptance criterion is covered; +- no unknown configuration dependencies exist in the PLAN. + +The PLAN-validation step passed on both Ubuntu and Windows. + +## 6. Full regression result + +GitHub Actions run `37659800053` completed successfully on both: + +- Ubuntu; +- Windows. + +The branch passed the existing validation matrix including: + +- skills/agents; +- plugin synchronization; +- installer integrity; +- Development Contract foundation; +- execution safety; +- evidence/control coverage; +- orchestration runtime and integration; +- v0.9 reliability regressions; +- documentation validation; +- platform/OpenCode adapters; +- research contract; +- v0.7.1 regressions; +- Live UI Preview; +- token/context efficiency; +- Phase 0/1 baseline; +- Phase 2 deterministic PLAN; +- Design Authority; +- Autopilot/evaluations; +- exact release-command validation. + +No production implementation was introduced. + +## 7. Phase 2 release identity + +The reconciled feature target is: + +```text +Development Kit v0.12.0 +Scale, Context & Iteration +``` + +The package version remains v0.11.2 during planning. + +## 8. Implementation boundary + +Phase 2 authorises the PLAN, not implementation. + +The first production increment is: + +```text +DKF120-T01 — Cost Observatory +``` + +Before production code changes, DKF should create the Development Contract for T01 using the v0.12 specification, technical design, PLAN, and Phase 0/1 baseline as authoritative sources. + +## 9. Verdict + +**PHASE 2: COMPLETE** + +The v0.12 direction is reconciled against the actual repository, explicitly general-purpose, technically designed, decomposed into stable increments, and deterministically PLAN-valid. + +The next lifecycle step is **Phase 3 / implementation increment DKF120-T01 — Cost Observatory**, governed by a Development Contract and the existing independent verification/acceptance controls. diff --git a/docs/04-architecture/dkf-scale-context-iteration-phase03-t01-validation.md b/docs/04-architecture/dkf-scale-context-iteration-phase03-t01-validation.md new file mode 100644 index 00000000..d9927b87 --- /dev/null +++ b/docs/04-architecture/dkf-scale-context-iteration-phase03-t01-validation.md @@ -0,0 +1,200 @@ +# Development Kit v0.12.0 — Phase 3 / DKF120-T01 Validation + +**Status:** COMPLETE — ACCEPTED FOR PROGRESSION +**Date:** 7 October 2026 +**Increment:** DKF120-T01 — Cost Observatory +**Implementation branch:** `feature/v0.12-cost-observatory` +**Validated head:** `3b6164dc67a6ceb98ca43e971c2b4636a6c95aa3` +**Validation PR:** #61 +**Final CI run:** `37665136625` + +## 1. Development Contract + +The T01 contract is generated through normal DKF runtime contract machinery and persisted as git-ignored runtime state. + +Observed final contract: + +- Contract ID: `INC-DKF120-T01` +- Task ID: `DKF120-T01` +- Contract schema: `1.2.0` +- Source fingerprint: `sha256:5a89b0c0dd841d09cfcbf465ffc36c4c0e768574282c4c94aa389d713231f661` +- Risk: 2 +- Stale: false + +Bound acceptance criteria: + +- `DKF-120-AC-001` +- `DKF-120-AC-002` +- `DKF-120-AC-003` +- `DKF-120-AC-033` + +Derived gates: + +```text +verification: + specification + tests + +reviewers: + code-reviewer + +control domains: + none + +human approvals: + none + +configuration readiness: + not required +``` + +## 2. Implemented capability + +T01 adds a generic Cost Observatory foundation with: + +- versioned `CostRecord` contract; +- canonical newline-normalised logical context measurement; +- deterministic estimated context token metric; +- optional provider input/output/cached-token fields; +- null/unavailable semantics for unobserved host/provider counts; +- explicit DKF-controlled counters; +- operation elapsed-time measurement; +- append-only project-local JSONL telemetry ledger; +- validated ledger reload; +- baseline/current comparison; +- public orchestration API export. + +The telemetry ledger is: + +```text +.development-kit/telemetry/cost-records.jsonl +``` + +and is ignored by Git. + +## 3. Acceptance-criterion evidence + +### DKF-120-AC-001 + +PASS. + +The cost meter and `createCostRecord()` accept explicit DKF-controlled measurements and deterministic context estimates. Invalid values fail closed. + +### DKF-120-AC-002 + +PASS. + +Provider token values are `null` unless actual values are supplied. Unknown repository reads/scans, agent invocations, tool calls, and verification-command counts also remain `null`; they are not inferred as zero. + +### DKF-120-AC-003 + +PASS. + +LF and CRLF representations of the same logical text produce the same logical character count, byte count, and estimated context-token count. + +The pre-existing historical static `token:audit` path was not modified. + +### DKF-120-AC-033 + +PASS for the T01 foundation. + +`compareCostRecords()` produces baseline/current deltas and reduction percentages for available metrics and explicitly reports unavailable comparisons as unavailable/null. + +Final release-wide before/after conclusions remain owned by T10. + +## 4. Focused tests + +Final focused result on Ubuntu: + +```text +tests: 9 +pass: 9 +fail: 0 +``` + +The same T01 Cost Observatory gate passed on Windows. + +Covered behaviours include: + +- newline normalisation; +- explicit counters; +- optional provider telemetry; +- unavailable-host-metric semantics; +- invalid-value fail-closed behaviour; +- append-only JSONL persistence; +- cost comparison; +- acceptance isolation; +- schema alignment. + +## 5. Regression validation + +Final GitHub Actions run `37665136625` passed on: + +- Ubuntu; +- Windows. + +The final head passed: + +- existing skill/agent validation; +- plugin synchronization; +- installer integrity; +- Development Contract tests; +- execution safety; +- evidence/control coverage; +- orchestration core/integration; +- v0.9 reliability regressions; +- documentation; +- OpenCode/platform adapters; +- external research; +- v0.7.1 regressions; +- Live UI Preview; +- existing token/context efficiency; +- Phase 0/1 baseline; +- Phase 2 deterministic PLAN; +- T01 Cost Observatory validation; +- Design Authority; +- Autopilot/evaluations; +- exact `release:validate`. + +## 6. Correction history + +The evidence trail preserves two corrections: + +1. an invalid deep-equality test over a volatile acceptance timestamp; +2. a Development Contract classification that unnecessarily derived a security-review gate. + +Both were corrected and independently rerun through the full CI matrix. + +See: + +`dkf-cost-observatory-t01-review.md` + +## 7. General-purpose framework check + +PASS. + +T01 contains no application-specific or industry-specific business logic. + +The capability is applicable to any DKF-managed software project independent of application type, repository topology, host, model, provider, or technology stack. + +## 8. Acceptance decision + +**DKF120-T01 — ACCEPTED FOR PROGRESSION** + +This decision authorises progression to the next validated PLAN increment only. + +It does not: + +- publish v0.12.0; +- bump the package version; +- merge to `main`; +- authorise release; +- authorise T03 or later work before T02 is completed and accepted. + +## 9. Next increment + +```text +DKF120-T02 — Workspace Target Engine +``` + +T02 must receive its own Development Contract before production implementation. diff --git a/docs/04-architecture/dkf-scale-context-iteration-phase04-t02-validation.md b/docs/04-architecture/dkf-scale-context-iteration-phase04-t02-validation.md new file mode 100644 index 00000000..73a95e85 --- /dev/null +++ b/docs/04-architecture/dkf-scale-context-iteration-phase04-t02-validation.md @@ -0,0 +1,271 @@ +# Development Kit v0.12.0 — Phase 4 / DKF120-T02 Validation + +**Status:** COMPLETE — ACCEPTED FOR PROGRESSION +**Date:** 7 October 2026 +**Increment:** DKF120-T02 — Workspace Target Engine +**Implementation branch:** `feature/v0.12-workspace-target-engine` +**Validated implementation head:** `94a10a77d38bdeba57ee398cb533b2d1409afc5c` +**Validation PR:** #63 +**Implementation validation run:** `37668415974` + +## 1. Development Contract + +T02 was executed under: + +```text +Contract: INC-DKF120-T02 +Task: DKF120-T02 +Schema: 1.3.0 +Risk: 2 +Stale: false +``` + +Observed source fingerprint: + +```text +sha256:80af00ee728b98a0055e9971272d8fc3d6f436c32a6a040109f4e5ffde27bbf2 +``` + +Bound acceptance criteria: + +- `DKF-120-AC-004` +- `DKF-120-AC-005` +- `DKF-120-AC-006` +- `DKF-120-AC-007` +- `DKF-120-AC-039` + +Derived gates: + +```text +verification: + specification + tests + +reviewers: + code-reviewer + +control domains: + none + +human approvals: + none + +configuration readiness: + not required +``` + +T02's own contract has no target binding because it bootstraps the target engine itself. Target-aware contracts created after this increment can bind the workspace registry normally. + +## 2. Implemented capability + +T02 adds: + +- project-local Workspace Target Registry; +- arbitrary project-defined target IDs; +- single-target and multi-target workspace support; +- generic repository-boundary discovery; +- direct target dependencies; +- reverse dependants; +- dependency closure; +- affected-target closure; +- verification-target closure; +- generic target commands; +- project-defined target capabilities; +- optional opaque adapter metadata; +- capability/configuration-driven target routing; +- primary/affected/verification Development Contract bindings; +- workspace-registry fingerprint binding; +- workspace-registry staleness enforcement; +- Development Contract schema v1.3.0. + +Canonical workspace configuration: + +```text +.development-kit/workspace.json +``` + +## 3. Acceptance-criterion evidence + +### DKF-120-AC-004 + +**PASS.** + +Arbitrary valid target IDs are accepted and new target IDs require no DKF core modifications. + +### DKF-120-AC-005 + +**PASS.** + +Target discovery recognises generic repository boundaries but does not assign application/domain meaning to them. + +Single-root projects are supported without requiring monorepo structure. + +### DKF-120-AC-006 + +**PASS.** + +Development Contracts can bind: + +- primary target; +- affected targets; +- verification targets; +- the exact registry fingerprint. + +A registry change makes the target-aware contract stale. + +### DKF-120-AC-007 + +**PASS.** + +Target-specific commands resolve from target configuration. + +There are no built-in mandatory target names. + +### DKF-120-AC-039 + +**PASS.** + +Routing uses declared capabilities, commands, optional preferred target IDs, and project-owned adapter metadata. + +It does not dispatch on target kind, technology name, application name, or business domain. + +## 4. Authority and staleness + +A target-aware Development Contract stores the exact fingerprint of the workspace registry used to resolve its target binding. + +If the registry later changes, normal contract staleness detects that drift. + +Final hardening evidence proves registry drift reaches the existing deterministic acceptance engine and changes the decision to: + +```text +BLOCKED + reason: STALE_CONTRACT +``` + +No parallel acceptance authority was introduced. + +## 5. Focused verification + +Final focused suite on the validated implementation head: + +```text +tests: 14 +pass: 14 +fail: 0 +``` + +The suite covers: + +- arbitrary IDs; +- generic discovery; +- single-root fallback; +- graph closures; +- unknown dependency rejection; +- cycle rejection; +- capability-driven routing; +- adapter pass-through; +- path escape rejection; +- registry persistence; +- optimistic fingerprint protection; +- target-bound Development Contracts; +- target registry staleness; +- stale target authority blocking acceptance; +- project registry routing; +- schema alignment. + +The focused T02 gate passed on both Ubuntu and Windows. + +## 6. Full regression result + +GitHub Actions run `37668415974` passed on: + +- Ubuntu; +- Windows. + +The validated implementation head passed: + +- skills and agents; +- plugin synchronization; +- installer integrity; +- Development Contract foundation; +- execution safety; +- evidence/control coverage; +- core orchestration; +- orchestration integration; +- v0.9 reliability regressions; +- documentation validation; +- OpenCode/platform adapters; +- research contract; +- v0.7.1 regressions; +- Live UI Preview; +- token/context efficiency; +- Phase 0/1 baseline; +- Phase 2 PLAN validation; +- T01 Cost Observatory; +- T02 Workspace Target Engine; +- Design Authority; +- Autopilot/evaluations; +- exact `release:validate`. + +## 7. Correction history + +The verification trail preserves: + +1. stale hard-coded Development Contract schema version in the orchestration contract regression test; +2. missing documentation reference for the new contract-generator script; +3. stale hard-coded Development Contract schema version in Configuration Readiness integration testing; +4. explicit verification hardening proving target-registry drift blocks normal deterministic acceptance. + +See: + +`dkf-workspace-target-engine-t02-review.md` + +## 8. General-purpose framework check + +**PASS.** + +The T02 runtime contains no application-specific business policy and no industry-specific target type. + +Examples of repository markers are used only for boundary discovery. They do not determine target semantics or routing. + +Target names, kinds, capabilities, commands, dependency relationships, and adapter metadata are supplied by the project. + +The governing rule remains: + +> **DKF core provides generic mechanisms. Projects and adapters provide domain-specific policy.** + +## 9. Backward compatibility + +**PASS.** + +Existing projects are not required to define a workspace registry for legacy non-target-aware contracts. + +Single-root projects remain first-class. + +Existing supported Development Contract schema versions remain readable. + +The current generated contract schema advances to **1.3.0** for optional Workspace Target bindings. + +## 10. Acceptance decision + +**DKF120-T02 — ACCEPTED FOR PROGRESSION** + +This authorises progression to the next validated v0.12 PLAN increment only. + +It does not: + +- publish v0.12.0; +- bump the package version; +- merge to `main`; +- create a release/tag; +- authorise T04 or later work before T03 is completed and accepted. + +## 11. Next increment + +```text +DKF120-T03 — Execution Capsules & Context Cache +``` + +T03 must receive its own Development Contract before production implementation. + +Because Workspace Target Engine is now available, T03 should use a target-aware Development Contract where its scope is target-specific. diff --git a/docs/04-architecture/dkf-scale-context-iteration-phase05-t03-validation.md b/docs/04-architecture/dkf-scale-context-iteration-phase05-t03-validation.md new file mode 100644 index 00000000..49fe8e34 --- /dev/null +++ b/docs/04-architecture/dkf-scale-context-iteration-phase05-t03-validation.md @@ -0,0 +1,309 @@ +# Development Kit v0.12.0 — Phase 5 / DKF120-T03 Validation + +**Status:** COMPLETE — ACCEPTED FOR PROGRESSION +**Date:** 7 October 2026 +**Increment:** DKF120-T03 — Execution Capsules & Context Cache +**Implementation branch:** `feature/v0.12-execution-capsules-context-cache` +**Validated implementation head:** `765a64a577238d73169fc9ae91bb8697a76ded6d` +**Validation PR:** #65 +**Implementation validation run:** `37672806212` + +## 1. Development Contract + +Observed T03 contract: + +```text +Contract: INC-DKF120-T03 +Task: DKF120-T03 +Schema: 1.3.0 +Risk: 3 +Stale: false +``` + +Source fingerprint: + +```text +sha256:db93ea4d721808aa351bfedffb17d5b65ad6defca34b8322089a18d742602baf +``` + +Workspace registry fingerprint: + +```text +sha256:b5c3d286ab2b1b65c68e9ac17fbb6f45698986449ffb2129fa4666eff60f9c3e +``` + +Target binding: + +```text +primary: dkf-framework +affected: dkf-framework +verification: dkf-framework +``` + +Acceptance criteria: + +- `DKF-120-AC-008` +- `DKF-120-AC-009` +- `DKF-120-AC-010` +- `DKF-120-AC-011` +- `DKF-120-AC-035` + +Derived gates: + +```text +verification: + specification + tests + +reviewers: + architecture-reviewer + code-reviewer + security-reviewer + +control domains: + security + +human approvals: + none +``` + +## 2. Implemented capability + +T03 adds: + +- deterministic Execution Capsule schema/runtime; +- project-local repository context cache; +- target-scoped structural repository facts; +- relevant file fingerprints; +- relevant test fingerprints; +- target/dependency delta; +- upstream accepted-task references; +- verified runtime facts with explicit provenance; +- required future gate/profile references; +- deterministic cache-entry fingerprints; +- deterministic capsule fingerprints; +- cache/capsule staleness checks; +- target-aware invalidation; +- persisted configuration-state invalidation; +- capsule-aware `buildContextPackage()`; +- Cost Observatory comparison evidence. + +Canonical acceleration storage: + +```text +.development-kit/cache/context/ +``` + +The cache is disposable and non-authoritative. + +## 3. Acceptance criteria + +### DKF-120-AC-008 — PASS + +Fresh implementation and verification/review contexts can receive deterministic Execution Capsules. + +Identical semantic inputs produce the same capsule identity across cache miss/hit and differing audit timestamps. + +Verifier contexts remain independently rehydrated. + +### DKF-120-AC-009 — PASS + +Capsules become stale when relevant authority/repository state changes. + +Covered invalidators include: + +- Development Contract/source drift; +- workspace authority drift through normal contract staleness; +- relevant file content change; +- relevant test change through the same fingerprint mechanism; +- selected-target structure change; +- persisted configuration-registry change; +- missing or changed cache entry. + +Stale capsules are rejected before role context construction. + +### DKF-120-AC-010 — PASS + +Repository/context caching has deterministic invalidation. + +Verified behaviours: + +- first use: MISS; +- same semantic inputs: HIT; +- relevant file change: MISS; +- selected target structural change: MISS; +- persisted configuration change: MISS; +- corrupted cache: safe MISS/rebuild; +- cache deletion: safe MISS/rebuild; +- unrelated unselected target change: existing target-scoped cache remains usable. + +### DKF-120-AC-011 — PASS + +Existing section-aware authoritative-source materialisation and source-fingerprint protections remain active. + +With a capsule present: + +- authoritative sources are still re-read; +- whole-file source fingerprints are still verified; +- only requested source sections are delivered where selectors resolve; +- stale Development Contracts continue to block context construction. + +### DKF-120-AC-035 — PASS + +Measured comparison evidence is available through the Cost Observatory. + +Two-target fixture: + +```text +logical context bytes: + 9,591 -> 7,053 + reduction: 26.46% + +estimated context tokens: + 2,398 -> 1,764 + reduction: 26.44% + +target-orientation scans: + 2 -> 1 + reduction: 50.00% + +current cache: + hits: 1 + misses: 0 +``` + +These are focused T03 fixture results, not final release-wide v0.12 performance claims. + +## 4. Authority model + +Execution Capsules contain references and fingerprints rather than copied project history. + +The capsule carries: + +- contract/task/run/lifecycle references; +- authoritative source references/selectors/fingerprints; +- workspace target binding; +- repository cache reference/fingerprint; +- compact target facts; +- relevant file/test fingerprints; +- target/dependency delta; +- upstream accepted-task references; +- verified runtime facts with provenance; +- future gate/profile references. + +Runtime facts are explicitly marked: + +```text +non-authoritative-observation +``` + +The capsule cannot override specifications, architecture, Design Authority, Development Contracts, or acceptance evidence. + +## 5. Context-package integration + +`buildContextPackage()` remains the role-context constructor. + +When a capsule is supplied it must pass freshness checks first. + +Context isolation metadata can now distinguish: + +```text +authoritativeSourcesReRead: true +repositoryReRead: false +repositoryContextFromCapsule: true +capsuleFreshnessVerified: true +``` + +This preserves independent source authority while avoiding repeated full repository reorientation. + +## 6. Safety and cache failure behaviour + +The cache is not canonical state. + +If cache state is: + +- absent -> rebuild; +- corrupt -> discard/rebuild; +- stale -> miss/rebuild; +- path-escaping -> reject; +- internally inconsistent -> reject. + +An old capsule that references a missing/stale cache entry is itself stale. + +The system does not silently fall back to trusting the old capsule. + +## 7. Backward compatibility + +Legacy non-target-aware Development Contracts remain supported. + +They receive one generic project-root cache boundary rather than being forced to migrate to named targets before using T03. + +Existing public commands remain unchanged. + +Existing token/context efficiency and verification-isolation regressions remain green. + +## 8. Verification + +Focused T03 result: + +```text +tests: 19 +pass: 19 +fail: 0 +``` + +Implementation validation run `37672806212` passed on: + +- Ubuntu; +- Windows. + +It also passed the exact `release:validate` command on both operating systems. + +## 9. Correction history + +The T03 evidence trail preserves: + +1. semantic cache identity originally included creation time; +2. absent Configuration Readiness state originally fingerprinted a synthetic timestamp, causing false cache misses; +3. cache I/O confinement was hardened; +4. changed files were made automatically relevant to invalidation; +5. cache-entry integrity is now validated directly during capsule creation; +6. explicit cost evidence was added. + +See: + +`dkf-execution-capsule-t03-review.md` + +## 10. Performance interpretation + +T03 proves that context and target-orientation reduction is measurable while preserving mandatory source rehydration and verification isolation. + +The focused fixture does **not** yet meet or claim the final release objectives of: + +- >=40% repeated context-volume reduction; +- >=60% redundant repository-read reduction; +- >=70% repeated full-repository orientation reduction. + +Those remain release engineering objectives to be evaluated on representative workloads after the later v0.12 increments are integrated. + +## 11. Acceptance decision + +**DKF120-T03 — ACCEPTED FOR PROGRESSION** + +This authorises only the next validated PLAN increment. + +It does not: + +- release v0.12.0; +- change package version 0.11.2; +- merge to `main`; +- create a release/tag; +- authorise later increments before their dependencies and Development Contracts are satisfied. + +## 12. Next increment + +```text +DKF120-T04 — State Engine V2 +``` + +T04 must receive its own Development Contract before production implementation. diff --git a/docs/04-architecture/dkf-scale-context-iteration-phase06-t04-validation.md b/docs/04-architecture/dkf-scale-context-iteration-phase06-t04-validation.md new file mode 100644 index 00000000..da7655c5 --- /dev/null +++ b/docs/04-architecture/dkf-scale-context-iteration-phase06-t04-validation.md @@ -0,0 +1,268 @@ +# Development Kit v0.12.0 — Phase 6 / DKF120-T04 Validation + +**Status:** COMPLETE — ACCEPTED FOR PROGRESSION +**Date:** 7 October 2026 +**Increment:** DKF120-T04 — State Engine V2 +**Implementation branch:** `feature/v0.12-state-engine-v2` +**Validated implementation head:** `e3e4b16e372b40042398cfcf210f471e4f762b1b` +**Validation PR:** #67 +**Implementation validation run:** `37678375215` + +## 1. Development Contract + +Observed contract: + +- Contract: `INC-DKF120-T04` +- Task: `DKF120-T04` +- Schema: `1.3.0` +- Risk: `3` +- Stale: `false` +- Source fingerprint: `sha256:f463f9a42340329c2fcf6969559edf48f014ca4159352e983d2cb0c86b2a4729` +- Workspace registry fingerprint: `sha256:b5c3d286ab2b1b65c68e9ac17fbb6f45698986449ffb2129fa4666eff60f9c3e` + +Target binding: + +- primary: `dkf-framework` +- affected: `dkf-framework` +- verification: `dkf-framework` + +Bound acceptance criteria: + +- `DKF-120-AC-012` +- `DKF-120-AC-013` +- `DKF-120-AC-014` +- `DKF-120-AC-015` +- `DKF-120-AC-034` + +Derived gates: + +- verification: migration, specification, tests; +- reviewers: architecture-reviewer, code-reviewer, security-reviewer; +- control domains: security; +- human approvals: none. + +## 2. Implemented capability + +T04 introduces State Engine schema v2 with: + +- append-only logical canonical event history; +- monotonic event sequence; +- deterministic event IDs; +- SHA-256 hash chaining; +- deterministic entity-state patch events; +- materialized current snapshot; +- disposable/rebuildable query index; +- project-local atomic state commits; +- State Engine integrity verification; +- Autopilot fresh-project V2 persistence; +- orchestration-run fresh-project V2 persistence; +- legacy fallback before migration; +- explicit legacy migration; +- migration recovery bundle; +- semantic-equivalence validation; +- verified migration-completion cutover; +- migration idempotence; +- migration recovery after interruption; +- legacy retention through T04. + +Canonical layout: + +- `.development-kit/state/events.jsonl` +- `.development-kit/state/snapshot.json` +- `.development-kit/state/index.db` +- `.development-kit/state/schema.json` +- `.development-kit/state/legacy-backup.json` after legacy migration. + +## 3. Acceptance-criterion evidence + +### DKF-120-AC-012 — PASS + +Canonical runtime history is reconstructable from `events.jsonl`. + +Verified: + +- monotonic sequence; +- linked event hashes; +- mutation detection; +- reordering detection; +- missing legacy-history detection during migration; +- canonical-tail truncation rejection when the materialized snapshot witnesses later history; +- deterministic event replay. + +### DKF-120-AC-013 — PASS + +`snapshot.json` is a materialized view. + +Verified: + +- deleting snapshot -> rebuild from events; +- invalid/corrupt snapshot JSON -> rebuild from events; +- stale-behind snapshot -> rebuild forward; +- snapshot witnessing history ahead of canonical ledger -> fail closed rather than downgrade. + +### DKF-120-AC-014 — PASS + +`index.db` is disposable. + +Verified: + +- delete index; +- rebuild from current snapshot/canonical state; +- query results before and after rebuild are semantically equivalent; +- integrity verification passes after rebuild. + +The current `index.db` format is a portable deterministic JSON acceleration artifact rather than SQLite, preserving DKF Node >=18 compatibility. It is not canonical authority. + +### DKF-120-AC-015 — PASS + +Supported legacy migration is: + +- validated before cutover; +- semantic-equivalence checked; +- idempotent; +- resumable after partial import; +- fail closed on corrupt source; +- fail closed on missing revisions; +- fail closed on manifest/revision anchor mismatch; +- recoverable from bundled backup; +- safe after legitimate post-migration V2 progress. + +Existing legacy projects do not switch to V2 merely because partial V2 entities exist. + +Cutover requires a verified `MIGRATION_COMPLETED` record. + +### DKF-120-AC-034 — PASS with explicit retention disclosure + +Measured representative legacy transition shape: + +- legacy active transition-state files: **27** +- T04 active State Engine canonical/recovery files: **5** +- measured reduction: **81.48%** + +This exceeds the >=80% transition-generated state micro-file engineering objective. + +T04 deliberately retains the original 27 legacy source files during the recovery window. Consequently total physical files during that window are 32. The reduction claim applies to the new active canonical/recovery representation, not temporary retained recovery originals. + +## 4. Fresh-project behavior + +Fresh projects do not need a migration. + +Autopilot and orchestration transition state use V2 immediately. + +Fresh Autopilot state no longer creates one revision JSON file per transition. + +Fresh orchestration runs retain immutable `manifest.json` as an artifact, while transition state moves to the State Engine. + +## 5. Existing-project behavior + +If legacy state exists and no verified migration completion exists, the legacy store remains canonical. + +After successful migration, a `MIGRATION_COMPLETED` record with `semanticEquivalence=true` authorizes V2 as canonical runtime transition state. + +Legacy files remain available for recovery. + +## 6. Acceptance meaning + +An explicit legacy orchestration fixture with: + +- `state=ACCEPTED` +- `acceptanceState=ACCEPTED` +- `verificationVerdict=PASS` + +retains those same values after migration. + +T04 does not alter acceptance rules or create a second acceptance authority. + +## 7. Crash and integrity behavior + +Verified failure behavior includes: + +- orphan temporary write -> ignored/cleanable; +- missing snapshot -> rebuild; +- corrupt snapshot -> rebuild; +- missing index -> rebuild; +- canonical mutation/reordering -> reject; +- witnessed canonical tail truncation -> reject; +- state-root symlink/junction escape -> reject; +- schema tampering -> reject; +- incomplete V2 layout -> reject; +- corrupt legacy input -> migration rejected before cutover; +- partial migration -> no cutover. + +## 8. Backward compatibility + +**PASS** + +Existing unmigrated projects can continue using their legacy state store. + +Fresh projects receive V2 automatically. + +Public `/dk-*` command surfaces are unchanged. + +The historical Phase 1 baseline remains historically stable by constructing the old persistence fixture directly rather than invoking the current backend. + +## 9. Verification result + +Focused T04 suite: + +- tests: **22** +- pass: **22** +- fail: **0** + +Full implementation CI run `37678375215`: + +- Ubuntu: PASS; +- Windows: PASS; +- exact `release:validate`: PASS on both. + +The matrix also confirms T01, T02, T03, historical v0.7.1, v0.9 reliability, token/context efficiency, Design Authority, Autopilot/evaluations, documentation, installer, plugin synchronization, and other existing gates remain green. + +## 10. Review + +Required risk-3 review: + +- code-reviewer: PASS; +- architecture-reviewer: PASS; +- security-reviewer: PASS; +- security control domain: PASS. + +See `docs/04-architecture/dkf-state-engine-v2-t04-review.md`. + +Unresolved findings: + +- Critical: 0 +- Major: 0 +- Minor: 0 + +## 11. General-purpose framework check + +**PASS** + +State Engine core contains no application-specific or industry-specific business semantics. + +It persists generic DKF development/runtime entities and references. + +The governing rule remains: + +> **DKF core provides generic development-control mechanisms. Projects and adapters provide domain-specific policy.** + +## 12. Acceptance decision + +**DKF120-T04 — ACCEPTED FOR PROGRESSION** + +This authorises only the next validated PLAN increment. + +It does not: + +- release v0.12.0; +- change package version 0.11.2; +- merge to `main`; +- delete retained legacy state; +- publish/tag a release; +- authorise later increments without their own Development Contracts. + +## 13. Next increment + +`DKF120-T05 — Lifecycle Instances` + +T05 must receive its own Development Contract before production implementation. diff --git a/docs/04-architecture/dkf-scale-context-iteration-v0.12-implementation-plan.md b/docs/04-architecture/dkf-scale-context-iteration-v0.12-implementation-plan.md new file mode 100644 index 00000000..87e21340 --- /dev/null +++ b/docs/04-architecture/dkf-scale-context-iteration-v0.12-implementation-plan.md @@ -0,0 +1,598 @@ +# Development Kit v0.12.0 — Scale, Context & Iteration Implementation PLAN + +**Status:** Phase 2 implementation PLAN +**Specification:** `dkf-scale-context-iteration-v0.12-specification.md` +**Technical design:** `dkf-scale-context-iteration-v0.12-technical-design.md` +**Deterministic model:** `dkf-scale-context-iteration-v0.12-plan-model.json` + +## 1. Execution rule + +Implementation proceeds in the following order: + +```text +DKF120-T01 Cost Observatory + -> DKF120-T02 Workspace Target Engine + -> DKF120-T03 Execution Capsules & Context Cache + -> DKF120-T04 State Engine V2 + -> DKF120-T05 Lifecycle Instances + -> DKF120-T06 Discovery & Controlled Re-entry + -> DKF120-T07 Generic Gate Profiles + -> DKF120-T08 Verification Extension SDK + -> DKF120-T09 Complexity Delta Guard + -> DKF120-T10 Control Center + Comparative Release Validation +``` + +Every task receives its own Development Contract during implementation. + +No task may begin before its dependency reaches deterministic acceptance unless the PLAN is explicitly revalidated after an approved amendment. + +## 2. Global implementation constraints + +All tasks must comply with: + +- general-purpose DKF core; +- no application-specific or industry-specific business logic; +- arbitrary project-defined target IDs; +- host/provider independence; +- reuse-before-create; +- existing `/dk-*` command compatibility; +- independent verification; +- source fingerprint/staleness protection; +- deterministic acceptance; +- human approval preservation; +- Design Authority preservation; +- no fabricated telemetry; +- no SQLite-only canonical state. + +## 3. Task DKF120-T01 — Cost Observatory + +### Objective + +Add generic, truthful orchestration-cost measurement before optimising later components. + +### Dependencies + +None. + +### Risk + +Medium. Instrumentation crosses orchestration boundaries but must not alter acceptance decisions. + +### Owned resources + +- `cost-observatory-runtime` +- `cost-telemetry-schema` +- `cost-baseline-comparison` + +### Reuse targets + +Inspect and extend, where appropriate: + +- `runtime/orchestration/token-efficiency.mjs` +- context-package token profiles; +- current run/contract identifiers; +- existing evidence/runtime APIs; +- Phase 0/1 baseline runner. + +### Requirements + +- record DKF-controlled measurable counts; +- accept optional provider/host token telemetry; +- keep unavailable metrics null; +- add canonical newline-normalised logical text/context comparison; +- preserve the existing historical static token audit; +- enable before/after workload comparison. + +### Exclusions + +- no context caching yet; +- no target engine; +- no provider-specific mandatory SDK; +- no optimisation claims before measurement. + +### Acceptance criteria + +- DKF-120-AC-001 +- DKF-120-AC-002 +- DKF-120-AC-003 +- DKF-120-AC-033 + +### Required verification + +- unit tests for measurement record validation; +- cross-platform newline-normalisation test; +- unavailable-provider-metric test; +- regression test proving acceptance semantics are unchanged. + +--- + +## 4. Task DKF120-T02 — Workspace Target Engine + +### Objective + +Introduce generic project-defined target boundaries and capability-driven target routing. + +### Dependencies + +DKF120-T01. + +### Risk + +Medium. + +### Owned resources + +- `workspace-target-registry` +- `target-resolution` +- `target-routing` + +### Requirements + +- support arbitrary target IDs; +- support single-target and multi-target repositories; +- discover common repository boundaries without privileging them in core; +- support target dependency relationships; +- bind tasks/contracts to primary, affected, and verification targets; +- resolve target commands/capabilities from project configuration/adapters. + +### Exclusions + +- no hard-coded `web`, `mobile`, `database`, `calculations`, or other mandatory target names; +- no requirement for monorepos; +- no domain/business inference from target names. + +### Acceptance criteria + +- DKF-120-AC-004 +- DKF-120-AC-005 +- DKF-120-AC-006 +- DKF-120-AC-007 +- DKF-120-AC-039 + +### Required verification + +- single-root repository fixture; +- arbitrary-name multi-target fixture; +- target dependency closure tests; +- target cycle/error tests; +- capability-driven command routing tests. + +--- + +## 5. Task DKF120-T03 — Execution Capsules & Context Cache + +### Objective + +Reduce repeated repository/runtime reorientation while preserving current source authority and fresh-role isolation. + +### Dependencies + +DKF120-T02. + +### Risk + +High. Incorrect caching could feed stale context into verification. + +### Owned resources + +- `execution-capsule` +- `repository-context-cache` +- `context-invalidation` + +### Requirements + +- build deterministic Execution Capsules; +- reuse current source section materialisation; +- reference source/file fingerprints rather than duplicate whole history; +- persist/cache repository structural facts only with deterministic invalidation; +- include target/dependency delta and verified runtime facts; +- reject stale capsules. + +### Exclusions + +- do not replace `buildContextPackage()`; +- do not weaken whole-file fingerprint checks; +- do not treat summaries as authority. + +### Acceptance criteria + +- DKF-120-AC-008 +- DKF-120-AC-009 +- DKF-120-AC-010 +- DKF-120-AC-011 +- DKF-120-AC-035 + +### Required verification + +- stale source invalidates capsule; +- relevant file change invalidates capsule; +- irrelevant target change does not force unrelated full rescan when safe; +- full cache miss still produces correct context; +- current token-efficiency and verification-isolation tests remain green. + +--- + +## 6. Task DKF120-T04 — State Engine V2 + +### Objective + +Replace transition-proportional canonical snapshot chains with compact event-backed state while preserving auditability and recovery. + +### Dependencies + +DKF120-T03. + +### Risk + +High. Persistent-state migration and recovery are control-plane critical. + +### Owned resources + +- `canonical-event-ledger` +- `state-snapshot` +- `state-index` +- `legacy-state-migration` + +### Requirements + +- append-only canonical event history; +- materialised current snapshot; +- rebuildable acceleration index; +- atomic state mutation; +- semantic-equivalence migration from supported legacy state; +- migration idempotence; +- legacy recovery retained until verified cutover; +- index/snapshot rebuild tests. + +### Exclusions + +- no binary-only source of truth; +- no deletion of legacy state before verified migration; +- no migration that changes acceptance meaning. + +### Acceptance criteria + +- DKF-120-AC-012 +- DKF-120-AC-013 +- DKF-120-AC-014 +- DKF-120-AC-015 +- DKF-120-AC-034 + +### Required verification + +- event ordering/integrity; +- interrupted write; +- snapshot rebuild; +- index deletion/rebuild; +- migration/repeated migration; +- corrupted legacy input; +- semantic equivalence; +- baseline state-file-count comparison. + +--- + +## 7. Task DKF120-T05 — Lifecycle Instances + +### Objective + +Make bounded/nested development lifecycles first-class without changing the familiar nine-stage user workflow. + +### Dependencies + +DKF120-T04. + +### Risk + +High. Lifecycle completion semantics affect orchestration authority. + +### Owned resources + +- `lifecycle-instance-model` +- `lifecycle-completion-semantics` +- `state-namespaces` + +### Requirements + +- optional parent/child lifecycle relationships; +- each lifecycle retains canonical nine stages; +- `COMPLETE` scoped to one lifecycle; +- separate application/domain/release/deployment state from development lifecycle state; +- existing simple projects behave as one lifecycle. + +### Exclusions + +- no app-specific lifecycle types; +- no automatic business-status-to-DKF-completion mapping. + +### Acceptance criteria + +- DKF-120-AC-016 +- DKF-120-AC-017 +- DKF-120-AC-018 + +### Required verification + +- child complete while parent remains active; +- parent acceptance independently evaluated; +- arbitrary domain state cannot complete lifecycle; +- legacy single-lifecycle compatibility. + +--- + +## 8. Task DKF120-T06 — Discovery & Controlled Re-entry + +### Objective + +Turn technical discovery into a controlled, evidence-backed feedback loop. + +### Dependencies + +DKF120-T05. + +### Risk + +High. Re-entry changes previously accepted planning/authority state. + +### Owned resources + +- `discovery-event-model` +- `reentry-resolver` +- `selective-invalidation` + +### Requirements + +- persist discovery evidence and affected authority; +- classify the type/scope of invalidation; +- determine minimum safe re-entry stage; +- calculate dependent affected work; +- preserve unrelated accepted work; +- invoke existing human approval when upstream authority changes require it. + +### Exclusions + +- no silent specification rewrite from IMPLEMENT; +- no automatic global lifecycle restart unless dependency evidence requires it. + +### Acceptance criteria + +- DKF-120-AC-019 +- DKF-120-AC-020 +- DKF-120-AC-021 + +### Required verification + +- implementation-only defect -> IMPLEMENT; +- plan invalidation -> PLAN; +- architecture invalidation -> DESIGN; +- specification invalidation -> DEFINE; +- selective invalidation fixture; +- human-gated upstream amendment fixture. + +--- + +## 9. Task DKF120-T07 — Generic Gate Profiles + +### Objective + +Extend current gate selection into composable, project-neutral verification profiles. + +### Dependencies + +DKF120-T06. + +### Risk + +High. Gate resolution must never reduce mandatory control coverage. + +### Owned resources + +- `gate-profile-contract` +- `gate-profile-resolver` + +### Requirements + +- extend current `gate-selector` behaviour; +- resolve profiles from contract, risk, target capabilities, and project configuration; +- permit project extensions to add verification classes; +- preserve mandatory security/design/configuration/human gates; +- fail closed on unresolved required profiles. + +### Exclusions + +- no application-specific gate names or business rules in core; +- no profile that suppresses existing mandatory gates. + +### Acceptance criteria + +- DKF-120-AC-022 +- DKF-120-AC-023 + +### Required verification + +- low/high risk resolution; +- target capability resolution; +- attempted mandatory-gate downgrade rejected; +- unknown required profile fails closed. + +--- + +## 10. Task DKF120-T08 — Verification Extension SDK + +### Objective + +Provide generic primitives through which any project can add rigorous project/domain verification without modifying DKF core. + +### Dependencies + +DKF120-T07. + +### Risk + +High. Extensions may participate in acceptance and therefore require strict evidence semantics. + +### Owned resources + +- `verification-extension-contract` +- `generic-verification-primitives` +- `extension-fixtures` + +### Requirements + +- project-owned extension registration/contract; +- generic absolute/relative tolerance primitive; +- generic ranges/invariants/custom deterministic verifier support; +- reference/golden dataset fingerprinting; +- optional unit/dimension adapter interface; +- replay/reference verification; +- results flow through existing evidence/trust/gate semantics; +- at least two materially different example extensions prove generality. + +### Exclusions + +- no solar/energy formulas in DKF core; +- no financial/accounting rules in DKF core; +- no healthcare/legal/CAD/ecommerce rules in DKF core; +- example fixtures are not built-in policy. + +### Acceptance criteria + +- DKF-120-AC-024 +- DKF-120-AC-025 +- DKF-120-AC-026 +- DKF-120-AC-027 +- DKF-120-AC-037 +- DKF-120-AC-038 + +### Required verification + +- tolerance boundary tests; +- invariant fail blocks normal gate; +- stale reference dataset invalidates evidence; +- two different extension fixtures run without core changes; +- adversarial extension cannot self-certify or bypass mandatory gates. + +--- + +## 11. Task DKF120-T09 — Complexity Delta Guard + +### Objective + +Add cheap deterministic complexity signals to avoid unnecessary heavy review while retaining full simplicity governance. + +### Dependencies + +DKF120-T08. + +### Risk + +Medium. + +### Owned resources + +- `complexity-delta` +- `simplicity-trigger-policy` + +### Requirements + +- derive deterministic complexity facts from repository/diff information; +- define policy threshold/conditions; +- trigger existing simplicity review where required; +- never remove a currently mandatory simplicity review. + +### Exclusions + +- no LLM-only complexity score represented as deterministic fact; +- no replacement of simplicity reviewer. + +### Acceptance criteria + +- DKF-120-AC-028 +- DKF-120-AC-029 + +### Required verification + +- identical diff produces identical delta; +- low-complexity case avoids optional heavy invocation; +- policy/risk case still invokes required simplicity review. + +--- + +## 12. Task DKF120-T10 — Control Center + Comparative Release Validation + +### Objective + +Expose the new control-plane state, replay the Phase 1 workloads, and prepare evidence for the v0.12.0 release gate. + +### Dependencies + +DKF120-T09. + +### Risk + +High because it closes the release and compatibility loop. + +### Owned resources + +- `control-center-integration` +- `comparative-replay` +- `release-docs-evidence` + +### Requirements + +- extend existing Control Center/runtime API; +- expose lifecycle/targets/capsules/cost/discoveries/gates/state health; +- keep UI non-authoritative; +- replay Phase 1 single-package, multi-target, calculation-heavy/general verification, and legacy-state workloads; +- compare actual results against v0.12 objectives; +- run complete regression/release suites; +- document migration, compatibility, limitations, and measured outcomes. + +### Exclusions + +- no new competing Control Center; +- no release/publish action without explicit release approval. + +### Acceptance criteria + +- DKF-120-AC-030 +- DKF-120-AC-031 +- DKF-120-AC-032 +- DKF-120-AC-036 +- DKF-120-AC-040 + +### Required verification + +- Control Center reads canonical APIs; +- all existing supported commands remain discoverable/functional; +- full release validation on supported CI; +- false-acceptance adversarial suite; +- before/after Phase 1 replay; +- generic-core review proving no project-specific logic entered core. + +## 13. Deterministic PLAN invariants + +The machine-readable plan must prove: + +- exactly 10 stable task IDs; +- every dependency points to a declared task; +- no dependency cycle; +- declared dependency diagram equals computed dependency graph; +- every required logical resource has exactly one owner; +- every `DKF-120-AC-001` through `DKF-120-AC-040` criterion has at least one task owner; +- no unknown configuration dependencies are introduced at planning time. + +The authoritative deterministic model is: + +`docs/04-architecture/dkf-scale-context-iteration-v0.12-plan-model.json` + +## 14. Implementation boundary + +Phase 2 authorises planning only. + +Production implementation begins only after: + +1. the deterministic PLAN validator returns `valid: true`; +2. repository CI remains green; +3. the specification, design, PLAN, and project memory are updated consistently. + +The first production Development Contract will then be **DKF120-T01 — Cost Observatory**. diff --git a/docs/04-architecture/dkf-scale-context-iteration-v0.12-plan-model.json b/docs/04-architecture/dkf-scale-context-iteration-v0.12-plan-model.json new file mode 100644 index 00000000..4d532b1b --- /dev/null +++ b/docs/04-architecture/dkf-scale-context-iteration-v0.12-plan-model.json @@ -0,0 +1,268 @@ +{ + "schemaVersion": "1.0.0", + "planId": "DKF-V012-SCALE-CONTEXT-ITERATION", + "declaredTaskCount": 10, + "declaredDependencyEdges": [ + "DKF120-T01->DKF120-T02", + "DKF120-T02->DKF120-T03", + "DKF120-T03->DKF120-T04", + "DKF120-T04->DKF120-T05", + "DKF120-T05->DKF120-T06", + "DKF120-T06->DKF120-T07", + "DKF120-T07->DKF120-T08", + "DKF120-T08->DKF120-T09", + "DKF120-T09->DKF120-T10" + ], + "requiredResources": [ + "cost-observatory-runtime", + "cost-telemetry-schema", + "cost-baseline-comparison", + "workspace-target-registry", + "target-resolution", + "target-routing", + "execution-capsule", + "repository-context-cache", + "context-invalidation", + "canonical-event-ledger", + "state-snapshot", + "state-index", + "legacy-state-migration", + "lifecycle-instance-model", + "lifecycle-completion-semantics", + "state-namespaces", + "discovery-event-model", + "reentry-resolver", + "selective-invalidation", + "gate-profile-contract", + "gate-profile-resolver", + "verification-extension-contract", + "generic-verification-primitives", + "extension-fixtures", + "complexity-delta", + "simplicity-trigger-policy", + "control-center-integration", + "comparative-replay", + "release-docs-evidence" + ], + "requiredAcceptanceCriteria": [ + "DKF-120-AC-001", + "DKF-120-AC-002", + "DKF-120-AC-003", + "DKF-120-AC-004", + "DKF-120-AC-005", + "DKF-120-AC-006", + "DKF-120-AC-007", + "DKF-120-AC-008", + "DKF-120-AC-009", + "DKF-120-AC-010", + "DKF-120-AC-011", + "DKF-120-AC-012", + "DKF-120-AC-013", + "DKF-120-AC-014", + "DKF-120-AC-015", + "DKF-120-AC-016", + "DKF-120-AC-017", + "DKF-120-AC-018", + "DKF-120-AC-019", + "DKF-120-AC-020", + "DKF-120-AC-021", + "DKF-120-AC-022", + "DKF-120-AC-023", + "DKF-120-AC-024", + "DKF-120-AC-025", + "DKF-120-AC-026", + "DKF-120-AC-027", + "DKF-120-AC-028", + "DKF-120-AC-029", + "DKF-120-AC-030", + "DKF-120-AC-031", + "DKF-120-AC-032", + "DKF-120-AC-033", + "DKF-120-AC-034", + "DKF-120-AC-035", + "DKF-120-AC-036", + "DKF-120-AC-037", + "DKF-120-AC-038", + "DKF-120-AC-039", + "DKF-120-AC-040" + ], + "tasks": [ + { + "id": "DKF120-T01", + "dependsOn": [], + "owns": [ + "cost-observatory-runtime", + "cost-telemetry-schema", + "cost-baseline-comparison" + ], + "acceptanceCriteria": [ + "DKF-120-AC-001", + "DKF-120-AC-002", + "DKF-120-AC-003", + "DKF-120-AC-033" + ], + "configurationDependencies": [] + }, + { + "id": "DKF120-T02", + "dependsOn": [ + "DKF120-T01" + ], + "owns": [ + "workspace-target-registry", + "target-resolution", + "target-routing" + ], + "acceptanceCriteria": [ + "DKF-120-AC-004", + "DKF-120-AC-005", + "DKF-120-AC-006", + "DKF-120-AC-007", + "DKF-120-AC-039" + ], + "configurationDependencies": [] + }, + { + "id": "DKF120-T03", + "dependsOn": [ + "DKF120-T02" + ], + "owns": [ + "execution-capsule", + "repository-context-cache", + "context-invalidation" + ], + "acceptanceCriteria": [ + "DKF-120-AC-008", + "DKF-120-AC-009", + "DKF-120-AC-010", + "DKF-120-AC-011", + "DKF-120-AC-035" + ], + "configurationDependencies": [] + }, + { + "id": "DKF120-T04", + "dependsOn": [ + "DKF120-T03" + ], + "owns": [ + "canonical-event-ledger", + "state-snapshot", + "state-index", + "legacy-state-migration" + ], + "acceptanceCriteria": [ + "DKF-120-AC-012", + "DKF-120-AC-013", + "DKF-120-AC-014", + "DKF-120-AC-015", + "DKF-120-AC-034" + ], + "configurationDependencies": [] + }, + { + "id": "DKF120-T05", + "dependsOn": [ + "DKF120-T04" + ], + "owns": [ + "lifecycle-instance-model", + "lifecycle-completion-semantics", + "state-namespaces" + ], + "acceptanceCriteria": [ + "DKF-120-AC-016", + "DKF-120-AC-017", + "DKF-120-AC-018" + ], + "configurationDependencies": [] + }, + { + "id": "DKF120-T06", + "dependsOn": [ + "DKF120-T05" + ], + "owns": [ + "discovery-event-model", + "reentry-resolver", + "selective-invalidation" + ], + "acceptanceCriteria": [ + "DKF-120-AC-019", + "DKF-120-AC-020", + "DKF-120-AC-021" + ], + "configurationDependencies": [] + }, + { + "id": "DKF120-T07", + "dependsOn": [ + "DKF120-T06" + ], + "owns": [ + "gate-profile-contract", + "gate-profile-resolver" + ], + "acceptanceCriteria": [ + "DKF-120-AC-022", + "DKF-120-AC-023" + ], + "configurationDependencies": [] + }, + { + "id": "DKF120-T08", + "dependsOn": [ + "DKF120-T07" + ], + "owns": [ + "verification-extension-contract", + "generic-verification-primitives", + "extension-fixtures" + ], + "acceptanceCriteria": [ + "DKF-120-AC-024", + "DKF-120-AC-025", + "DKF-120-AC-026", + "DKF-120-AC-027", + "DKF-120-AC-037", + "DKF-120-AC-038" + ], + "configurationDependencies": [] + }, + { + "id": "DKF120-T09", + "dependsOn": [ + "DKF120-T08" + ], + "owns": [ + "complexity-delta", + "simplicity-trigger-policy" + ], + "acceptanceCriteria": [ + "DKF-120-AC-028", + "DKF-120-AC-029" + ], + "configurationDependencies": [] + }, + { + "id": "DKF120-T10", + "dependsOn": [ + "DKF120-T09" + ], + "owns": [ + "control-center-integration", + "comparative-replay", + "release-docs-evidence" + ], + "acceptanceCriteria": [ + "DKF-120-AC-030", + "DKF-120-AC-031", + "DKF-120-AC-032", + "DKF-120-AC-036", + "DKF-120-AC-040" + ], + "configurationDependencies": [] + } + ] +} diff --git a/docs/04-architecture/dkf-scale-context-iteration-v0.12-specification.md b/docs/04-architecture/dkf-scale-context-iteration-v0.12-specification.md new file mode 100644 index 00000000..71feedfe --- /dev/null +++ b/docs/04-architecture/dkf-scale-context-iteration-v0.12-specification.md @@ -0,0 +1,360 @@ +# Development Kit v0.12.0 — Scale, Context & Iteration Specification + +**Status:** Phase 2 reconciled specification +**Date:** 7 October 2026 +**Target release:** v0.12.0 +**Baseline:** v0.11.2 at `62badbf7d0abee3344b1f5db2b2fbc876cf004e2` +**Evidence:** `dkf-scale-context-iteration-phase01-baseline.md` + +## 1. Product decision + +Development Kit will remain a **general-purpose, host-independent software-development reliability and orchestration framework**. + +v0.12.0 must improve large-project scale, iterative engineering, multi-target workspaces, context economy, state efficiency, and extensible verification without making DKF specific to any application, industry, programming stack, repository shape, or business domain. + +The governing rule is: + +> **DKF core provides generic mechanisms. Projects and adapters provide domain-specific policy.** + +Examples such as web/mobile/database/calculation targets or engineering/numerical verification are test cases only. They are not privileged concepts in DKF core. + +## 2. Existing capabilities that must be extended, not rebuilt + +Phase 0/1 proved that v0.11.2 already provides: + +- Development Contracts and source fingerprints; +- independent verification contexts; +- section-aware authoritative-source materialisation; +- context token profiles and static token budgets; +- fail-safe full-source fallback; +- deterministic acceptance; +- risk/reviewer gate selection; +- Design Authority; +- configuration readiness; +- Live UI Preview; +- immutable Autopilot and orchestration-run state revisions. + +v0.12.0 must reuse these mechanisms wherever possible. + +## 3. Non-negotiable invariants + +### INV-120-001 — General-purpose core +No DKF core module may contain application-specific business rules or assume a particular industry. + +### INV-120-002 — Arbitrary project targets +Workspace target IDs are project-defined. Names such as `web`, `mobile`, `database`, or `calculations` are examples only. + +### INV-120-003 — Extensible verification +DKF may supply generic verification primitives and profile contracts, but project/domain rules are supplied through configuration, adapters, fixtures, or project-owned extensions. + +### INV-120-004 — Reliability preserved +Efficiency changes may not weaken Development Contracts, source staleness, independent verification, deterministic acceptance, Design Authority, security controls, or human approval gates. + +### INV-120-005 — Host independence +The design may not require one AI provider, model, IDE, operating system, package manager, or execution host. + +### INV-120-006 — Backward compatibility +Existing supported projects and public `/dk-*` commands must continue to work, subject only to documented migration where persistent-state format changes. + +### INV-120-007 — Evidence over claims +Measured cost/performance fields must remain unavailable when the host cannot supply them. DKF must not fabricate provider-token or tool-call counts. + +### INV-120-008 — Transparent state authority +A binary index may accelerate state queries but may not become the sole source of truth. + +## 4. Capability scope + +### 4.1 Cost Observatory + +Add orchestration-cost telemetry that can record, when observable: + +- context bytes and deterministic estimated tokens; +- provider-reported input/output/cached tokens; +- repository file reads; +- repository orientation/scans; +- agent invocations; +- tool calls; +- verification commands; +- cache hits/misses; +- elapsed operation timing. + +New cross-platform comparisons must normalise line endings for logical text-size metrics. + +Existing static token regression metrics remain available for historical compatibility. + +### 4.2 Workspace Target Engine + +Introduce a generic project target registry. + +A target describes a bounded execution unit such as a package, application, service, library, database area, infrastructure component, firmware module, documentation site, or project-defined unit. + +A target may declare: + +- arbitrary target ID; +- path/root; +- optional descriptive kind; +- dependencies on other targets; +- commands/capabilities; +- test/verification entry points; +- optional adapter metadata. + +Core routing must depend on target capabilities and configuration, not hard-coded target names. + +### 4.3 Execution Capsules and context cache + +Build on the current `buildContextPackage()` and section-aware materialisation. + +An Execution Capsule contains deterministic references to: + +- Development Contract; +- authoritative sources and fingerprints; +- relevant target(s); +- direct dependency state; +- relevant repository files/tests; +- runtime facts with provenance; +- recent relevant deltas; +- required gate/profile information. + +Repository/context knowledge may be cached only where deterministic invalidation exists. + +### 4.4 State Engine V2 + +Replace transition-proportional snapshot chains as the future canonical runtime-state model with: + +```text +events.jsonl — append-only canonical event history +snapshot.json — materialised current state +index.db — rebuildable acceleration index +``` + +Existing immutable state must migrate safely. + +The index is disposable. + +The event history and authoritative artifacts remain inspectable. + +### 4.5 Lifecycle Instances + +Retain the user-facing lifecycle: + +```text +UNDERSTAND -> DEFINE -> DESIGN -> PLAN -> IMPLEMENT -> VERIFY -> REVIEW -> SIMPLIFY -> COMPLETE +``` + +Internally, allow bounded Lifecycle Instances with optional parent/child relationships. + +`COMPLETE` means the current Lifecycle Instance is accepted. It does not implicitly mark parent lifecycles or project/domain state complete. + +### 4.6 Discovery and controlled re-entry + +Introduce first-class discovery events for evidence that invalidates an approved assumption, specification element, architecture decision, plan element, or implementation expectation. + +The runtime determines the minimum safe re-entry stage. + +Only affected downstream work is invalidated. + +Implementation agents may not silently rewrite upstream authority. + +### 4.7 Generic Gate Profiles + +Extend existing gate selection with reusable profiles resolved from: + +- Development Contract; +- risk; +- target capabilities; +- explicit project configuration; +- required controls; +- project-owned verification extensions. + +Profiles must not encode application-specific business rules in DKF core. + +### 4.8 Verification Extension SDK + +The previous working label “Domain Verification SDK” is refined to **Verification Extension SDK** to preserve DKF generality. + +DKF core may supply generic primitives such as: + +- absolute/relative numeric tolerance; +- range checks; +- generic invariant evaluation; +- unit/dimension adapters; +- reference/golden datasets; +- replay verification; +- deterministic custom validators. + +Projects provide the actual rules, units, formulas, thresholds, datasets, and policy. + +A solar project may supply energy constraints. A financial project may supply accounting constraints. A CAD project may supply geometry constraints. DKF core remains unchanged. + +### 4.9 Complexity Delta Guard + +Add an inexpensive deterministic complexity delta using repository/diff facts such as: + +- files added/removed; +- dependencies added; +- exported abstractions; +- single-use abstractions where deterministically detectable; +- duplicated implementation indicators; +- changed lines/size. + +It may trigger the full simplicity reviewer according to policy. + +It does not replace the simplicity reviewer. + +### 4.10 Control Center integration + +Expose: + +- Lifecycle Instances; +- workspace targets; +- context/capsule state; +- Cost Observatory; +- discoveries; +- gate/profile resolution; +- State Engine health. + +The Control Center remains observational/control-plane UI rather than a second source of truth. + +## 5. Explicit non-goals + +v0.12.0 will not: + +- hard-code solar, energy, finance, healthcare, legal, CAD, ecommerce, or other domain rules; +- hard-code `@web`, `@mobile`, `@db`, or any target name as mandatory; +- require monorepos; +- require multiple AI providers; +- replace Development Contracts; +- replace current source-section materialisation; +- allow implementation self-certification; +- remove human approval gates; +- require SQLite as canonical state; +- introduce unrestricted concurrent writes; +- automatically infer business truth from source-code shape; +- hide migration or performance misses. + +## 6. State and schema direction + +Proposed versioned contracts: + +- State Engine schema v2; +- Workspace Target Registry schema v1; +- Execution Capsule schema v1; +- Lifecycle Instance schema v1; +- Discovery Event schema v1; +- Gate Profile schema v1; +- Verification Extension contract v1; +- Cost Record schema v1. + +Exact files and module boundaries are decided during implementation by reuse-first repository inspection. + +## 7. Migration requirements + +Migration from current state must: + +1. detect legacy Autopilot and orchestration state; +2. validate legacy state before conversion; +3. preserve a recoverable legacy backup until cutover is accepted; +4. generate canonical events/snapshot/index; +5. verify semantic equivalence; +6. be idempotent; +7. fail closed on mismatch; +8. permit index rebuild from canonical state; +9. not require users to manually reconstruct projects. + +## 8. Performance objectives + +These are engineering objectives, not release claims. + +Against representative repeated workflows, v0.12.0 should target: + +- >=40% reduction in repeated context volume where the same project knowledge would otherwise be re-supplied; +- >=60% reduction in redundant repository reads where telemetry can measure them; +- >=70% reduction in repeated full repository orientation where telemetry can measure it; +- >=80% reduction in transition-generated state micro-file count after State Engine V2 migration; +- zero reduction in mandatory verification/control coverage; +- zero increase in false `ACCEPTED` regression cases; +- 100% rebuild recovery after deleting the disposable state index; +- 100% supported migration-fixture success. + +Actual results must be reported even if objectives are missed. + +## 9. Acceptance criteria + +### Cost Observatory +- **DKF-120-AC-001:** Every instrumented orchestration operation can emit a cost record containing only measured or deterministic-estimate fields. +- **DKF-120-AC-002:** Provider token fields remain null/unavailable unless supplied by the host/provider. +- **DKF-120-AC-003:** New cross-platform logical text/context comparisons normalise newline representation while preserving historical audit compatibility. +- **DKF-120-AC-033:** Release evidence includes baseline-versus-v0.12 cost comparison. + +### Workspace targets +- **DKF-120-AC-004:** A project can register arbitrary target IDs without changing DKF core. +- **DKF-120-AC-005:** Target discovery/configuration does not assume a specific application domain or fixed repository topology. +- **DKF-120-AC-006:** Tasks/contracts can bind primary, affected, and verification targets. +- **DKF-120-AC-007:** Target-specific commands/capabilities resolve without hard-coded target names. +- **DKF-120-AC-039:** Core routing is capability/configuration-driven rather than technology-name driven. + +### Context economy +- **DKF-120-AC-008:** Fresh implementation/review contexts can receive deterministic Execution Capsules. +- **DKF-120-AC-009:** Capsule fingerprints invalidate stale capsules when relevant authority/repository state changes. +- **DKF-120-AC-010:** Repository/context caching has deterministic invalidation. +- **DKF-120-AC-011:** Existing section-aware source materialisation and source-fingerprint protections remain green. +- **DKF-120-AC-035:** Comparative evidence reports measured context/orientation improvement where observable. + +### State Engine V2 +- **DKF-120-AC-012:** Canonical runtime history is append-only and reconstructable. +- **DKF-120-AC-013:** Current snapshot can be rebuilt from canonical history. +- **DKF-120-AC-014:** Disposable index deletion/rebuild produces semantically equivalent query state. +- **DKF-120-AC-015:** Supported legacy migration is idempotent, fail-closed, and recoverable before cutover acceptance. +- **DKF-120-AC-034:** Representative migrated runtime state achieves the state micro-file reduction objective or reports the miss explicitly. + +### Lifecycle Instances +- **DKF-120-AC-016:** Lifecycle Instances support optional parent/child relationships. +- **DKF-120-AC-017:** `COMPLETE` applies only to the active Lifecycle Instance. +- **DKF-120-AC-018:** Domain/application state cannot implicitly change DKF lifecycle acceptance/completion state. + +### Discovery and re-entry +- **DKF-120-AC-019:** Discovery events persist evidence and affected-authority references. +- **DKF-120-AC-020:** The runtime computes the minimum safe re-entry stage from affected authority. +- **DKF-120-AC-021:** Selective invalidation preserves unrelated accepted work. + +### Gate profiles +- **DKF-120-AC-022:** Gate profiles resolve generically from contract/risk/target capabilities/project configuration. +- **DKF-120-AC-023:** Profiles cannot downgrade existing mandatory verification, control, or human-approval gates. + +### Verification Extension SDK +- **DKF-120-AC-024:** Project-owned verification extensions can be added without modifying DKF core. +- **DKF-120-AC-025:** Generic numeric checks support absolute and relative tolerances. +- **DKF-120-AC-026:** Project-supplied invariants/custom validators can block acceptance through normal evidence/gate semantics. +- **DKF-120-AC-027:** Reference/golden dataset fingerprints participate in evidence freshness. +- **DKF-120-AC-037:** DKF core contains no application-specific domain verification rules. +- **DKF-120-AC-038:** At least two materially different example verification extensions can use the same SDK without core changes. + +### Complexity +- **DKF-120-AC-028:** Complexity Delta is deterministic from repository/diff facts used by the implementation. +- **DKF-120-AC-029:** Complexity policy may trigger, but may not remove, the full simplicity review where existing policy requires it. + +### Integration and release +- **DKF-120-AC-030:** Control Center exposes the new state without becoming canonical authority. +- **DKF-120-AC-031:** Existing supported public commands remain compatible. +- **DKF-120-AC-032:** Existing control-plane and release-validation regression suites remain green. +- **DKF-120-AC-036:** No new false-`ACCEPTED` regression scenario is introduced. +- **DKF-120-AC-040:** Release documentation includes migration, compatibility, measurement results, and known limitations. + +## 10. Release gate + +v0.12.0 is releasable only when: + +- all required acceptance criteria have evidence; +- the deterministic PLAN remains valid; +- migration tests pass; +- current release-validation suites remain green on supported CI platforms; +- no critical security or architecture finding remains unresolved; +- no app-specific logic has entered DKF core; +- comparative efficiency evidence is published honestly; +- Product Owner release approval is recorded. + +## 11. Phase 2 specification decision + +This specification supersedes the earlier provisional v0.11.0 label and the domain-specific wording “Domain Verification SDK”. + +The architectural intent remains the same, but it is now explicitly constrained to a **general-purpose core with project-defined targets, profiles, adapters, and verification rules**. diff --git a/docs/04-architecture/dkf-scale-context-iteration-v0.12-technical-design.md b/docs/04-architecture/dkf-scale-context-iteration-v0.12-technical-design.md new file mode 100644 index 00000000..a7a94b7e --- /dev/null +++ b/docs/04-architecture/dkf-scale-context-iteration-v0.12-technical-design.md @@ -0,0 +1,512 @@ +# Development Kit v0.12.0 — Scale, Context & Iteration Technical Design + +**Status:** Phase 2 technical design +**Specification:** `dkf-scale-context-iteration-v0.12-specification.md` +**Baseline evidence:** `dkf-scale-context-iteration-phase01-baseline.md` + +## 1. Design objective + +Implement v0.12.0 by extending the existing DKF control plane with generic scale/iteration primitives while preserving current command behaviour and reliability contracts. + +The design follows three rules: + +1. **Reuse existing runtime modules before creating parallel systems.** +2. **Core mechanisms remain domain/application neutral.** +3. **New persistence and orchestration features must fail closed when authority, migration, or evidence is ambiguous.** + +## 2. Existing components to extend + +| Existing component | v0.12 role | +|---|---| +| `runtime/orchestration/context-package.mjs` | Base for Execution Capsules and bounded context assembly | +| `runtime/orchestration/token-efficiency.mjs` | Base for deterministic logical context sizing | +| `scripts/token-audit.mjs` | Historical/static regression audit retained | +| `runtime/orchestration/gate-selector.mjs` | Base for generic Gate Profile resolution | +| `runtime/autopilot/state-store.mjs` | Legacy state source and migration input | +| `runtime/orchestration/orchestration-run.mjs` | Legacy run-state source and migration/integration input | +| `runtime/orchestration/evidence-store.mjs` | Existing evidence/trust semantics remain authoritative | +| Development Contract runtime | Target/capsule/gate binding authority | +| Control Center/runtime API | Observability surface for new state | +| Repository Scout | Target discovery, repository-map maintenance, invalidation hints | + +No new subsystem should duplicate these responsibilities. + +## 3. Logical architecture + +```text + DKF command surface + | + Development Conductor + | + +------------+------------+ + | | + Lifecycle Engine Contract Control + | | + +--------+--------+ +-------+-------+ + | | | | + Discovery/Re-entry Target Engine | Gate Resolver + | | | | + +--------+--------+--------+-------+-------+ + | + Context Engine + | + Execution Capsule + | + fresh isolated role context + | + implementation/review + | + evidence + acceptance engine + | + State Engine V2 + | + events.jsonl + snapshot.json + rebuildable index +``` + +Cost telemetry observes orchestration boundaries without changing acceptance authority. + +## 4. Workspace Target Engine + +### 4.1 Registry + +Recommended project-local authority: + +```text +.development-kit/workspace.json +``` + +Conceptual schema: + +```json +{ + "schemaVersion": 1, + "targets": { + "arbitrary-project-id": { + "path": "relative/path", + "kind": "optional-descriptive-string", + "dependencies": ["another-target"], + "commands": { + "test": "project-defined command" + }, + "capabilities": ["project-defined-capability"], + "adapter": null + } + } +} +``` + +### 4.2 Generality rules + +- Target IDs are arbitrary. +- `kind` is descriptive metadata, not a hard-coded DKF dispatch enum. +- Core routing uses declared commands/capabilities and adapter contracts. +- Built-in discovery may recognise common repository conventions, but recognition produces generic target records. +- A project can override or add targets. +- A single-root repository is represented as one target; monorepo use is optional. + +### 4.3 Dependency closure + +The Target Engine provides: + +- direct dependencies; +- reverse dependants; +- affected-target closure; +- verification-target closure; +- command resolution. + +It must reject cycles where target dependencies require acyclic execution semantics. + +## 5. Execution Capsules and Context Engine + +### 5.1 Relationship to current context package + +Do not replace `buildContextPackage()`. + +Add a deterministic capsule layer that selects repository/target/runtime facts before current role-context materialisation. + +Conceptual flow: + +```text +Development Contract + + +Target binding + + +repository map/index + + +direct dependency deltas + + +runtime facts + | + v +Execution Capsule + | + v +buildContextPackage() + | + v +fresh/rehydrated role context +``` + +### 5.2 Capsule contents + +A capsule contains references/fingerprints, not copied project history, wherever possible: + +- lifecycle ID; +- contract/task/run IDs; +- target binding; +- authoritative source refs and selectors; +- relevant repository file refs/fingerprints; +- relevant tests; +- direct upstream accepted tasks; +- changed-file delta; +- verified runtime facts with provenance; +- required gate profile IDs; +- cache/invalidation fingerprint. + +### 5.3 Cache design + +Repository-map/context caches are acceleration artifacts only. + +Invalidation inputs should include, as applicable: + +- target registry fingerprint; +- relevant file fingerprints; +- dependency target fingerprints; +- contract/source fingerprint; +- configuration revision; +- development-mode snapshot; +- gate/profile revision. + +A cache miss is safe and may cost more work. + +A stale cache hit is unsafe and must be prevented deterministically. + +## 6. Cost Observatory + +### 6.1 Measurement boundary + +Add a generic cost-record interface around DKF-controlled orchestration operations. + +Conceptual record: + +```ts +interface CostRecord { + operationId: string; + lifecycleId?: string; + contractId?: string; + taskId?: string; + role?: string; + + contextBytes?: number; + estimatedContextTokens?: number; + + providerInputTokens?: number | null; + providerOutputTokens?: number | null; + providerCachedTokens?: number | null; + + repositoryReads?: number | null; + repositoryScans?: number | null; + agentInvocations?: number | null; + toolCalls?: number | null; + verificationCommands?: number | null; + + cacheHits?: number; + cacheMisses?: number; + + startedAt: string; + completedAt: string; +} +``` + +### 6.2 Measurement truth + +- Deterministic DKF-controlled counts may be recorded directly. +- Host/provider values are recorded only when actually exposed. +- Missing host metrics remain null. +- New logical text-size metrics normalise `CRLF`/ `LF` before comparison. +- Existing `token:audit` historical values remain unchanged for regression continuity. + +## 7. State Engine V2 + +### 7.1 Canonical layout + +Recommended project-local state area: + +```text +.development-kit/state/ + events.jsonl + snapshot.json + index.db + schema.json +``` + +Existing artifact/evidence/contract storage may remain in current locations initially. + +### 7.2 Event model + +Each canonical state mutation emits an event with: + +- monotonic sequence; +- stable event ID; +- timestamp; +- actor class; +- lifecycle/task/contract/run refs; +- event type; +- payload; +- optional previous event hash; +- event hash. + +Hash chaining is an integrity check, not a blockchain or distributed-consensus feature. + +### 7.3 Snapshot + +`snapshot.json` is a materialised view derived from canonical events. + +It may contain: + +- active lifecycle; +- active task/contract/run; +- stage; +- current source fingerprint; +- acceptance state; +- last event sequence/hash. + +### 7.4 Index + +`index.db` is rebuildable. + +It may index relationships needed for: + +- lifecycle lookup; +- task/criterion ownership; +- source dependency/invalidation; +- evidence lookup; +- target lookup; +- telemetry querying. + +Deleting it must not destroy canonical state. + +### 7.5 Migration + +Legacy Autopilot and run-state revisions remain migration inputs. + +Migration should: + +1. read/validate current state chains; +2. convert semantic transitions into canonical events; +3. derive snapshot; +4. build index; +5. compare reconstructed current state to legacy current state; +6. preserve legacy backup before cutover; +7. mark migration complete only after equivalence verification. + +## 8. Lifecycle Engine + +### 8.1 Lifecycle Instance + +Generic model: + +```ts +interface LifecycleInstance { + lifecycleId: string; + parentLifecycleId: string | null; + type: string; + name: string; + stage: CanonicalStage; + status: string; + targets: string[]; + sourceRevision: number; + sourceFingerprint: string; +} +``` + +`type` remains extensible. DKF may provide common defaults but must not make project-specific types mandatory. + +### 8.2 Completion semantics + +- `COMPLETE` is scoped to one Lifecycle Instance. +- Parent completion is separately evaluated. +- Application/business/domain status remains an external/project namespace. +- No business status string may implicitly drive DKF acceptance. + +## 9. Discovery and controlled re-entry + +A Discovery Event references: + +- evidence; +- affected source/authority; +- current lifecycle/task/contract; +- classification; +- affected criteria/tasks; +- proposed minimum re-entry stage. + +The re-entry resolver applies generic authority rules: + +- implementation-only defect -> IMPLEMENT; +- plan decomposition invalid -> PLAN; +- architecture authority invalid -> DESIGN; +- requirement/spec authority invalid -> DEFINE. + +Project policy may require human approval for particular discovery classes or risk levels. + +Affected downstream work is calculated from persisted dependency relationships. + +## 10. Gate Profiles + +Gate Profiles extend current gate selection rather than replace it. + +A profile describes generic required/conditional verification classes. + +Resolution inputs include: + +- Development Contract; +- risk; +- target capabilities; +- explicit project configuration; +- security/design/configuration constraints; +- Verification Extension requirements. + +Profiles may add gates. + +They may not suppress mandatory gates already selected by current safety/control policy. + +## 11. Verification Extension SDK + +### 11.1 Core primitives + +Generic SDK primitives may include: + +```text +numeric tolerance +range assertion +generic invariant callback/DSL +reference dataset comparison +replay fixture +unit/dimension adapter interface +custom deterministic verifier adapter +``` + +### 11.2 Project-owned policy + +Project extensions define: + +- formulas; +- domain units; +- invariants; +- thresholds; +- datasets; +- replay cases; +- custom verification code. + +DKF core stores/validates results through existing evidence and gate semantics. + +### 11.3 Generality proof + +Release tests must include at least two materially different extension fixtures, for example: + +- a numeric/scientific-style extension; +- a non-numeric state/business-rule extension. + +Their purpose is to prove the SDK is generic. Neither becomes built-in DKF domain policy. + +## 12. Complexity Delta Guard + +Implement the smallest deterministic signal set supported by repository facts. + +A policy layer decides when to invoke the existing full simplicity reviewer. + +The guard must never convert a required simplicity review into an optional one where current policy already requires it. + +## 13. Control Center + +Extend the existing Control Center/runtime API. + +New views/data should be read from canonical runtime APIs: + +- lifecycle graph; +- target graph; +- capsule/context state; +- cost records; +- discovery queue; +- gate/profile resolution; +- state health/migration status. + +The UI is not an authority source. + +## 14. Compatibility strategy + +### Commands +Keep existing public command names and normal workflow. + +### Existing projects +Projects without target registry or State Engine V2 are detected and upgraded/initialised safely. + +### Single-package projects +One-target operation remains simple and should require no unnecessary configuration. + +### Hosts +If host telemetry is unavailable, cost fields remain null and orchestration proceeds with existing controls. + +### Existing context hardening +Section-aware source delivery, staleness checks, fresh verification isolation, and role budgets remain intact. + +## 15. Expected implementation change surface + +Exact file creation remains subject to reuse-first inspection, but likely areas include: + +```text +runtime/orchestration/ +runtime/autopilot/ +runtime/bootstrap/ +runtime/control-center or equivalent runtime API +schemas/ +scripts/*tests* +agents/ +commands/ +skills/ +docs/ +``` + +Likely new logical responsibilities: + +```text +cost observability +workspace target registry/resolution +execution capsule/cache +event ledger/snapshot/index +lifecycle instances +discovery/re-entry +gate profiles +verification extension contracts +complexity delta +``` + +Where an existing module can own the responsibility cleanly, extend it instead of creating a new file. + +## 16. Security and failure model + +Fail closed for: + +- invalid/stale target registry; +- stale capsule; +- migration semantic mismatch; +- corrupted canonical ledger; +- unresolved required gate profile; +- extension attempting to bypass mandatory control; +- project extension returning unverifiable PASS; +- discovery requiring unapproved upstream-authority change. + +Degrade safely for: + +- missing disposable index; +- absent optional telemetry; +- unavailable cache; +- unavailable host provider-token data. + +## 17. Design decision + +The implementation architecture is approved for planning with one terminology correction: + +> **Verification Extension SDK** is the canonical v0.12 term. “Domain Verification SDK” may be used only when describing the original feedback that motivated it. + +This prevents DKF from becoming specialised while retaining the ability for any project to add rigorous domain-specific verification outside core. diff --git a/docs/04-architecture/dkf-state-engine-v2-offline-recovery-boundary.md b/docs/04-architecture/dkf-state-engine-v2-offline-recovery-boundary.md new file mode 100644 index 00000000..f4dadfc1 --- /dev/null +++ b/docs/04-architecture/dkf-state-engine-v2-offline-recovery-boundary.md @@ -0,0 +1,66 @@ +# DKF v0.12 T04 — Offline recovery boundary and crash protocol + +**Status:** Candidate architecture contract; subject to independent code, architecture and security review. +**Scope:** `restoreLegacyBackup`, `recoverLegacyRestore`, `recoverAbandonedStateLock`. +**Owner:** DKF State Engine V2 (T04); no T05 or release scope. + +## 1. Explicit environment and trust boundary + +A backup restore changes an entire project directory, including its files and operating-system metadata. Node.js 18+ cannot guarantee descriptor-relative `openat2`/equivalent anti-swap semantics on every supported Windows and Unix filesystem. The framework must **not** claim that path validation, a random staging name, `0700` permissions, or `confirmOffline: true` defeats another malicious process running under the **same OS identity**. + +Similarly, runtime atomic publication (`atomicWrite` and `pinnedUnlinkAndSync` for journals, snapshots, and indexes) pins directory descriptors via `/proc/self/fd` on Linux. On supported non-Linux hosts (e.g. Windows and macOS), portable Node.js does not provide kernel-level descriptor-relative pathname resolution; publication revalidates directory device/inode identity immediately before publication and unlink, but cannot guarantee protection against concurrent, hostile same-UID processes manipulating paths. The framework explicitly acknowledges that protection from hostile co-tenant processes sharing the local OS account is not supported on non-Linux platforms without native OS handle primitives. + +**Supported T04 restore mode:** an operator-initiated, offline maintenance session with exclusive control of the target project and its parent directory. The operator must stop DKF writers, processes with access to the directory, and any untrusted process sharing the OS account before invocation. Backup bytes are validated, but the local host, operator and offline execution identity are trusted. This is a deliberate environmental scope, **not** an automatic protection enforced by a boolean option. If exclusivity cannot be established, **do not invoke the restore**. Production runtime should remain on the existing canonical state, and no provider should invoke restore autonomously. + +The `confirmOffline: true` parameter is a mandatory explicit caller attestation, not proof of isolation. Missing confirmation fails closed before any restore mutation. The restore operation is *not* a security sandbox for attacker-controlled local filesystem writers. + +A future capability supporting actively hostile same-UID processes across all platforms requires a separately reviewed process/OS security boundary and handle-relative filesystem APIs. That capability is excluded from the T04 public contract. + +## 2. Journaled promotion transaction + +- Validate the backup's source fingerprint and all destination relative paths before modifying the target tree. +- Assemble the candidate restore in a private sibling staging directory. Preserve the existing root's POSIX mode, ownership and timestamps when supported, without silently claiming Windows custom ACL preservation. +- Persist and fsync a deterministic sibling promotion journal before renaming directories. The journal includes the target path and transaction ID plus original and candidate directory device/inode witnesses serialized as decimal strings (Node BigInt to avoid unsafe-number rounding). +- Rename the original target to a transaction-specific displaced path; fsync the parent on POSIX. Rename the candidate into place, fsync the parent, then delete the journal and sync the parent. +- Cleanup a displaced old tree only after promotion is durably acknowledged. No retained original recovery bundle inside the new root is deleted by the migration itself. + +**Recovery:** If a crash interrupts promotion, new `restoreLegacyBackup` calls refuse to run while the journal exists. A trusted operator explicitly calls `recoverLegacyRestore({targetRoot, confirmOffline:true})`. Recovery verifies the exact same target, transaction name, and persisted inode witnesses; restores a displaced original if the target is absent; or acknowledges an already promoted candidate. Ambiguous, swapped or malformed states fail closed without guessing. + +**Windows namespace-durability review:** The restore and recovery transactions enforce a deterministic directory-entry flush barrier (`FlushFileBuffers` on a write-capable directory descriptor) on Windows before mutating past each transaction boundary and before unlinking the recovery journal. If the host filesystem or descriptor does not support directory flush barriers, the transaction fails closed before destructive modification, retaining the original tree, candidate stage, and recovery journal. Windows regression fixtures (including `T04-H47` and `T04-H48`) verify the parent flush barrier precedes journal removal across simulated interruptions and process exits. See Issue [#73](https://github.com/eybersjp/development-kit/issues/73). + +## 3. State lock guard failure recovery + +Normal State Engine writers never discard an uncertain `state.lock.reclaim` guard. The guard contains local PID, hostname, unique owner token and creation timestamp. A guard can be left behind by a crash or host termination. + +`recoverAbandonedStateLock({rootDir, confirmOffline:true})` is an **offline operator action**. It requires an aged (at least 60 seconds) guard, checks the guard's local owner process is gone where owner metadata exists, checks any remaining local state lock owner is gone, and uses conservative directory cleanup. A live owner, malformed state or recent guard refuses recovery. The call cannot prove no unauthorized same-UID process exists; the offline boundary is mandatory. + +## 4. Compatibility and failure discipline + +- Node.js >=18; no OS-specific native module required for the scoped offline contract. +- Existing State Engine runtime operations remain available; migration cutover remains conditional on semantic-equivalence checks. +- Backup restoration now requires explicit `confirmOffline: true` — a safety-related API tightening for operator callers, not a silent fallback. +- Replays and restoration never publish partial snapshots or claim successful acceptance on a failed integrity witness. +- An inability to preserve required platform-specific ACLs is an operational blocker, not permission to misrepresent equivalence. +- No T04 change here authorizes a package release, pruning legacy state, or bypassing Design Authority and human approval gates. + +## 5. Verification and remaining decision + +Automated tests H24–H28 validate offline confirmation, journal recovery after an injected failure and a real child-process termination, and operator-recoverable orphan guards. Earlier H01–H23 cover ledger integrity, concurrent writers, symlinks/hardlinks, snapshot recovery and permission preservation. + +**Formal acceptance decision must record:** whether the explicitly narrower *trusted offline operator* boundary is acceptable to the product owner/security authority for the T04 legacy restore capability. Automated tests cannot confer that authorization; independent risk-3 review must confirm that the code matches this scope, no in-scope Critical/Major findings remain, and the deterministic DKF governance evidence is persisted. + + +## Supported runtime threat model (T04 scoped Product Owner decision) + +State Engine V2 automatic append and recovery support a **trusted, exclusive local runtime**. The Product Owner explicitly deferred stronger same-account OS isolation to a future revision (tracked in [Issue #72](https://github.com/eybersjp/development-kit/issues/72), with the decision recorded on [PR #70](https://github.com/eybersjp/development-kit/pull/70#issuecomment-6134158112)); this scoped operating limitation still requires independent risk-3 security and architecture review before T04 acceptance. Hostile concurrent mutation by a process holding the same OS credentials (same-UID write authority) is **out of scope**: Node.js `lstat`/`fstat`/`rename`/`unlink`, `/proc/self/fd` and cooperative project locks cannot atomically protect two directory entries against such an actor, and `0700` modes or `confirmOffline` flags do not isolate it. This supported operating scope is separate from, and does not extend, the offline-only restore exception above. + +Guarantees that remain: + +- The canonical ledger (`events.jsonl`) is the sole authority. Snapshot and index are derived and verified against ledger contents on every load; a snapshot ahead of, or hash-divergent from, the ledger fails closed (`loadStateSnapshot`, `verifyStateEngineIntegrity`). +- An append is acknowledged only after the ledger append is fsynced, derived state is published and the pending journal is removed. +- A commit that cannot be proven (for example the ledger entry was replaced mid-commit) is *ambiguous*: it is detected as ledger/derived inconsistency on the next load rather than prevented. No after-the-fact check claims to prevent a concurrent authorized adversary. +- Directory/descriptor pinning, symlink/hardlink/inode checks and held ledger descriptors remain as defence-in-depth against accidental and non-concurrent tampering. + +Supporting hostile same-UID mutation would require an OS-enforced boundary (separate credentials/ACLs or a native helper service) proven on every supported platform and is not part of v0.12. + +Regression evidence: `T04-H43` (append) and `T04-H44` (recovery) replace the ledger entry between the final check and the journal `unlink`, capture ledger/journal/snapshot witnesses and assert fail-closed detection on next load. The Windows fixture is skipped when the host denies renaming an open file; Ubuntu CI provides the inode-swap evidence and the skip is not claimed as cross-platform resistance. diff --git a/docs/04-architecture/dkf-state-engine-v2-t04-hardening-validation.md b/docs/04-architecture/dkf-state-engine-v2-t04-hardening-validation.md new file mode 100644 index 00000000..3789a6c6 --- /dev/null +++ b/docs/04-architecture/dkf-state-engine-v2-t04-hardening-validation.md @@ -0,0 +1,155 @@ +# DKF v0.12 T04 — State Engine V2 Hardening Validation + +**Date:** 9 October 2026 +**Status:** Engineering remediation implemented; updated CI and independent risk-3 review must pass before acceptance. +**Branch:** fix/v0.12-t04-state-engine-hardening +**Stacked PR:** https://github.com/eybersjp/development-kit/pull/70 +**Cumulative CI PR:** https://github.com/eybersjp/development-kit/pull/69 (draft; do not merge as replacement for the approved sequence) +**Confirmed cross-platform code CI baseline:** https://github.com/eybersjp/development-kit/actions/runs/37972680014 (H01–H28; final stage-fsync/documentation review requires latest branch checks) + +## Corrected engineering risks + +- **AUD-05 — event storage:** Canonical events are now physically appended rather than rewriting all prior bytes. An fsynced pending-commit journal records the expected prefix hash and complete suffix before appending. Verified incomplete suffixes can be repaired; conflicting suffixes fail closed. On POSIX, relevant rename/unlink directory metadata is also fsynced. +- **AUD-06 — multi-process coordination:** The project-local lock spans current-state reads, sequence allocation, event construction and write. Owners are recorded by PID and hostname. A live local lock is not stolen merely because it is aged; a provably dead local owner is recoverable. Remote/unknown ownership fails closed. This is not a distributed multi-host lock. +- **AUD-07 — backup restore:** Read-only target preflight rejects symlinks, junctions and multiply-linked files. Full restore is assembled in a private sibling staging directory, fingerprint-verified, then promoted by renaming the destination root. No restored content is opened through mutable pre-existing target parents or final hard links. Original target tree is moved aside and rolled back on a failed promotion where possible; retained legacy source and backups are never removed. +- **Additional protection:** Canonical State Engine internal files may not be symlinks, multiply-linked inodes or nonregular files. Lock reclamation is serialized under an exclusive .reclaim directory; a stopped process during reclamation can leave a guard that must be inspected before manual recovery. The pending journal is cleared only after snapshot and index have been reconstructed and fsynced; stale materialized views are replayed under the State Engine writer lock. Existing Autopilot same-revision transition behavior was preserved after an initial regression was corrected. + +## Reproduction and verification + +Thirty-one T04-H01..H31 regression cases verify on Ubuntu and Windows: physical append file identity, aged live lock, four-process/48-event contention, interrupted append recovery, restore directory and file symlinks, conflicting pending suffix, dead-owner recovery and canonical-file symlink rejection. + +Additional test-driven cases are **H10** (missing terminal newline), **H11** (stale-lock recheck race), **H12** (legitimate concurrent snapshot advance) and **H13** (hard-linked restore destination). The unfixed earlier heads failed as expected: CI runs `37963478049` (H10/H11) and `37963732490` (H12/H13). All four new cases subsequently passed on both platforms. + +**Independent-review status:** the initial GitHub Codex review on commit `949ec8c` identified H10/H11. Both are corrected with reproducing tests. A fresh independent code/security review of the updated source was requested on PR #70. Successful CI is not a deterministic DKF `ACCEPTED` verdict; formal review and acceptance must still be recorded before merge. +The previous cumulative CI run 37965890821 passed every required step on Ubuntu and Windows, including T04 focused tests and the exact npm run release:validate command. No skipped test was reported. + +## 300-transition endurance fixture + +| Metric | Ubuntu / Node 22.23.3 | Windows / Node 22.23.3 | +|---|---:|---:| +| Canonical transitions | 300 | 300 | +| Write workload | 6,068 ms | 16,417 ms | +| Full read/integrity verify | 54 ms | 48 ms | +| Canonical ledger | 189,651 bytes | 189,651 bytes | +| Historical full-rewrite byte model | 28,479,948 bytes | 28,479,948 bytes | +| Modeled write traffic avoided | 99.33% | 99.33% | +| Integrity | PASS | PASS | + +**Measurement limitation:** Historical full-rewrite traffic is modeled from observed event file lengths, not an experimentally timed legacy run. CPU parsing and full-history replay remain O(history) per read/verification; the 300-event fixture does not prove unlimited-scale latency. + +## Authority and release boundaries + +Original T04 contract and acceptance criteria remain unchanged; no T05+ scope, version bump, legacy file deletion, tag or release occurred. Main remains on published 0.11.2. Independently execute the existing risk-3 code/architecture/security review and capture persisted runtime acceptance evidence before promoting/merging the stacked T04 fix. Re-run final integration checks after any source change. T10 still owns release-wide performance objectives. + +## Further P1/P2 review corrections — H14 to H21 + +- **H14:** Windows transient state.lock creation sharing violations retry within the lock timeout, without stealing a lock (Windows-only fixture). +- **H15:** four independent processes contend to recover one stale owner; exclusive reclamation guard prevents deletion of a subsequently acquired live lock. +- **H16:** replacing canonical events.jsonl with an external hard link is refused before append, preserving the outside inode. +- **H17:** interrupted journal recovery rebuilds and persists snapshot/index before deleting the commit marker; immediate strict integrity verification succeeds. +- **H18:** a writer interleaving with an older reader cannot have its derived snapshot/index rolled back by stale replay. +- **H19:** switching a legacy restore destination to an external hard link at the former open point cannot modify outside content; staged restore avoids target-path opens. +- **H20:** restore preflight does not create or mutate directories in the target tree before stage promotion. +- **H21:** staged restore preserves support for a previously nonexistent target root with an existing parent. + +The review that identified H15–H19 was recorded on PR #70 against head `ffe49ffb60`. Its five associated threads must be resolved only after validating the revised code and obtaining an independent current-head review. Passing the CI suite does not supersede those risk-3 approval gates. Restore is an offline maintenance operation: protect target-parent access and avoid other writers to the recovery directory while staging and promoting. Staging relies on same-filesystem sibling directory renames. Source/workflow authority, the published version (0.11.2), and the prohibition on merging validation PR #69 remain unchanged. + +## Third independent-review correction — H22 and H23 + +Code review of `cf504f7` identified two additional P1 issues, reproduced before repair: + +- **H22 — event symlink substitution at append/open:** pre-fix run `37967267072` showed an external sentinel file receiving appended canonical JSON after a symlink was swapped into the event pathname. The fixed append opens with `O_NOFOLLOW` where supported, verifies `lstat()` and `fstat()` refer to the same one-link inode, and checks the opened descriptor against the exact canonical inode and byte length witnessed before the commit was prepared. The write is rejected before modifying external content. +- **H23 — root metadata preservation:** pre-fix run `37967284714` showed an existing `0755` restore root becoming `0700`. The corrected staged restore restores the original directory's mode and timestamps before promotion, and on POSIX preserves UID/GID or fails closed when ownership cannot be set. The existing-stage private mode is retained for previously absent roots. + +**Verification authority:** previous H01–H21 source passed Ubuntu and Windows CI `37966537455`, including exact release:validate. The final H01–H23 implementation and documentation require CI at the latest branch head. This is correction evidence, not an independent final acceptance decision. + +**Metadata limitation:** generic Node.js APIs cannot establish full equivalence of every filesystem ACL/security-descriptor property (especially custom Windows ACLs). Restoring into an existing root with nonstandard ACLs requires separately verified host-native permission preservation before declaring the backup operation complete. No claim of universal Windows ACL preservation is made. + +## Fourth hardening increment — H24 to H28 and recovery trust boundary + +- **H24:** `restoreLegacyBackup` rejects callers that have not explicitly attested a trusted exclusive offline workspace; no target mutation is permitted without `confirmOffline:true`. +- **H25:** a failed promotion leaves a durable transaction journal. A new restore is refused until `recoverLegacyRestore({targetRoot,confirmOffline:true})` validates original/candidate device/inode witnesses and resolves the transaction. +- **H26:** an abandoned `state.lock.reclaim` guard from a provably dead owner can be removed only by explicit offline recovery, after a 60-second inactivity threshold and local process liveness checks. +- **H27:** orphan recovery refuses a live guard owner even when the guard is old. +- **H28:** a child-process termination immediately after moving the target root leaves a journal permitting exact original-tree restoration without running the original process's `finally`. + +The crash journal uses BigInt-backed decimal device/inode witnesses for portable Windows/Unix verification. The staged restore tree is fsynced before journal publication; POSIX directory entries are synced at transaction boundaries. The previous full cross-platform run `37972680014` passed at the journal-identity revision; latest fsync/documentation changes require current-head CI. + +**Security boundary decision pending independent review:** see [Offline Recovery Boundary](dkf-state-engine-v2-offline-recovery-boundary.md). The restore capability is *operator-initiated offline maintenance only*. A malicious concurrent process with the same OS credentials cannot be isolated by cross-platform Node.js pathname guards and must be excluded operationally. The `confirmOffline` flag is an attestation, **not technical evidence** that the host is safe. If the environment is not exclusively controlled, restoration is unsupported and must not run. This scope must be independently accepted rather than silently dismissing security findings. + +**Review status:** PR #70 remains unmerged; Issue #71 tracks the remaining recovery and isolation contract. Source implementation and targeted CI passing do not on their own create a valid risk-3 code/architecture/security review or persisted deterministic ACCEPTED state. Published version remains v0.11.2. + +## Fifth independent-review corrections — H29 to H31 + +Independent review at source `c80dc253eadf6676fa195628aebd10cb3f82dae9` identified: + +- **P1 runtime pending-commit path race:** after validating a canonical ledger prefix, pathname `truncateSync` could shorten an external hardlinked inode swapped into `events.jsonl`. H29 captured the external-file modification on the unfixed source (GitHub Actions `37974023511`, Windows and Ubuntu). The fixed recovery opens a no-follow, inode-verified and single-link FD; validates all bytes and performs `ftruncateSync`, positioned suffix writes and `fsyncSync` on that same descriptor. +- **P1 recursive staged ownership:** staging an existing root copied unrelated files under the maintenance identity, potentially losing access for service accounts. A metadata inventory is collected before copying, and post-order replay restores descendant UID/GID, modes and timestamps on POSIX (including symlink ownership without dereferencing symlink targets). H30 checks descendant identity; CI runs the same fixture with an alternate source owner under a restricted internal-branch Ubuntu job. +- **P2 timestamp drift:** scanning the stage after `utimesSync` advanced the restored root atime. The stage sync precedes post-order metadata replay. H31 uses a historical 2001 atime/mtime fixture. + +**Security caveat:** The supported restoration trust boundary remains *explicit, operator-controlled offline maintenance* (see the separate recovery boundary contract). It does not claim immunity to a concurrently malicious same-UID process, and this environmental condition must be independently approved. The automatic runtime journal-recovery path does **not** rely on the offline attestation and must satisfy stronger descriptor-bound checks. + +**Acceptance gate:** CI/review must be repeated on the final documented head. The H01–H31 implementation is not deemed ACCEPTED until risk-3 code, architecture and security review findings are closed by evidence, deterministic acceptance is persisted, and PR #70 is merged only into its approved T04 base. Do not merge cumulative PR #69 or publish v0.12.0. + +## Sixth review cycle — H32–H35 and authenticated acceptance + +Independent PR #70 reviews discovered that copying could rewrite unrelated relative symlinks, duplicate backup entries could overwrite one target, platform case sensitivity differed, and malformed UTF-16 surrogate strings could map to the same filesystem filename. + +- **H32:** stage copy now uses verbatim symlink semantics to preserve unrelated relative targets. +- **H33:** duplicate normalized restore destinations are rejected before stage mutation. +- **H34:** normalized case/Unicode alias detection is conservative on every supported filesystem. +- **H35:** reject unpaired UTF-16 surrogate paths before filesystem encoding. +- **Authority Graph:** ten unchanged approved T04 requirements are explicitly mapped to the existing five acceptance criteria, with negative tests for missing or unknown edges. +- **Risk-3 review:** the former acceptance driver constructed synthetic reviewer PASS records and hardcoded an older commit. That verdict is invalid as independent approval. The new gate requires three separate, externally authenticated role-specific reviews, binds Git blob hashes and modes for the entire checked-out tree to the reviewed commit, and permits only the receipts document to change after review. +- **Platform gate:** the acceptance job depends on success of both Windows and Ubuntu validation jobs, not on one matrix leg. +- **Fail-closed behavior:** missing, stale, inauthentic or finding-bearing review evidence never grants T04 acceptance. + +**Promotion boundary:** A persisted deterministic ACCEPTED result with zero runtime blockers, authenticated code/architecture/security reviewer roles and full matrix success is required. Earlier synthetic acceptance outputs are explicitly superseded. No main-branch merge, release or T05 authority is implied. + +## Seventh review cycle — H42–H45, P1/P2 resolution and skipped-test audit + +Following engineering review #6133448943 on PR #70: + +- **P1 missing canonical ledger with pending journal:** in `runtime/orchestration/state-engine-v2.mjs`, when `events.jsonl` is missing while `pending-commit.json` is present, `loadCanonicalEvents()`, `loadStateSnapshot()`, `ensureStateEngineLayout()`, and `recoverPendingCommitLocked()` fail closed with a descriptive `StateEngineError` rather than treating state as uninitialized or creating an empty ledger. Verified by new regression fixture `T04-H45`. +- **P2 contract requirement validation shape guards:** in `runtime/orchestration/development-contract.mjs`, `validateDevelopmentContract()` guards `criterion.requirementIds` and `contract.requirements` arrays before iteration or `.some()` calls, ensuring malformed non-array or null values aggregate into `ContractValidationError` without throwing raw `TypeError`. Verified by new fixtures in `scripts/orchestration-contract.test.mjs`. +- **H43/H44 test hardening:** removed synthetic post-operation file overwrites; tests now inspect and assert the authentic swapped filesystem state directly. +- **Threat model provenance:** updated `docs/04-architecture/dkf-state-engine-v2-offline-recovery-boundary.md` to accurately define the exclusive local runtime as a candidate engineering proposal subject to recorded PO/security authorization. + +### Audit of skipped tests on Windows (11 tests in `scripts/state-engine-v2-hardening.test.mjs`) + +All 11 skipped tests are designed to execute on Linux (`ubuntu-latest` CI) where the underlying OS primitives exist: + +1. **`T04-H06`** (*Host does not permit file symlink creation*): Non-elevated Windows environments prohibit creating unprivileged file symlinks without Developer Mode enabled. +2. **`T04-H09`** (*Host does not permit file symlink creation*): Tests internal canonical events file redirection rejection via symlinks; requires file symlink capability. +3. **`T04-H22`** (*Host denies file symlink fixture*): Tests symlink swap immediately before open; requires file symlink support. +4. **`T04-H23`** (*POSIX mode/uid semantics are not portable to Windows*): Validates exact POSIX `0755`/`0700` octal modes and numerical UID/GID preservation during staged restore. Windows NTFS ACLs do not map 1:1 to POSIX mode bits. +5. **`T04-H30`** (*POSIX ownership and mode semantics require a POSIX host*): Validates recursive descendant UID/GID preservation in staged restore. +6. **`T04-H31`** (*POSIX access-time semantics require a POSIX host*): Tests historical 2001 `atime` preservation without scan drift; Windows timestamp precision and update rules differ. +7. **`T04-H32`** (*Relative symlink unavailable*): Tests preservation of relative target strings in restored symlinks; requires unprivileged symlink creation. +8. **`T04-H36`** (*Windows locks directory handles against renaming while pinned open*): Renaming an open directory handle returns `EPERM` on Windows NTFS. On Linux, `/proc/self/fd` directory pinning allows directory renames. +9. **`T04-H37`** (*Windows locks directory handles against renaming while pinned open*): Directory swap test during commit; Windows kernel file locks prevent renaming the pinned state directory. +10. **`T04-H38`** (*Windows locks directory handles against renaming while pinned open*): Directory replacement with ordinary directory; Windows kernel locks prevent open directory renaming. +11. **`T04-H40`** (*Windows locks directory handles against renaming while pinned open*): Directory replacement before append; Windows kernel locks prevent open directory renaming. + +These 11 tests run unconditionally and provide full regression coverage on the `ubuntu-latest` GitHub Actions CI matrix runner. + +## Eighth corrective cycle — recovered suffix durability, rollback durability and review-thread acceptance + +Independent source review at `cf0fd8208cd262da29dfe0d91576c4808efa6873` surfaced three additional P1 findings, tracked on PR #70. This increment implements corrections pending cross-platform CI and new independent approval: + +- **T04-H46:** automatic pending-commit recovery must `fsync` the opened canonical ledger even if all expected suffix bytes are already present, because the interrupted writer may have stopped between its last `write` and `fsync`. The new regression guards journal unlink until the ledger descriptor has actually been flushed. +- **T04-H47:** offline restore recovery must durably sync the parent directory immediately after rolling back a displaced root and before unlinking the restore journal. The POSIX regression uses a real child-process exit and asserts the ordering. Windows parent-directory `fsync` has different/unsupported portable semantics and is not claimed by this fixture. +- **Independent review acceptance:** `t04-review-thread-blockers.mjs` now rejects all unresolved review threads regardless of P1/P2 label. Three pure regression tests are included in `state-engine-v2:test`; reviewer replies and authenticated receipts are still mandatory. + +The Product Owner's decision defers hostile same-account concurrent-filesystem isolation to future [Issue #72](https://github.com/eybersjp/development-kit/issues/72). For v0.12.0 the runtime boundary is an exclusively controlled trusted local workspace. The stronger attack case remains an explicitly unsupported risk; independent security review must confirm this boundary is accurately enforced and disclosed. The restricted offline recovery trust boundary is separate. + +**Evidence status:** these new corrections have been committed on the T04 hardening branch, but do not constitute an independent accepted finding or authorize T05. Only a complete Windows/Ubuntu validation run at the final immutable head, three authentic source-bound role reviews, zero unresolved in-scope findings and a deterministic `ACCEPTED` record can close T04. + +## Ninth review cycle — authenticated review evidence and Windows-native rollback candidate + +- **P1 source-bound review request:** Role-specific `codex-clean-reaction` requests now require exactly one anchored `Commit:` or `Immutable source commit:` field containing the full reviewed SHA; conflicting or extra 40-character hashes fail closed. `T04-A04` tests alternate baseline, excluded, repeated and malformed source claims. +- **P2 GitHub evidence shape:** GraphQL review-thread pagination now rejects missing/null `nodes`, malformed resolution flags, invalid pagination and nonadvancing cursors. `T04-A05` exercises the negative cases. +- **P1 Windows rollback ordering:** Offline `recoverLegacyRestore()` on Windows calls a bundled PowerShell/Win32 `MoveFileExW(MOVEFILE_WRITE_THROUGH)` helper for the `displaced → root` rollback, with errors propagated and the journal retained if the native operation fails. `T04-H48` is a Windows-specific real process-interruption fixture verifying the native path is exercised before journal removal. +- **Limitations:** A Windows CI fixture can prove API invocation and error-path ordering, but does not itself prove crash persistence on every filesystem. The original two-step restore promotion and journal-publication Windows durability remain subject to Issue [#73](https://github.com/eybersjp/development-kit/issues/73) and independent security/architecture review. No in-scope P1 is waived by the separate same-account isolation deferral in Issue #72. + +**Acceptance:** Keep T04 `PENDING` until exact final-source Windows and Ubuntu CI passes, independent role reviews have no outstanding in-scope blockers, all review threads have documented dispositions, receipts are authenticated and the deterministic risk-3 acceptance gate produces `ACCEPTED`. Source changes and passing tests alone confer no approval. diff --git a/docs/04-architecture/dkf-state-engine-v2-t04-review.md b/docs/04-architecture/dkf-state-engine-v2-t04-review.md new file mode 100644 index 00000000..72c27b31 --- /dev/null +++ b/docs/04-architecture/dkf-state-engine-v2-t04-review.md @@ -0,0 +1,270 @@ +# DKF120-T04 — State Engine V2 Review + +**Review status:** PASS +**Date:** 7 October 2026 +**Increment:** DKF120-T04 — State Engine V2 +**Branch:** `feature/v0.12-state-engine-v2` +**Reviewed implementation head:** `e3e4b16e372b40042398cfcf210f471e4f762b1b` +**Implementation validation run:** `37678375215` + +## 1. Review gates + +T04 is risk level 3 because persistent-state migration and recovery affect DKF control-plane authority. + +Derived gates: + +- specification verification; +- tests; +- migration verification; +- code-reviewer; +- architecture-reviewer; +- security-reviewer; +- security control domain. + +No consequential-action human approval is derived at risk level 3. + +## 2. Code review + +**Verdict: PASS** + +The implementation adds a generic State Engine core and keeps legacy-format knowledge in a separate migration adapter. + +Primary production changes: + +- `runtime/orchestration/state-engine-v2.mjs`; +- `runtime/orchestration/state-engine-migration.mjs`; +- Autopilot state-store cutover integration; +- orchestration-run state cutover integration; +- State Engine event/snapshot/index schemas; +- T04 migration and integrity tests. + +No T05+ capability is implemented. + +### Correction history + +#### REV-T04-001 — Fresh-state regressions still asserted legacy pointer files + +**Status:** Corrected + +The v0.7.1 and Autopilot regression suites contained storage-shape assertions that a fresh project must create the legacy `autopilot/state/current.json` pointer. + +T04 intentionally changes the persistence implementation while preserving the behavioral invariant: lifecycle state must be durably persisted before progress is reported. + +The regressions now verify durable State Engine V2 history/snapshot for fresh projects while migration tests separately preserve legacy compatibility. + +#### REV-T04-002 — Historical Phase 1 baseline accidentally followed the new live backend + +**Status:** Corrected + +The Phase 1 baseline originally recreated the legacy 26-revision fixture by calling today's `saveStateRevision()`. + +After T04, that API correctly uses V2 for fresh projects, making the historical baseline self-modifying. + +The baseline now constructs the historical legacy file layout directly, preserving the pre-v0.12 measurement of 26 revision files plus one current pointer. + +#### REV-T04-003 — Canonical tail truncation could be silently rebuilt downward + +**Severity:** Major integrity risk +**Status:** Corrected + +A valid previously materialized snapshot can witness that canonical history once reached a later sequence/hash. + +If `events.jsonl` is later truncated behind that witness, `loadStateSnapshot()` now fails closed rather than replacing the later snapshot with older reconstructed state. + +Interior mutation, insertion, deletion, and reordering remain protected by sequence/hash-chain validation. + +#### REV-T04-004 — State-root realpath escape + +**Severity:** Security hardening +**Status:** Corrected + +Lexical path checks alone did not prevent `.development-kit/state` from being a symlink/junction to a directory outside the project. + +State Engine read/write paths now realpath-check the State Engine root and reject escape outside the project boundary. + +#### REV-T04-005 — Partial migration could have triggered premature V2 cutover + +**Severity:** Critical control-plane risk during review +**Status:** Corrected + +Imported V2 entities are not sufficient to switch an existing legacy project. + +Legacy state remains canonical until a verified `MIGRATION_COMPLETED` event records semantic equivalence. + +This rule is enforced for Autopilot reads/recovery/writes and orchestration-run reads/writes. + +Interrupted migration can therefore leave resumable imported events without changing active state authority. + +#### REV-T04-006 — Re-running migration after legitimate V2 progress + +**Severity:** Major idempotence issue +**Status:** Corrected + +After migration, V2 state may legitimately advance beyond the old legacy endpoint. + +Repeated migration now recognizes the recorded completed source fingerprint, validates State Engine integrity and the retained backup, and returns an idempotent no-op without demanding that current V2 state still equal the historical legacy endpoint. + +#### REV-T04-007 — Legacy chain gaps and anchor mismatch + +**Severity:** Major auditability hardening +**Status:** Corrected + +Migration now rejects missing Autopilot revisions, workflow identity changes inside an Autopilot revision chain, missing orchestration revisions, and orchestration revision 1 that differs from its immutable run manifest. + +Legacy audit history cannot silently compress around missing source revisions. + +#### REV-T04-008 — Same-version schema tampering could be under-detected + +**Severity:** Security hardening +**Status:** Corrected + +`schema.json` must now exactly match the runtime State Engine schema contract. + +An incomplete V2 layout or a same-version tampered schema fails closed on read. + +## 3. Architecture review + +**Verdict: PASS** + +### 3.1 Canonical authority + +T04 implements the project-local State Engine area: + +- `events.jsonl` — canonical runtime history; +- `snapshot.json` — derived current-state view; +- `index.db` — disposable/rebuildable acceleration index; +- `schema.json` — State Engine schema/authority descriptor; +- `legacy-backup.json` — migration recovery bundle when legacy migration occurs. + +The snapshot and index can be reconstructed from canonical history. + +The index cannot authorize state changes or replace the event ledger. + +### 3.2 Portable index choice + +The current `index.db` implementation is a deterministic portable JSON-encoded acceleration artifact, not SQLite. + +This is deliberate for T04: DKF's public Node >=18 compatibility is preserved, no Node-22-only SQLite runtime dependency is introduced, and the index remains disposable. + +A later index adapter may use SQLite without changing canonical state authority. + +### 3.3 Generic entity model + +State Engine core understands generic entity type, entity ID, state revision, lifecycle/task/contract/run references, state patch operations, and generic metadata events. + +It contains no application or industry business-state semantics. + +Autopilot and orchestration formats are integration/migration adapters around the generic core. + +### 3.4 Event-backed state + +Initial entity state is stored once. + +Subsequent changes are represented as deterministic path-based set/delete operations rather than another full canonical snapshot file. + +Events carry monotonic sequence, deterministic event ID, timestamp, actor class, entity references, event type, payload, previous-event hash, and event hash. + +### 3.5 Crash/recovery model + +Canonical event writes use temporary file -> fsync -> atomic rename. + +A short State Engine lock prevents overlapping local commits. + +If failure occurs before canonical rename, the existing ledger remains authoritative. If failure occurs after ledger commit but before snapshot/index completion, derived state rebuilds from events. Orphan temporary files do not advance canonical state. + +## 4. Migration review + +**Verdict: PASS** + +Migration validates all detected supported legacy inputs before State Engine creation/cutover. + +Supported legacy inputs include Autopilot revision chains/current pointer and orchestration manifest/revisions/current pointer/final state where present. + +Migration then creates a fingerprinted recoverable backup bundle, imports validated history, reconstructs V2 current state, verifies semantic equivalence, appends `MIGRATION_COMPLETED` only after equivalence, and verifies State Engine integrity. + +Repeated execution is idempotent. Partial execution remains recoverable and does not trigger cutover. Legacy state is not deleted by T04. + +## 5. Acceptance-semantics review + +**Verdict: PASS** + +The orchestration migration fixture begins with `state=ACCEPTED`, `acceptanceState=ACCEPTED`, and `verificationVerdict=PASS`. + +After migration, V2 reproduces those same values. + +T04 changes persistence, not acceptance authority. + +## 6. Security review + +**Verdict: PASS** + +Reviewed threats include event mutation/insertion/reordering/deletion, canonical-tail loss, path escape, schema tampering, partial migration, corrupt legacy input, backup mismatch, index authority confusion, and premature cutover. + +Controls include monotonic sequence, chained SHA-256 hashes, snapshot sequence/hash integrity witness, exact schema contract, project-root realpath confinement, short write lock, atomic canonical replacement, derived-state rebuild, fail-closed legacy validation, fingerprinted recovery bundle, verified migration-completion cutover, and semantic-equivalence verification. + +Unresolved security findings: none. + +## 7. Security control-domain review + +**Verdict: PASS** + +State authority cannot move from legacy to V2 merely because V2 files or partially imported entities exist. + +The cutover condition is a completed migration record produced only after semantic-equivalence verification. + +Corrupt or incomplete source state blocks migration. Corrupt or tampered State Engine authority blocks reads/integrity verification instead of silently falling back. + +## 8. State micro-file result + +Representative Phase 1 legacy transition shape is 27 files: 26 revision snapshots plus one current pointer. + +T04 active canonical/recovery representation is 5 State Engine files: `events.jsonl`, `snapshot.json`, `index.db`, `schema.json`, and `legacy-backup.json`. + +Measured active canonical/recovery file-count reduction: + +**27 -> 5 = 81.48%** + +This exceeds the >=80% v0.12 engineering objective for transition-generated state micro-file count. + +### Recovery-retention caveat + +T04 deliberately retains all 27 legacy source files until a later explicitly governed cleanup/cutover decision. + +Therefore, during the recovery-retention window, physical disk file count is 27 retained legacy files + 5 State Engine files = 32 files. + +The 81.48% result describes the new active canonical/recovery state representation, not total temporary on-disk file count while recovery originals are retained. + +## 9. Verification evidence + +Implementation validation run `37678375215`: + +- Ubuntu: PASS; +- Windows: PASS; +- focused T04 gate: PASS on both; +- exact `release:validate`: PASS on both. + +Focused T04 suite: + +- tests: 22; +- pass: 22; +- fail: 0. + +## 10. General-purpose framework check + +**PASS** + +State Engine V2 is project-local, host-neutral, provider-neutral, repository-topology-neutral, and application-domain-neutral. + +No application-specific state type is required by core. + +## 11. Outstanding findings + +Critical: **0** +Major: **0** +Unresolved minor: **0** + +## 12. Review decision + +**PASS** + +DKF120-T04 is technically suitable for acceptance and progression to DKF120-T05. diff --git a/docs/04-architecture/dkf-t04-independent-review-receipts.json b/docs/04-architecture/dkf-t04-independent-review-receipts.json new file mode 100644 index 00000000..649a17d2 --- /dev/null +++ b/docs/04-architecture/dkf-t04-independent-review-receipts.json @@ -0,0 +1,7 @@ +{ + "schemaVersion": "1.0.0", + "reviewedCommit": null, + "reviews": [], + "status": "PENDING_INDEPENDENT_REVIEWS", + "policy": "Each role requires a distinct authenticated external GitHub review of the exact source and zero outstanding comments." +} diff --git a/docs/04-architecture/dkf-t04-windows-restore-durability-protocol.md b/docs/04-architecture/dkf-t04-windows-restore-durability-protocol.md new file mode 100644 index 00000000..8b918b99 --- /dev/null +++ b/docs/04-architecture/dkf-t04-windows-restore-durability-protocol.md @@ -0,0 +1,51 @@ +# DKF120-T04 — Windows restore and State Engine durability protocol + +**Status:** Proposed engineering protocol; **not** an independently approved Windows power-loss guarantee. +**Decision authority:** Product Owner requires full Windows legacy-backup restore/recovery support in v0.12.0 (Issue #73). The unrelated hostile same-account process-isolation enhancement remains deferred (Issue #72). + +## Exact persistence dependencies + +The canonical `events.jsonl` ledger is authoritative; `snapshot.json` and `index.db` are derived. The pending commit journal must be durably published **before** an in-place event-ledger write. Recovery must not delete it until the ledger suffix, derived materializations and relevant directory entries are safely persisted. + +The offline legacy restore transaction is separate and requires explicit `confirmOffline:true` and an exclusive, trusted local runtime: + +1. Validate backup fingerprint, target path, root inode and trust boundary; do not mutate the existing target. +2. Create stage as a sibling of the target, copy and verify all candidate bytes, fsync files and descendant directories. +3. Persist recovery journal file contents and flush the *parent-directory namespace* before moving either root. A journal file fsync alone is insufficient evidence of its durable directory entry. +4. Move existing root to a distinct displaced-original sibling; flush the parent namespace before moving the candidate. +5. Move staged candidate to root; flush parent namespace before acknowledging success. +6. Delete recovery journal only after durable promotion/verification, then flush parent namespace. Preserve the displaced original until this acknowledgement is complete. +7. Delete the displaced original as a separate best-effort cleanup. Flush parent after cleanup when supported. +8. On interruption, operator-recovery verifies path/transaction identity and inode witnesses. Recovery either restores displaced original when root is absent, retains displaced original if promotion completed, or fails closed on ambiguous evidence. Never delete the sole recovery evidence before a successful barrier. +9. Any unsupported namespace flush or failed barrier fails closed. Preserve the original root, displaced original, stage and journal to the maximum safe extent. The API must never claim a successful restore after an ambiguous outcome. +10. Distinguish fresh initialization, ordinary State Engine append/migration and offline restore; none may silently substitute for another. + +## OS primitives and evidence boundary + +**Implementation currently under review:** Node.js `fs.fsyncSync` on a directory descriptor opened `r+` on Windows, `r` on POSIX. `state-engine-v2.mjs` now also flushes the State Engine parent after publishing `pending-commit.json` and before any in-place canonical append. The migration module flushes parent namespace after restore renames and journal cleanup. + +**Primary Microsoft documentation:** + +- `FlushFileBuffers`: https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-flushfilebuffers — writes buffers for the provided write-capable handle; notes hardware cache and file handle limitations. +- `CreateFileW`: https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew — a directory handle requires `FILE_FLAG_BACKUP_SEMANTICS`; NTFS metadata persistence is discussed in relation to `FILE_FLAG_WRITE_THROUGH`. +- `MoveFileExW`: https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-movefileexw — the explicit `MOVEFILE_WRITE_THROUGH` flush guarantee is described for copy/delete moves; a successful move does not alone prove every same-volume directory rename/journal boundary. +- `NtFlushBuffersFileEx` (kernel documentation): https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/nf-ntifs-ntflushbuffersfileex — normal flag 0 concerns file data and metadata + storage sync, while FILE_DATA_SYNC_ONLY is not valid for directory handles. This is **context**, not proof that every user-mode Node/libuv filesystem combination has identical semantics. + +**Supported guarantee not yet independently established:** Passing `fs.fsyncSync(fd)` in GitHub-hosted Windows CI shows the syscall path returned without error and proves the observed software call ordering. It does not simulate power interruption, certify drive write-cache behavior, establish power-loss atomicity across multiple entries, or guarantee remote SMB, ReFS, FAT/exFAT, removable media or every storage controller. Do not assert universal NTFS or all-filesystem power-loss durability from these references alone. + +Production-support requirements include a documented local-volume/NTFS test environment, exact operating-system and filesystem identification, supported handle semantics, fault injection and independent architecture/security approval. Any inability to substantiate the required guarantee remains a **blocking Issue #73 risk**, not authorization to remove Windows restore support. + +## Test evidence required + +- **Runtime journal ordering:** T04-H49 injects a parent-directory flush failure and asserts no ledger bytes are appended and the pending marker is retained; T04-H50 verifies directory flush completes before physical canonical append. +- **Restore rollback ordering:** T04-H47/H48 observe the proper parent-directory flush before removal of the restore journal on applicable hosts. +- **Transaction fault injection:** before and after journal publication, original displacement, stage promotion, journal unlink, original cleanup and explicit recovery. Validate original-file identity, fingerprint, canonical ledger history and recovery witnesses. +- **Hardware/OS durability:** dedicated power-loss/reboot test harness is still required to distinguish normal process termination from forced host power loss and to validate persistence on each claimed Windows filesystem. + +For each test record OS build, filesystem, disk/cache environment, Node/libuv version, source SHA, exact fault boundary, directory witnesses and final hashes. Skips are not passes. + +## Acceptance and release + +T04 remains `BLOCKED` until Issue #73 is satisfactorily resolved with durable Windows functionality, the independent three-role reviews authenticate the **exact source**, all in-scope P1/P2 findings are dispositioned, and risk-3 acceptance produces `ACCEPTED`. CI passing and this document do not confer security signoff. + +Related source: `runtime/orchestration/state-engine-migration.mjs`, `runtime/orchestration/state-engine-v2.mjs`, `scripts/state-engine-v2-migration.test.mjs`, `scripts/state-engine-v2-hardening.test.mjs`. diff --git a/docs/04-architecture/dkf-workspace-target-engine-t02-review.md b/docs/04-architecture/dkf-workspace-target-engine-t02-review.md new file mode 100644 index 00000000..dafd8d31 --- /dev/null +++ b/docs/04-architecture/dkf-workspace-target-engine-t02-review.md @@ -0,0 +1,301 @@ +# DKF120-T02 — Workspace Target Engine Review + +**Review status:** PASS +**Date:** 7 October 2026 +**Branch:** `feature/v0.12-workspace-target-engine` +**Reviewed implementation head:** `94a10a77d38bdeba57ee398cb533b2d1409afc5c` +**Validation run:** GitHub Actions `37668415974` + +## 1. Scope reviewed + +Production/runtime changes: + +- `runtime/orchestration/workspace-targets.mjs` +- `runtime/orchestration/development-contract.mjs` +- `runtime/orchestration/index.mjs` +- `schemas/workspace-target-registry.schema.json` +- `schemas/development-contract.schema.json` +- `.development-kit/workspace.json` + +Contract/test/integration changes: + +- `scripts/phase04-workspace-target-engine-contract.mjs` +- `scripts/workspace-targets.test.mjs` +- `scripts/orchestration-contract.test.mjs` +- `scripts/configuration-readiness-integration.test.mjs` +- `package.json` +- `.github/workflows/ci.yml` +- script reference documentation and documentation index. + +## 2. Contract boundary + +The implementation remains inside **DKF120-T02**. + +It does not implement: + +- Execution Capsules or context caching; +- State Engine V2; +- Lifecycle Instances; +- discovery/re-entry; +- Gate Profiles; +- Verification Extension SDK; +- Complexity Delta Guard; +- Control Center v0.12 integration. + +No application-specific or industry-specific routing semantics were introduced. + +## 3. General-purpose architecture review + +### 3.1 Arbitrary target IDs + +PASS. + +Target IDs are project-defined and validated only for safe persisted identifier/path syntax. + +Tests use deliberately unrelated examples such as: + +```text +@alpha +service:beta +shared/common +custom/project-unit +``` + +No DKF core change is required to add a new valid target ID. + +### 3.2 Target kind + +PASS. + +`kind` is descriptive metadata only. + +Routing does not branch on `kind`, application type, framework name, language, infrastructure technology, or business domain. + +### 3.3 Capability/configuration-driven routing + +PASS. + +Target selection uses: + +- declared capability; +- declared command; +- optional preferred target IDs; +- project-owned adapter metadata. + +The router does not infer behaviour from target IDs or descriptive kinds. + +### 3.4 Adapter boundary + +PASS. + +Adapter metadata is opaque to the Workspace Target Engine and remains project-owned. + +The engine may return adapter metadata with a resolved route, but it does not assign domain semantics to it. + +## 4. Workspace registry + +Canonical project-local registry: + +```text +.development-kit/workspace.json +``` + +Registry schema: + +```text +Workspace Target Registry schema v1 +``` + +Supported generic target properties: + +- path; +- optional descriptive kind; +- dependencies; +- commands; +- capabilities; +- verification command references; +- optional adapter metadata. + +The DKF repository itself is represented through one project-defined target, `dkf-framework`. That identifier is project configuration, not a reserved DKF core target name. + +## 5. Single-target and multi-target behaviour + +PASS. + +A project may explicitly configure one target or many targets. + +Generic discovery recognises repository boundary markers but emits only neutral `discovered-boundary` records. It does not infer business role or routing policy from those markers. + +If no recognised marker is found, discovery produces one generic root target. + +Monorepo use is therefore optional. + +## 6. Target graph + +PASS. + +The engine provides: + +- direct dependencies; +- reverse dependants; +- dependency closure; +- affected-target closure; +- verification-target closure. + +Unknown dependencies, self-dependencies, and dependency cycles fail closed. + +Target paths are project-relative and path escape is rejected. + +## 7. Development Contract integration + +PASS. + +The current Development Contract schema advances to **1.3.0** and remains compatible with readable older supported schemas. + +A target-aware contract may persist: + +```json +{ + "registryPath": ".development-kit/workspace.json", + "registryFingerprint": "sha256:...", + "primary": "project-defined-target", + "affected": ["project-defined-target"], + "verification": ["project-defined-target"] +} +``` + +The registry fingerprint is separate from authoritative-source fingerprints but participates in normal Development Contract staleness evaluation. + +This prevents target paths, dependency relationships, commands, capabilities, or adapter configuration from changing silently after target-aware contract approval. + +## 8. Acceptance integration + +PASS. + +No second acceptance mechanism was created. + +Workspace-registry drift flows through the existing: + +```text +checkContractStaleness() + -> acceptance-engine + -> STALE_CONTRACT + -> BLOCKED +``` + +The final focused suite proves a previously acceptable target-aware contract becomes `BLOCKED` after the bound workspace registry changes. + +## 9. T02 bootstrap contract + +PASS with documented bootstrap exception. + +The Development Contract for T02 itself has no target binding because T02 is the increment that creates the Workspace Target Engine. + +Its final observed runtime contract is: + +- Contract ID: `INC-DKF120-T02` +- Contract schema: `1.3.0` +- Source fingerprint: `sha256:80af00ee728b98a0055e9971272d8fc3d6f436c32a6a040109f4e5ffde27bbf2` +- Risk: 2 +- Stale: false +- Target binding: null + +This exception does not apply to later target-aware work after T02 is available. + +## 10. Review findings and corrections + +### REV-T02-001 — Legacy orchestration test pinned the previous contract schema + +**Classification:** compatibility test maintenance +**Status:** Corrected + +The first CI run failed because `orchestration-contract.test.mjs` asserted literal schema version `1.2.0`. + +The production contract correctly emitted `1.3.0`. + +Correction: + +- the regression test now imports `DEVELOPMENT_CONTRACT_SCHEMA_VERSION`; +- it therefore verifies the current canonical contract version rather than a stale literal. + +Correction commit: + +`10e2f6bd6a58deda5e91e7daf43ee328935a9250` + +### REV-T02-002 — New executable script lacked its required documentation reference + +**Classification:** documentation-integrity defect +**Status:** Corrected + +The DKF documentation validator correctly rejected the new T02 contract-generator script because no corresponding reference page existed. + +Correction: + +- added `docs/03-reference/scripts/phase04-workspace-target-engine-contract.md`; +- indexed it in `docs/SUMMARY.md`. + +Final documentation correction head: + +`74a0f56f3c7c0f54bfb2fa2cb4f8973178225b2a` + +### REV-T02-003 — Configuration Readiness integration test pinned schema 1.2.0 + +**Classification:** compatibility test maintenance +**Status:** Corrected + +The focused T02 gate passed, but exact `release:validate` found a second historical test that asserted literal contract schema `1.2.0`. + +Correction: + +- `configuration-readiness-integration.test.mjs` now uses the exported canonical schema version. + +Correction commit: + +`a472a7ab0559bbc5b39c425138cb1f121c8f9b8b` + +### REV-T02-004 — Explicit acceptance-blocking proof added + +**Classification:** verification hardening +**Status:** Completed + +Code review identified that staleness detection should be proven through the final acceptance path, not only through `checkContractStaleness()`. + +A focused integration test was added proving: + +1. valid target-aware contract + PASS verification + PASS code review -> `ACCEPTED`; +2. bound registry changes; +3. same evidence is reevaluated; +4. acceptance becomes `BLOCKED` with `STALE_CONTRACT`. + +Hardening commit: + +`94a10a77d38bdeba57ee398cb533b2d1409afc5c` + +## 11. Final verification + +Focused Workspace Target Engine suite: + +```text +tests: 14 +pass: 14 +fail: 0 +``` + +GitHub Actions run `37668415974`: + +- Ubuntu: PASS; +- Windows: PASS; +- T02 focused gate: PASS on both; +- exact `release:validate`: PASS on both. + +## 12. Outstanding findings + +Critical: **0** +Major: **0** +Unresolved minor: **0** + +## 13. Review verdict + +**PASS** + +The Workspace Target Engine is generic, configuration-driven, contract-aware, fail-closed on target-authority drift, and compatible with the existing DKF acceptance model. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 2f307092..fc61c2f7 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -189,6 +189,16 @@ * [install-platform-adapters](03-reference/scripts/install-platform-adapters.md) * [next-step](03-reference/scripts/next-step.md) * [orchestration](03-reference/scripts/orchestration.md) +* [phase01-scale-context-baseline](03-reference/scripts/phase01-scale-context-baseline.md) +* [phase02-scale-context-plan-validate](03-reference/scripts/phase02-scale-context-plan-validate.md) +* [phase03-cost-observatory-contract](03-reference/scripts/phase03-cost-observatory-contract.md) +* [phase04-workspace-target-engine-contract](03-reference/scripts/phase04-workspace-target-engine-contract.md) +* [phase05-execution-capsule-contract](03-reference/scripts/phase05-execution-capsule-contract.md) +* [phase06-state-engine-v2-contract](03-reference/scripts/phase06-state-engine-v2-contract.md) +* [state-engine-v2-benchmark](03-reference/scripts/state-engine-v2-benchmark.md) +* [t04-hardening-acceptance-gate](03-reference/scripts/t04-hardening-acceptance-gate.md) +* [t04-review-thread-blockers](03-reference/scripts/t04-review-thread-blockers.md) +* [windows-durable-rename](03-reference/scripts/windows-durable-rename.md) * [sync-plugin](03-reference/scripts/sync-plugin.md) * [ui-preview](03-reference/scripts/ui-preview.md) * [token-audit](03-reference/scripts/token-audit.md) @@ -233,6 +243,22 @@ * [DK Token & Context Efficiency Hardening Specification (v0.10.1)](04-architecture/dk-token-context-efficiency-hardening-specification.md) * [DKF Development Modes — v0.11.1 Integration Specification](04-architecture/dkf-development-modes-v0.11.1-integration.md) * [DKF IDEA Authority Hardening — v0.11.1+](04-architecture/dkf-idea-authority-hardening-v0.11.1.md) +* [DKF Scale, Context & Iteration — Phase 0/1 Baseline](04-architecture/dkf-scale-context-iteration-phase01-baseline.md) +* [DKF Scale, Context & Iteration — v0.12 Specification](04-architecture/dkf-scale-context-iteration-v0.12-specification.md) +* [DKF Scale, Context & Iteration — v0.12 Technical Design](04-architecture/dkf-scale-context-iteration-v0.12-technical-design.md) +* [DKF Scale, Context & Iteration — v0.12 Implementation PLAN](04-architecture/dkf-scale-context-iteration-v0.12-implementation-plan.md) +* [DKF Scale, Context & Iteration — Phase 2 Validation](04-architecture/dkf-scale-context-iteration-phase02-validation.md) +* [DKF Cost Observatory — T01 Review](04-architecture/dkf-cost-observatory-t01-review.md) +* [DKF Scale, Context & Iteration — Phase 3 T01 Validation](04-architecture/dkf-scale-context-iteration-phase03-t01-validation.md) +* [DKF Workspace Target Engine — T02 Review](04-architecture/dkf-workspace-target-engine-t02-review.md) +* [DKF Scale, Context & Iteration — Phase 4 T02 Validation](04-architecture/dkf-scale-context-iteration-phase04-t02-validation.md) +* [DKF Execution Capsules & Context Cache — T03 Review](04-architecture/dkf-execution-capsule-t03-review.md) +* [DKF Scale, Context & Iteration — Phase 5 T03 Validation](04-architecture/dkf-scale-context-iteration-phase05-t03-validation.md) +* [DKF State Engine V2 — T04 Review](04-architecture/dkf-state-engine-v2-t04-review.md) +* [DKF Scale, Context & Iteration — Phase 6 T04 Validation](04-architecture/dkf-scale-context-iteration-phase06-t04-validation.md) +* [DKF State Engine V2 — T04 Hardening Validation](04-architecture/dkf-state-engine-v2-t04-hardening-validation.md) +* [DKF State Engine V2 — Offline Recovery Boundary](04-architecture/dkf-state-engine-v2-offline-recovery-boundary.md) +* [DKF T04 — Windows Durability Protocol](04-architecture/dkf-t04-windows-restore-durability-protocol.md) ## 05. Developer Guide * [Local Development Setup](05-developer-guide/local-development-setup.md) diff --git a/memory.md b/memory.md index 2caa172f..c8e50e5e 100644 --- a/memory.md +++ b/memory.md @@ -65,3 +65,241 @@ Integrated through PR #54; Issue #51 is completed. - Green automated tests alone do not authorize publication. - Do not claim post-v0.11.1 mainline functionality is published until version, tag, GitHub Release and package publication gates are explicitly completed. - After any release-impacting change, run the full release-validation/package-consumer gates on the final merge base and verify the published distribution. + +## Scale, Context & Iteration — Phase 0/1 Baseline (7 October 2026) + +- Phase 0/1 analysis is isolated on draft PR #59 / branch `analysis/scale-context-iteration-phase01`; no production runtime behaviour or package version has been changed. +- Live repository baseline is **v0.11.2** at main commit `62badbf7d0abee3344b1f5db2b2fbc876cf004e2`. The earlier `v0.11.0` proposal collides with released history; repository SemVer policy supports **v0.12.0** as the recommended feature-release target. +- Existing Token & Context Efficiency Hardening must be extended, not rebuilt. Current section-aware source delivery achieved 81.89% / 94.59% / 92.53% source-token reduction in representative single-package / multi-package / calculation-heavy fixtures. +- Current state persistence still creates immutable per-transition micro-files. A 26-revision Autopilot fixture produced 26 revision files + 1 pointer file (17,537 bytes), before counting per-run orchestration revisions and other DKF state. +- Current gate selection is risk/security/design aware but has no first-class engineering/numerical domain profile. The calculation-heavy fixture still resolved only specification + tests + code review. +- Current runtime does not centrally measure provider tokens, repository file reads, repository orientation scans, host agent invocations, or host tool calls. These remain unavailable rather than estimated. +- Cross-platform token-audit output differs because the chars/4 estimator counts checkout line endings; future Cost Observatory comparisons should normalize line endings while preserving historical regression compatibility. +- Phase 0/1 baseline CI passed the full Ubuntu + Windows matrix in GitHub Actions run `37618731275`. +- Canonical evidence: `docs/04-architecture/dkf-scale-context-iteration-phase01-baseline.md`. +- Next lifecycle action: reconcile the approved Scale, Context & Iteration specification with the measured baseline, then enter DEFINE/DESIGN/PLAN. Do not begin production implementation before the validated PLAN. + +## Scale, Context & Iteration — Phase 2 (7 October 2026) + +- Phase 2 is complete on draft PR #59 / branch `analysis/scale-context-iteration-phase01`; no production runtime implementation or package-version change has been made. +- Reconciled target release: **Development Kit v0.12.0 — Scale, Context & Iteration**. +- General-purpose architecture is locked: DKF core provides generic mechanisms; project-specific/domain rules live in project configuration, adapters, fixtures, or verification extensions. No app- or industry-specific business logic belongs in DKF core. +- Canonical terminology: **Verification Extension SDK** replaces the provisional “Domain Verification SDK”. +- Workspace targets use arbitrary project-defined IDs; examples such as web/mobile/database/calculations are non-authoritative examples only. Routing is capability/configuration driven. +- Phase 2 artifacts: + - `docs/04-architecture/dkf-scale-context-iteration-v0.12-specification.md` + - `docs/04-architecture/dkf-scale-context-iteration-v0.12-technical-design.md` + - `docs/04-architecture/dkf-scale-context-iteration-v0.12-implementation-plan.md` + - `docs/04-architecture/dkf-scale-context-iteration-v0.12-plan-model.json` + - `docs/04-architecture/dkf-scale-context-iteration-phase02-validation.md` +- Deterministic PLAN result: `valid: true`, 10 tasks, 9 declared dependency edges, all required logical resources owned exactly once, all `DKF-120-AC-001` through `DKF-120-AC-040` criteria covered, `issues: []`. +- PLAN validation passed on Ubuntu and Windows in GitHub Actions run `37659800053`, together with the existing release-validation matrix. +- Approved execution order: + `T01 Cost Observatory -> T02 Workspace Target Engine -> T03 Execution Capsules & Context Cache -> T04 State Engine V2 -> T05 Lifecycle Instances -> T06 Discovery & Controlled Re-entry -> T07 Generic Gate Profiles -> T08 Verification Extension SDK -> T09 Complexity Delta Guard -> T10 Control Center + Comparative Release Validation`. +- Next lifecycle action: create the Development Contract for **DKF120-T01 — Cost Observatory** from the Phase 0/1 baseline, v0.12 specification, technical design, and validated PLAN. Production implementation starts only under that contract. + +## Scale, Context & Iteration — Phase 3 / T01 (7 October 2026) + +- **DKF120-T01 — Cost Observatory is accepted for progression** on branch `feature/v0.12-cost-observatory`. +- Development Contract: `INC-DKF120-T01`, schema 1.2.0, final observed source fingerprint `sha256:5a89b0c0dd841d09cfcbf465ffc36c4c0e768574282c4c94aa389d713231f661`, risk 2, stale=false. +- T01 criteria: `DKF-120-AC-001`, `002`, `003`, `033`. +- Final derived gates: specification + tests; code-reviewer; no specialist control domain; no human approval; no configuration-readiness dependency. +- Implemented generic runtime: `runtime/orchestration/cost-observatory.mjs` + `schemas/cost-record.schema.json`. +- Cost records use newline-normalised logical context measurement, optional provider token observations, null for unavailable host/provider metrics, explicit DKF-controlled counters, elapsed time, append-only `.development-kit/telemetry/cost-records.jsonl`, and baseline/current comparison. +- No application-specific or industry-specific logic is present. T01 remains host/provider independent. +- Focused Cost Observatory suite: 9/9 PASS on Ubuntu; T01 gate PASS on Windows. +- Final full CI for validated T01 head `3b6164dc67a6ceb98ca43e971c2b4636a6c95aa3`: GitHub Actions run `37665136625`, Ubuntu PASS + Windows PASS, including exact `release:validate`. +- Correction history is preserved: + - `REV-T01-001`: volatile acceptance timestamp test assertion corrected. + - `REV-T01-002`: no-credentials architectural boundary moved out of `securityConstraints` so derived gates match the approved PLAN; no actual security gate was bypassed. +- Canonical evidence: + - `docs/04-architecture/dkf-cost-observatory-t01-review.md` + - `docs/04-architecture/dkf-scale-context-iteration-phase03-t01-validation.md` +- v0.12.0 is still unreleased; package version remains 0.11.2 and no merge/publish/release is authorised by T01 acceptance. +- Next validated increment: **DKF120-T02 — Workspace Target Engine**. Create a new Development Contract before implementation. + +## Scale, Context & Iteration — Phase 4 / T02 (7 October 2026) + +- **DKF120-T02 — Workspace Target Engine is accepted for progression** on branch `feature/v0.12-workspace-target-engine`. +- T02 Development Contract: `INC-DKF120-T02`, schema 1.3.0, observed source fingerprint `sha256:80af00ee728b98a0055e9971272d8fc3d6f436c32a6a040109f4e5ffde27bbf2`, risk 2, stale=false. +- T02 acceptance criteria: `DKF-120-AC-004`, `005`, `006`, `007`, `039`. +- Final derived gates: specification + tests; code-reviewer; no specialist control domain; no human approval; no configuration-readiness dependency. +- T02 bootstrap rule: the T02 contract itself is intentionally unbound to workspace targets because it creates the Workspace Target Engine. Target-aware contracts after T02 may bind targets normally. +- New canonical project-local registry: `.development-kit/workspace.json`, Workspace Target Registry schema v1. +- Workspace targets are fully project-defined: arbitrary safe IDs, relative paths, descriptive kind, dependencies, commands, capabilities, verification commands, and opaque adapter metadata. +- DKF core routing is capability/configuration-driven. Target names and `kind` are never dispatch policy. +- Engine provides direct dependencies, reverse dependants, dependency closure, affected closure, verification closure, command resolution, generic repository-boundary discovery, and single-root fallback. +- Invalid paths, unknown target dependencies, and dependency cycles fail closed. +- Development Contract schema advances to **1.3.0** with optional `workspaceTargets` binding: registry path/fingerprint + primary/affected/verification targets. Existing supported schema versions remain readable. +- Workspace-registry fingerprint participates in normal contract staleness. Final integration proof confirms registry drift converts a previously `ACCEPTED` target-aware contract to `BLOCKED` through the existing `STALE_CONTRACT` path. +- Focused Workspace Target Engine suite on implementation head `94a10a77d38bdeba57ee398cb533b2d1409afc5c`: 14/14 PASS. +- Full implementation CI: GitHub Actions run `37668415974`, Ubuntu PASS + Windows PASS, including the T02 gate and exact `release:validate`. +- Correction history preserved: + - `REV-T02-001`: orchestration regression test stopped pinning obsolete Development Contract schema 1.2.0. + - `REV-T02-002`: required reference documentation was added for the T02 contract-generator script. + - `REV-T02-003`: Configuration Readiness integration test stopped pinning obsolete schema 1.2.0. + - `REV-T02-004`: acceptance-path hardening added to prove workspace-registry drift blocks deterministic acceptance. +- Canonical T02 evidence: + - `docs/04-architecture/dkf-workspace-target-engine-t02-review.md` + - `docs/04-architecture/dkf-scale-context-iteration-phase04-t02-validation.md` +- DKF remains general-purpose: no application-specific or industry-specific target semantics entered core. +- v0.12.0 is still unreleased; package version remains 0.11.2; no merge, tag, publish, or release is authorised by T02 acceptance. +- Next validated increment: **DKF120-T03 — Execution Capsules & Context Cache**. T03 must receive its own Development Contract before implementation and should use target binding where applicable. + +## Scale, Context & Iteration — Phase 5 / T03 (7 October 2026) + +- **DKF120-T03 — Execution Capsules & Context Cache is accepted for progression** on branch `feature/v0.12-execution-capsules-context-cache`. +- T03 Development Contract: `INC-DKF120-T03`, schema 1.3.0, risk 3, stale=false. +- Observed T03 source fingerprint: `sha256:db93ea4d721808aa351bfedffb17d5b65ad6defca34b8322089a18d742602baf`. +- T03 is target-aware and binds `dkf-framework` as primary/affected/verification target against workspace registry fingerprint `sha256:b5c3d286ab2b1b65c68e9ac17fbb6f45698986449ffb2129fa4666eff60f9c3e`. +- T03 acceptance criteria: `DKF-120-AC-008`, `009`, `010`, `011`, `035`. +- Risk-3 derived gates: specification + tests; architecture-reviewer; code-reviewer; security-reviewer; security control domain; no human consequential-action approval. +- New runtime primitives: + - `runtime/orchestration/context-cache.mjs` + - `runtime/orchestration/execution-capsule.mjs` + - Execution Capsule schema v1 + - Context Cache Entry schema v1. +- Canonical cache acceleration area: `.development-kit/cache/context/`. It is disposable, rebuildable acceleration state and is never canonical project/business/specification authority. +- Execution Capsules are deterministic reference manifests. They carry contract/task/run/lifecycle references; authoritative source paths/selectors/fingerprints without copied source content; target binding; context-cache fingerprints; compact target facts; relevant file/test fingerprints; target/dependency delta; upstream accepted-task references; verified runtime facts with provenance; and future gate/profile references. +- Runtime facts in capsules are explicitly `non-authoritative-observation`. +- `buildContextPackage()` remains the authoritative role-context constructor. With a fresh capsule it still independently re-reads authoritative sources, checks whole-file fingerprints, and uses existing section-aware materialisation; repository reorientation may come from the validated capsule. +- Capsule-aware isolation metadata distinguishes `authoritativeSourcesReRead=true`, `repositoryReRead=false`, `repositoryContextFromCapsule=true`, and `capsuleFreshnessVerified=true`. +- Deterministic invalidation covers Development Contract/source drift, workspace authority drift, selected-target structural changes, relevant file/test content, persisted Configuration Readiness registry state, missing/changed/corrupt cache entries, and future gate-profile revision input. +- Present changed files are automatically added to the relevant-file fingerprint set so a caller cannot accidentally omit changed content from invalidation. +- Changes in an unrelated unselected target do not invalidate an unrelated target-scoped capsule when contract/workspace authority remains unchanged. +- Cache deletion/corruption causes a safe miss/rebuild; an old capsule referencing missing/stale cache state is rejected rather than trusted. +- Security hardening confines low-level cache reads/writes to `.development-kit/cache/context/`, rejects relevant/changed-file symlink reads, realpath-checks project boundaries, validates cache-entry integrity before capsule creation, and fails closed on stale/tampered cache state. +- T03 correction history: + - `REV-T03-001`: absent Configuration Readiness state originally included a synthetic changing timestamp and caused perpetual false cache misses; fixed by fingerprinting only the persisted registry file when present, absence = stable null. + - `REV-T03-002`: cache semantic identity originally included `createdAt`; timestamp removed from semantic fingerprint while retained as audit metadata. + - `REV-T03-003`: present changed files are now automatically unioned into relevant invalidation inputs. + - `REV-T03-004`: cache I/O and repository-path security boundaries hardened. + - `REV-T03-005`: capsule creation now validates supplied cache-entry integrity directly. +- Focused T03 suite on reviewed implementation head `765a64a577238d73169fc9ae91bb8697a76ded6d`: **19/19 PASS**. +- Implementation CI: GitHub Actions run `37672806212`, Ubuntu PASS + Windows PASS, focused T03 gate PASS on both, exact `release:validate` PASS on both. +- T03 focused two-target Cost Observatory evidence: + - logical context bytes: 9,591 -> 7,053 (**26.46% reduction**); + - estimated context tokens: 2,398 -> 1,764 (**26.44% reduction**); + - target-orientation scans: 2 -> 1 (**50.00% reduction**); + - current comparison used cache hit=1, miss=0. +- These are focused T03 fixture measurements only. They do **not** claim the final v0.12 release objectives (>=40% repeated context reduction, >=60% redundant repository-read reduction, >=70% repeated full orientation reduction) are achieved; release-wide validation remains later work. +- Canonical T03 evidence: + - `docs/04-architecture/dkf-execution-capsule-t03-review.md` + - `docs/04-architecture/dkf-scale-context-iteration-phase05-t03-validation.md` +- Architecture/code/security/security-control review verdicts: PASS; unresolved Critical 0, Major 0, Minor 0. +- DKF remains general-purpose and provider/host/domain neutral. No application-specific cache or capsule semantics entered core. +- v0.12.0 remains unreleased; package version remains 0.11.2; T03 acceptance does not merge to `main`, tag, publish, or release. +- Next validated increment: **DKF120-T04 — State Engine V2**. T04 requires its own Development Contract before production implementation. + +## Scale, Context & Iteration — Phase 6 / T04 (7 October 2026) + +- **DKF120-T04 — State Engine V2 is accepted for progression** on branch `feature/v0.12-state-engine-v2`. +- T04 Development Contract: `INC-DKF120-T04`, schema 1.3.0, risk 3, stale=false. +- Observed T04 source fingerprint: `sha256:f463f9a42340329c2fcf6969559edf48f014ca4159352e983d2cb0c86b2a4729`. +- T04 target binding: `dkf-framework` primary/affected/verification against workspace registry fingerprint `sha256:b5c3d286ab2b1b65c68e9ac17fbb6f45698986449ffb2129fa4666eff60f9c3e`. +- Acceptance criteria: `DKF-120-AC-012`, `013`, `014`, `015`, `034`. +- Risk-3 derived gates: migration + specification + tests; architecture-reviewer; code-reviewer; security-reviewer; security control domain; no human consequential-action approval. +- State Engine V2 canonical project-local layout: + - `.development-kit/state/events.jsonl` — canonical runtime history; + - `.development-kit/state/snapshot.json` — derived materialized state; + - `.development-kit/state/index.db` — disposable/rebuildable acceleration index; + - `.development-kit/state/schema.json` — exact State Engine schema/authority descriptor; + - `.development-kit/state/legacy-backup.json` — recoverable migration bundle when legacy state is migrated. +- State Engine core is generic: entity type/ID, state revision, lifecycle/task/contract/run references, deterministic patch operations, and metadata events. No application/industry business semantics entered core. +- `index.db` in T04 is a deterministic portable JSON-encoded index artifact, deliberately not a Node-22-only SQLite dependency; it is disposable and never canonical authority. +- Canonical events use monotonic sequence, deterministic event IDs, actor class, generic refs, event payload, previous-event hash and SHA-256 event hash. +- Initial entity state is stored once; later state changes are represented as deterministic set/delete path operations rather than one canonical snapshot file per transition. +- Atomic state mutation uses temporary file + fsync + atomic rename under a short project-local State Engine lock. Derived snapshot/index recover from committed canonical events. +- Fresh projects use State Engine V2 immediately for Autopilot and orchestration transition state. +- Existing projects with legacy state remain on the legacy store until a **verified `MIGRATION_COMPLETED` event** records semantic equivalence. Presence of partially imported V2 entities alone cannot trigger cutover. +- Legacy migration validates supported Autopilot and orchestration history, creates a fingerprinted recovery bundle, imports history, proves current semantic equivalence, appends migration completion only after proof, and verifies State Engine integrity. +- Migration is idempotent and remains a no-op even after legitimate post-migration V2 progress, provided the original legacy source fingerprint and recovery bundle still match. +- Legacy migration rejects corrupt JSON, missing revision-chain members, Autopilot workflow-identity changes, orchestration revision gaps, orchestration revision-1/manifest disagreement, final-state disagreement, and other semantic inconsistencies before cutover. +- Legacy source files are retained by T04. T04 does not delete them. +- Recovery bundle can reconstruct the complete retained legacy file set with per-file fingerprint verification. +- Acceptance semantics are preserved: migration fixture retains `state=ACCEPTED`, `acceptanceState=ACCEPTED`, `verificationVerdict=PASS`. +- Integrity hardening includes event mutation/reordering detection, witnessed tail-truncation rejection, exact schema-contract checking, incomplete-layout rejection, and state-root symlink/junction realpath confinement. +- Snapshot deletion/corruption rebuilds from canonical events where no integrity-witness conflict exists. +- Disposable index deletion/rebuild produces semantically equivalent query state. +- T04 correction history: + - `REV-T04-001`: fresh-state regressions updated from legacy pointer-shape assertions to durable V2 persistence invariants. + - `REV-T04-002`: Phase 1 historical legacy baseline decoupled from the live persistence backend. + - `REV-T04-003`: canonical tail truncation can no longer silently rebuild state backwards when a later snapshot witnesses history. + - `REV-T04-004`: State Engine realpath boundary rejects state-root symlink/junction escape. + - `REV-T04-005`: partial imported V2 entities cannot trigger migration cutover. + - `REV-T04-006`: completed migration remains idempotent after normal V2 progress. + - `REV-T04-007`: legacy revision chains must be contiguous and orchestration history anchored to its immutable manifest. + - `REV-T04-008`: same-version schema tampering/incomplete V2 layout now fails closed. +- Focused T04 suite on reviewed implementation head `e3e4b16e372b40042398cfcf210f471e4f762b1b`: **22/22 PASS**. +- Implementation CI: GitHub Actions run `37678375215`, Ubuntu PASS + Windows PASS, focused T04 gate PASS on both, exact `release:validate` PASS on both. +- AC-034 representative Phase 1 result: + - legacy active transition-state files: 27; + - T04 active canonical/recovery State Engine files: 5; + - reduction: **81.48%**, exceeding the >=80% engineering objective. +- Important disclosure: T04 deliberately retains the 27 original legacy files for recovery. During that safety window physical files total 32 (27 retained legacy + 5 State Engine). The 81.48% measurement is the reduction in the new active canonical/recovery representation, not temporary total on-disk file count. +- Canonical T04 evidence: + - `docs/04-architecture/dkf-state-engine-v2-t04-review.md` + - `docs/04-architecture/dkf-scale-context-iteration-phase06-t04-validation.md` +- Required code/architecture/security/security-control review verdicts: PASS; unresolved Critical 0, Major 0, Minor 0. +- DKF remains general-purpose, host/provider/domain neutral. +- v0.12.0 remains unreleased; package version remains 0.11.2; T04 acceptance does not merge to `main`, tag, publish, release, or delete retained legacy state. +- Next validated increment: **DKF120-T05 — Lifecycle Instances**. T05 requires its own Development Contract before production implementation. + +## v0.12 T04 Reliability Hardening — 9 October 2026 + +- Implemented bounded audit repairs on fix/v0.12-t04-state-engine-hardening; stacked PR #70, cumulative draft CI PR #69. Both are unmerged. +- AUD-05 physical event rewrites replaced with journal-backed fsynced append, deterministic recovery and fail-closed conflict detection. Full-history CPU replay remains a performance limitation. +- AUD-06 writers serialize current-state reads, event construction and sequence allocation under a PID/hostname owner-aware lock. Live aged owners are not stolen; verified-dead local owners can be reclaimed. +- AUD-07 restore destination parents/files are symlink/junction protected, with no-follow/exclusive final opens; internal State Engine files are also protected. +- Nine new T04-H01 through T04-H09 tests passed on Windows and Ubuntu. GitHub Actions run 37961832237 passed the full exact release-validation command on both. +- Disposable 300-event endurance fixture: Ubuntu 7,213 ms, Windows 19,986 ms, ledger 189,651 bytes; historical full-rewrite traffic modeled as 28,479,948 bytes (99.33% modeled avoidance). No broad CPU or release-performance claim. +- Canonical new evidence: docs/04-architecture/dkf-state-engine-v2-t04-hardening-validation.md. Existing T04 acceptance authority, approval and v0.11.2 published release remain unchanged. +- Next: independent risk-3 review and deterministic runtime acceptance of the hardening diff prior to merge, then continue T05. Never merge cumulative validation PR #69 as an accidental release. + +### T04 risk-3 regression validation follow-up (9 October 2026) + +- Codex independent review of earlier T04 hardening head detected two open findings: missing JSONL record separator (high) and stale-lock missing-file race (medium). Added red tests H10/H11, fixed and reran CI. +- Additional adversarial review identified concurrent snapshot read false truncation (H12) and legacy restore hardlink overwrite (H13). Added reproducing red tests, fixed both, reran CI. +- Both initial red suites were recorded (runs 37963478049 and 37963732490). Corrected source head ffe49ffb60a1d6eaa132c5caa0f4ff190a5a3351 passed Ubuntu and Windows CI run 37963827500 with H01–H13 tests, exact release:validate and GitGuardian success. +- Re-measured 300-event endurance: 6,068 ms Ubuntu; 16,417 ms Windows; 54/48 ms read/integrity; same 189,651-byte ledger and 99.33% modeled disk rewrite traffic avoided. Measurements are not deterministic latency acceptance. +- Formal independent risk-3 code/architecture/security review re-requested for the updated head. Runtime deterministic acceptance is NOT yet recorded; PR #70 unmerged, v0.12 unreleased, T05 not authorized by this entry. + +### T04 second independent-review remediation (9 October 2026) + +- Codex independent re-review of ffe49ffb60 found 5 new issues: P1 stale-lock recovery atomicity, P1 canonical ledger hardlink write, P1 restore write path TOCTOU, P2 pending journal derived-state recovery, P2 stale snapshot repair racing concurrent writes. +- Implemented exclusive reclamation guard for stale lock files, checked canonical file link counts including opened fd, staged whole-tree legacy restore with root promotion and read-only preflight, derived-state replay before journal deletion, and lock-scoped refresh of stale snapshots/indexes. +- Added H15..H21 adversarial/compatibility tests, in addition to H01..H14. H14 is Windows-only. Tests cover 4-process dead-lock reclaim contention, external event hardlinks, stale snapshot/index after interrupted commit, concurrent reader/writer update, restore open-time hardlink swap, pure preflight, and nonexistent target. +- 37965890821 fully passed Windows and Ubuntu for the prior H15..H19 code revision; latest H20/H21 restore code and documentation require fresh CI verification and current-head independent security/code review before formal T04 acceptance. +- Restore now stages a complete private sibling tree and promotes using root-directory renames; restoration requires a trusted parent and should run without concurrent external writers. Abandoned lock recovery guards require operator inspection, not blind removal. +- PR #70 remains DRAFT, unmerged; main/release remains 0.11.2. Only risk-3 independently validated and accepted T04 may be integrated before T05. + +### T04 independent-review H22/H23 correction (9 October 2026) + +- Third Codex code review (`cf504f7`) identified two additional P1 issues: a symlink swap at the exact canonical events append-open boundary, and lost destination root directory permissions after staged legacy restore. +- Red test H22 recorded external-file modification in Actions run 37967267072. Red test H23 recorded POSIX root 0755->0700 in 37967284714. +- Fixed canonical descriptor opening with O_NOFOLLOW where supported, lstat/fstat inode identity and nlink checks and binding to the observed pre-commit inode+size. Fixed restore promotion to preserve original root mode, timestamps and POSIX UID/GID (fails closed if ownership cannot be set). +- Total hardening tests now H01–H23; H14 Windows-only and H23 POSIX-only. Latest independent review and full CI remain mandatory for the final HEAD. Node filesystem APIs do not guarantee preservation of custom Windows ACLs; this requires explicit host-native verification. +- The stacked PR #70 remains unmerged and 0.11.2 remains published. Do not confuse successful GitHub Actions with a persisted DKF risk-3 acceptance verdict. + +### T04 recovery transaction and offline-boundary hardening (9 October 2026) + +- Implemented mandatory offline attestation for legacy backup restore, an OS-specific BigInt-safe journal for two-step promotion, explicit offline restore recovery, and orphan State Engine recovery-guard owner metadata / operator recovery. +- New tests H24–H28 cover missing authorization, interrupted two-rename transaction, orphan guard, live guard refusal, and real child-process crash recovery. Cross-platform regression baseline for H01–H28: GitHub Actions run 37972680014. +- Added `docs/04-architecture/dkf-state-engine-v2-offline-recovery-boundary.md` describing the explicit trusted-host/offline security model. Node-only cross-platform restore is not a same-UID adversarial security sandbox; the boolean attestation is an operator contract, not OS isolation. +- Current source includes fsync of staged tree and POSIX directory metadata before journaled promotion. Windows custom ACL fidelity must be verified through a host-native procedure before deployment where nonstandard ACLs are involved. +- T04 PR #70 is unmerged; independent code/architecture/security review, final checked HEAD, deterministic acceptance and integration are still pending. Do not conflate prior T04 ACCEPTED evidence with acceptance of these later hardening modifications or claim v0.12 released. + +### T04 fifth security review / FD-bound replay and full metadata preservation (9 October 2026) + +- Codex review of `c80dc253` identified P1 pathname truncate during automatic journal recovery; P1 staged copies changing descendant file ownership; P2 root timestamp drift. Source links are in PR #70. +- Added H29–H31. H29 reproduced actual outside-file truncation on both Ubuntu and Windows on red run 37974023511. Fixed read/truncate/repair write/fsync on one validated no-follow FD. +- Staged restore now inventories descendant metadata before copy and restores POSIX ownership, permissions and timestamps post-order after data durability, rather than updating only the root. Added a privileged **internal-branch only** Ubuntu CI fixture to exercise copying files originally owned by a different Unix UID. +- Current suite includes H01–H31; H14 is Windows-specific and some POSIX metadata fixtures are Windows-skipped as appropriate. Final exact-source CI, fresh independent reviews and the DKF runtime acceptance verdict remain mandatory. +- No main/release/T05 promotion, no legacy deletion; T04 PR #70 remains pending. + +### T04 source-bound risk-3 review and acceptance controls (9 October 2026) + +- T04 hardening H32–H35 adds verbatim relative symlink preservation, fail-closed duplicate/case-folded restoration path checks and malformed UTF-16 surrogate refusal; original contract ten requirements trace explicitly to the same five accepted ACs, with negative Authority Graph tests. +- Earlier deterministic ACCEPTED output from the acceptance script is superseded because reviewer PASS results were generated inside the gate without independently authenticated review. Do not use it as authority. +- CI now validates Ubuntu and Windows before a dependent T04 acceptance job may decide. Gate binds complete checked-out source Git blobs and executable modes to an immutable externally reviewed commit, apart from the review receipt document; role-specific receipts must authenticate genuine distinct Codex review events or clean reactions and match Development Contract fingerprint. Missing reviews must remain BLOCKED. +- PR #70 T04 stacked to feature/v0.12-state-engine-v2; cumulative main-targeted #69 must never be merged as a shortcut. Published version remains 0.11.2. No new tag/release or T05 progression until full deterministic risk-3 acceptance. diff --git a/package.json b/package.json index 00077e15..caefddce 100644 --- a/package.json +++ b/package.json @@ -82,9 +82,14 @@ "token:audit:check": "node scripts/token-audit.mjs --check", "token-efficiency:test": "node --test scripts/token-efficiency.test.mjs", "numbered-decisions:test": "node --test scripts/numbered-decisions.test.mjs scripts/numbered-decisions-adversarial.test.mjs", - "release:validate": "npm run validate && npm run skills:validate:test && npm run doctor && npm run sync:validate:test && npm run docs:validate && npm run docs:validate:test && npm run opencode:validate && npm run platform:validate && npm run research:validate && npm run next-step:test && npm run installer:validate:test && npm run git-bootstrap:test && npm run orchestration:validate && npm run execution-safety:validate && npm run evidence:validate && npm run orchestration-core:validate && npm run orchestration-integration:validate && npm run v09-reliability:validate && npm run intelligence:validate && npm run v071:validate && npm run ui-preview:test && npm run token-efficiency:test && npm run token:audit:check && npm run design-authority:validate && npm run autopilot:validate && npm run numbered-decisions:test && npm run configuration-readiness:validate && npm run development-modes:validate && npm run idea-authority:validate", + "release:validate": "npm run validate && npm run skills:validate:test && npm run doctor && npm run sync:validate:test && npm run docs:validate && npm run docs:validate:test && npm run opencode:validate && npm run platform:validate && npm run research:validate && npm run next-step:test && npm run installer:validate:test && npm run git-bootstrap:test && npm run orchestration:validate && npm run execution-safety:validate && npm run evidence:validate && npm run orchestration-core:validate && npm run orchestration-integration:validate && npm run v09-reliability:validate && npm run intelligence:validate && npm run v071:validate && npm run ui-preview:test && npm run token-efficiency:test && npm run token:audit:check && npm run cost-observatory:test && npm run workspace-targets:test && npm run execution-capsule:test && npm run state-engine-v2:test && npm run design-authority:validate && npm run autopilot:validate && npm run numbered-decisions:test && npm run configuration-readiness:validate && npm run development-modes:validate && npm run idea-authority:validate", "development-modes:validate": "node --test scripts/development-modes.test.mjs scripts/development-modes-init.test.mjs scripts/development-modes-integration.test.mjs", - "idea-authority:validate": "node --test scripts/idea-authority-hardening.test.mjs" + "idea-authority:validate": "node --test scripts/idea-authority-hardening.test.mjs", + "cost-observatory:test": "node scripts/phase03-cost-observatory-contract.mjs && node --test scripts/cost-observatory.test.mjs", + "workspace-targets:test": "node scripts/phase04-workspace-target-engine-contract.mjs && node --test scripts/workspace-targets.test.mjs", + "execution-capsule:test": "node scripts/phase05-execution-capsule-contract.mjs && node --test scripts/context-cache.test.mjs scripts/execution-capsule.test.mjs", + "state-engine-v2:test": "node scripts/phase06-state-engine-v2-contract.mjs && node --test scripts/state-engine-v2.test.mjs scripts/state-engine-v2-migration.test.mjs scripts/state-engine-v2-hardening.test.mjs scripts/t04-review-thread-blockers.test.mjs", + "state-engine-v2:benchmark": "node scripts/state-engine-v2-benchmark.mjs --events 300" }, "engines": { "node": ">=18.0.0" diff --git a/runtime/autopilot/state-store.mjs b/runtime/autopilot/state-store.mjs index 2c6e31e5..03c4e075 100644 --- a/runtime/autopilot/state-store.mjs +++ b/runtime/autopilot/state-store.mjs @@ -1,14 +1,26 @@ /** - * Development Kit Autopilot — Immutable Snapshot State Store + * Development Kit Autopilot — State Store * - * Implements crash-consistent versioned snapshot persistence: - * .development-kit/autopilot/state/revision-XXXXXX.json & current.json. + * State Engine V2 is canonical for: + * - fresh projects with no legacy Autopilot state chain; + * - legacy projects after their workflow has been migrated into V2. + * + * Unmigrated legacy projects continue to use the legacy immutable-snapshot store + * until explicit migration succeeds. This prevents silent history loss. */ import fs from 'node:fs'; import path from 'node:path'; + import { validateWorkflowState } from './validators.mjs'; import { acquireTransactionLock, releaseTransactionLock } from './lock-manager.mjs'; +import { + appendEntityState, + hasCompletedStateMigration, + isStateEngineV2Active, + loadLatestEntityState, + rebuildStateSnapshot, +} from '../orchestration/state-engine-v2.mjs'; export class StateStoreError extends Error { constructor(message) { @@ -21,13 +33,27 @@ export function getStateDir(rootDir = process.cwd()) { return path.join(rootDir, '.development-kit', 'autopilot', 'state'); } -export function getCurrentState(rootDir = process.cwd()) { +function legacyStateExists(rootDir) { + const stateDir = getStateDir(rootDir); + if (!fs.existsSync(stateDir)) return false; + return fs.readdirSync(stateDir).some((name) => /^revision-\d{6}\.json$/.test(name)); +} + +function migratedWorkflowState(rootDir) { + if (!isStateEngineV2Active(rootDir)) return null; + return loadLatestEntityState('autopilot-workflow', rootDir); +} + +function shouldUseV2(rootDir) { + if (!legacyStateExists(rootDir)) return true; + return hasCompletedStateMigration(rootDir) && migratedWorkflowState(rootDir) !== null; +} + +function getLegacyCurrentState(rootDir) { const stateDir = getStateDir(rootDir); const currentFile = path.join(stateDir, 'current.json'); - if (!fs.existsSync(currentFile)) { - return null; - } + if (!fs.existsSync(currentFile)) return null; try { const pointer = JSON.parse(fs.readFileSync(currentFile, 'utf8')); @@ -38,18 +64,27 @@ export function getCurrentState(rootDir = process.cwd()) { return state; } } catch { - // Pointer or state file corrupt — attempt recovery + // Pointer or state file corrupt — attempt legacy recovery. } - return recoverLatestValidState(rootDir); + return recoverLatestLegacyState(rootDir); } -export function saveStateRevision(state, rootDir = process.cwd()) { - validateWorkflowState(state); - const stateDir = getStateDir(rootDir); - if (!fs.existsSync(stateDir)) { - fs.mkdirSync(stateDir, { recursive: true }); +export function getCurrentState(rootDir = process.cwd()) { + if (shouldUseV2(rootDir)) { + const state = migratedWorkflowState(rootDir); + if (state) { + validateWorkflowState(state); + return state; + } + if (!legacyStateExists(rootDir)) return null; } + return getLegacyCurrentState(rootDir); +} + +function saveLegacyStateRevision(state, rootDir) { + const stateDir = getStateDir(rootDir); + if (!fs.existsSync(stateDir)) fs.mkdirSync(stateDir, { recursive: true }); const lock = acquireTransactionLock(rootDir); try { @@ -63,7 +98,6 @@ export function saveStateRevision(state, rootDir = process.cwd()) { fs.writeSync(fd, content, 'utf8'); fs.fsyncSync(fd); fs.closeSync(fd); - fs.renameSync(tmpFilePath, revFilePath); const currentPointerPath = path.join(stateDir, 'current.json'); @@ -72,14 +106,13 @@ export function saveStateRevision(state, rootDir = process.cwd()) { currentRevision: state.stateRevision, currentRevisionFile: revFileName, workflowId: state.workflowId, - updatedAt: new Date().toISOString() + updatedAt: new Date().toISOString(), }, null, 2); const pFd = fs.openSync(tmpPointerPath, 'w'); fs.writeSync(pFd, pointerContent, 'utf8'); fs.fsyncSync(pFd); fs.closeSync(pFd); - fs.renameSync(tmpPointerPath, currentPointerPath); return state; } finally { @@ -87,12 +120,30 @@ export function saveStateRevision(state, rootDir = process.cwd()) { } } -export function recoverLatestValidState(rootDir = process.cwd()) { +export function saveStateRevision(state, rootDir = process.cwd()) { + validateWorkflowState(state); + + if (!shouldUseV2(rootDir)) { + return saveLegacyStateRevision(state, rootDir); + } + + appendEntityState({ + rootDir, + entityType: 'autopilot-workflow', + entityId: state.workflowId, + state, + actorClass: 'autopilot', + timestamp: state.updatedAt, + }); + return state; +} + +function recoverLatestLegacyState(rootDir) { const stateDir = getStateDir(rootDir); if (!fs.existsSync(stateDir)) return null; const files = fs.readdirSync(stateDir) - .filter(f => /^revision-\d{6}\.json$/.test(f)) + .filter((file) => /^revision-\d{6}\.json$/.test(file)) .sort() .reverse(); @@ -102,21 +153,33 @@ export function recoverLatestValidState(rootDir = process.cwd()) { const state = JSON.parse(fs.readFileSync(filePath, 'utf8')); validateWorkflowState(state); - // Repair pointer const pointerContent = JSON.stringify({ currentRevision: state.stateRevision, currentRevisionFile: file, workflowId: state.workflowId, updatedAt: new Date().toISOString(), - recoveredAt: new Date().toISOString() + recoveredAt: new Date().toISOString(), }, null, 2); fs.writeFileSync(path.join(stateDir, 'current.json'), pointerContent, 'utf8'); return state; } catch { - // Try previous revision + // Try previous revision. } } return null; } + +export function recoverLatestValidState(rootDir = process.cwd()) { + if (shouldUseV2(rootDir) && isStateEngineV2Active(rootDir)) { + const current = migratedWorkflowState(rootDir); + if (current) { + rebuildStateSnapshot(rootDir); + const rebuilt = loadLatestEntityState('autopilot-workflow', rootDir); + validateWorkflowState(rebuilt); + return rebuilt; + } + } + return recoverLatestLegacyState(rootDir); +} diff --git a/runtime/orchestration/authority-graph.mjs b/runtime/orchestration/authority-graph.mjs index 3faa4ce7..b4d8d78d 100644 --- a/runtime/orchestration/authority-graph.mjs +++ b/runtime/orchestration/authority-graph.mjs @@ -181,7 +181,14 @@ export function buildAuthorityGraphFromContract({ contract, verification, rootDi const acId = typeof ac === 'string' ? ac : ac?.id; if (acId) { graph.addNode(acId, 'AC', typeof ac === 'object' ? ac : { description: ac }); - if (ac?.requirementId && graph.nodes.has(ac.requirementId)) { + if (Array.isArray(ac?.requirementIds) && ac.requirementIds.length > 0) { + for (const reqId of ac.requirementIds) { + if (!graph.nodes.has(reqId) || graph.nodes.get(reqId).type !== 'REQ') { + throw new AuthorityGraphError('Criterion explicitly references missing requirement: ' + reqId); + } + graph.addEdge(reqId, acId); + } + } else if (ac?.requirementId && graph.nodes.has(ac.requirementId)) { graph.addEdge(ac.requirementId, acId); } else if (contract.requirements && contract.requirements.length > 0) { // Link to first requirement if not explicitly keyed diff --git a/runtime/orchestration/context-cache.mjs b/runtime/orchestration/context-cache.mjs new file mode 100644 index 00000000..ccc506a0 --- /dev/null +++ b/runtime/orchestration/context-cache.mjs @@ -0,0 +1,586 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { + checkContractStaleness, + computeFileFingerprint, + validateDevelopmentContract, +} from './development-contract.mjs'; +import { getRequirementsRegistryPath } from './configuration-readiness.mjs'; +import { loadWorkspaceTargetRegistry } from './workspace-targets.mjs'; + +export const CONTEXT_CACHE_ENTRY_SCHEMA_VERSION = '1.0.0'; +export const CONTEXT_CACHE_ROOT = '.development-kit/cache/context'; + +const FINGERPRINT_PATTERN = /^sha256:[a-f0-9]{64}$/; +const CACHE_IGNORES = Object.freeze([ + '.git', + '.development-kit', + 'node_modules', + 'vendor', + 'dist', + 'build', + 'coverage', + '.next', + '.turbo', + 'out', +]); + +export class ContextCacheError extends Error { + constructor(message, details = null) { + super(message); + this.name = 'ContextCacheError'; + this.details = details; + } +} + +function plainObject(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} + +function stable(value) { + if (Array.isArray(value)) return value.map(stable); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stable(value[key])])); +} + +function sha256(value) { + return `sha256:${createHash('sha256').update(String(value)).digest('hex')}`; +} + +function fingerprintObject(value) { + return sha256(JSON.stringify(stable(value))); +} + +function text(value, label) { + if (typeof value !== 'string' || !value.trim()) { + throw new ContextCacheError(`${label} must be a non-empty string`); + } + return value.trim(); +} + +function stringArray(value = [], label = 'array') { + if (!Array.isArray(value)) throw new ContextCacheError(`${label} must be an array`); + return [...new Set(value.map((item) => text(item, `${label} entry`)))].sort(); +} + +function timestamp(value, label) { + if (typeof value !== 'string' || Number.isNaN(Date.parse(value))) { + throw new ContextCacheError(`${label} must be a valid timestamp`); + } + return value; +} + +function normalizeRelativePath(value, label = 'path') { + const raw = text(value, label).replaceAll('\\', '/'); + if (raw === '.') return '.'; + if (path.posix.isAbsolute(raw) || /^[A-Za-z]:\//.test(raw) || raw.startsWith('//')) { + throw new ContextCacheError(`${label} must be project-relative: ${value}`); + } + const normalized = path.posix.normalize(raw).replace(/^\.\//, '').replace(/\/$/, ''); + if (!normalized || normalized === '.') return '.'; + if (normalized === '..' || normalized.startsWith('../') || normalized.split('/').includes('..')) { + throw new ContextCacheError(`${label} may not escape the project root: ${value}`); + } + return normalized; +} + +function resolveProjectPath(rootDir, relativePath, label = 'path') { + const root = path.resolve(rootDir); + const normalized = normalizeRelativePath(relativePath, label); + const resolved = path.resolve(root, normalized === '.' ? '' : normalized); + const relative = path.relative(root, resolved); + if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new ContextCacheError(`${label} escapes the project root: ${relativePath}`); + } + return { normalized, resolved }; +} + +function assertRealPathInsideRoot(rootDir, candidatePath, label) { + const rootReal = fs.realpathSync(path.resolve(rootDir)); + const candidateReal = fs.realpathSync(candidatePath); + const relative = path.relative(rootReal, candidateReal); + if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new ContextCacheError(`${label} resolves outside the project root`); + } + return candidateReal; +} + +function assertCachePathInsideRoot(rootDir, cachePath) { + const cacheRoot = path.resolve(rootDir, CONTEXT_CACHE_ROOT); + const absolute = path.resolve(cachePath); + const relative = path.relative(cacheRoot, absolute); + if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new ContextCacheError('Context cache path escapes the cache root'); + } + return absolute; +} + +function normalizeFileList(paths = [], label = 'files') { + if (!Array.isArray(paths)) throw new ContextCacheError(`${label} must be an array`); + return [...new Set(paths.map((item) => normalizeRelativePath(item, `${label} entry`)))].sort(); +} + +function normalizeRevision(value, label) { + if (value === undefined || value === null) return null; + return text(value, label); +} + +function configurationFingerprint(rootDir) { + const registryPath = getRequirementsRegistryPath(rootDir); + if (!fs.existsSync(registryPath)) return null; + if (!fs.statSync(registryPath).isFile()) { + throw new ContextCacheError('Configuration readiness registry path is not a file'); + } + return sha256(fs.readFileSync(registryPath)); +} + +function isPathInsideTarget(filePath, targetPath) { + if (targetPath === '.') return true; + return filePath === targetPath || filePath.startsWith(`${targetPath}/`); +} + +function selectedTargetDefinitions(contract, rootDir, targetIds = null) { + validateDevelopmentContract(contract); + + if (!contract.workspaceTargets) { + const requested = targetIds === null ? [] : stringArray(targetIds, 'targetIds'); + if (requested.length > 0 && !(requested.length === 1 && requested[0] === '__project_root__')) { + throw new ContextCacheError('Legacy contracts without workspace target binding may not select named targets'); + } + return { + registryFingerprint: null, + targetIds: ['__project_root__'], + targets: { + __project_root__: { + path: '.', + dependencies: [], + kind: null, + }, + }, + }; + } + + const registry = loadWorkspaceTargetRegistry(rootDir); + const allowed = [...new Set([ + contract.workspaceTargets.primary, + ...contract.workspaceTargets.affected, + ...contract.workspaceTargets.verification, + ])].sort(); + const selected = targetIds === null + ? allowed + : stringArray(targetIds, 'targetIds'); + + if (selected.length === 0) throw new ContextCacheError('At least one target must be selected'); + for (const id of selected) { + if (!allowed.includes(id)) { + throw new ContextCacheError(`Target ${id} is outside the Development Contract target binding`); + } + if (!registry.targets[id]) throw new ContextCacheError(`Workspace target is unavailable: ${id}`); + } + + return { + registryFingerprint: contract.workspaceTargets.registryFingerprint, + targetIds: selected, + targets: Object.fromEntries(selected.map((id) => [id, registry.targets[id]])), + }; +} + +function assertFilesCovered(files, targetDefinitions, label) { + const targetPaths = Object.values(targetDefinitions.targets).map((target) => target.path); + for (const file of files) { + if (!targetPaths.some((targetPath) => isPathInsideTarget(file, targetPath))) { + throw new ContextCacheError(`${label} entry is outside selected targets: ${file}`); + } + } +} + +function collectTargetStructure(rootDir, targetId, target, { + ignoredDirectories = CACHE_IGNORES, +} = {}) { + const ignored = new Set(stringArray(ignoredDirectories, 'ignoredDirectories')); + const { resolved: targetRoot } = resolveProjectPath(rootDir, target.path, `target ${targetId} path`); + if (!fs.existsSync(targetRoot) || !fs.statSync(targetRoot).isDirectory()) { + throw new ContextCacheError(`Target path is not an existing directory: ${target.path}`); + } + assertRealPathInsideRoot(rootDir, targetRoot, `target ${targetId} path`); + + const files = []; + const directories = []; + + function walk(directory) { + const entries = fs.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)); + for (const entry of entries) { + if (entry.isDirectory() && ignored.has(entry.name)) continue; + const absolute = path.join(directory, entry.name); + const projectRelative = path.relative(path.resolve(rootDir), absolute).replaceAll('\\', '/'); + if (entry.isDirectory()) { + directories.push(projectRelative); + walk(absolute); + } else if (entry.isFile() || entry.isSymbolicLink()) { + files.push(projectRelative); + } + } + } + + walk(targetRoot); + const structurePayload = { + targetId, + root: target.path, + directories, + files, + }; + + return Object.freeze({ + targetId, + root: target.path, + kind: target.kind ?? null, + dependencies: [...(target.dependencies ?? [])].sort(), + structureFingerprint: fingerprintObject(structurePayload), + fileCount: files.length, + directoryCount: directories.length, + files, + directories, + }); +} + +function fingerprintFiles(rootDir, files, label) { + return files.map((file) => { + const { resolved } = resolveProjectPath(rootDir, file, label); + if (!fs.existsSync(resolved)) { + throw new ContextCacheError(`${label} is unavailable: ${file}`); + } + const stat = fs.lstatSync(resolved); + if (stat.isSymbolicLink()) { + throw new ContextCacheError(`${label} may not be a symbolic link: ${file}`); + } + if (!stat.isFile()) { + throw new ContextCacheError(`${label} is unavailable: ${file}`); + } + assertRealPathInsideRoot(rootDir, resolved, label); + return Object.freeze({ + path: file, + fingerprint: computeFileFingerprint(rootDir, file), + }); + }); +} + +function buildSelector({ + contract, + targetIds, + relevantFiles, + relevantTests, + gateProfileRevision, +}) { + return Object.freeze({ + contractId: contract.contractId, + taskId: contract.taskId, + targetIds: [...targetIds].sort(), + relevantFiles: [...relevantFiles].sort(), + relevantTests: [...relevantTests].sort(), + gateProfileRevision, + }); +} + +function cachePathFromSelector(rootDir, selector) { + const selectorFingerprint = fingerprintObject(selector); + const safeContract = selector.contractId.replace(/[^A-Za-z0-9._-]/g, '_'); + const filename = `${selectorFingerprint.slice('sha256:'.length, 'sha256:'.length + 32)}.json`; + return { + selectorFingerprint, + cachePath: path.join(rootDir, CONTEXT_CACHE_ROOT, safeContract, filename), + }; +} + +function atomicWrite(filePath, content) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + const tempPath = `${filePath}.tmp-${process.pid}-${Date.now()}`; + fs.writeFileSync(tempPath, content, 'utf8'); + fs.renameSync(tempPath, filePath); +} + +export function captureRepositoryContextInputs({ + contract, + rootDir = process.cwd(), + targetIds = null, + relevantFiles = [], + relevantTests = [], + gateProfileRevision = null, +} = {}) { + validateDevelopmentContract(contract); + + const staleness = checkContractStaleness(contract, rootDir); + if (staleness.stale) { + throw new ContextCacheError('Cannot capture repository context from a stale Development Contract', staleness.changes); + } + + const targetDefinitions = selectedTargetDefinitions(contract, rootDir, targetIds); + const files = normalizeFileList(relevantFiles, 'relevantFiles'); + const tests = normalizeFileList(relevantTests, 'relevantTests'); + assertFilesCovered(files, targetDefinitions, 'relevantFiles'); + assertFilesCovered(tests, targetDefinitions, 'relevantTests'); + + const targetFacts = targetDefinitions.targetIds.map((id) => collectTargetStructure( + rootDir, + id, + targetDefinitions.targets[id], + )); + const fileRefs = fingerprintFiles(rootDir, files, 'relevant file'); + const testRefs = fingerprintFiles(rootDir, tests, 'relevant test'); + + const inputs = { + contractId: contract.contractId, + taskId: contract.taskId, + sourceFingerprint: contract.sourceFingerprint, + developmentModeFingerprint: contract.developmentMode?.fingerprint ?? null, + workspaceRegistryFingerprint: targetDefinitions.registryFingerprint, + configurationFingerprint: configurationFingerprint(rootDir), + gateProfileRevision: normalizeRevision(gateProfileRevision, 'gateProfileRevision'), + targetFacts: targetFacts.map((fact) => ({ + targetId: fact.targetId, + root: fact.root, + dependencies: fact.dependencies, + structureFingerprint: fact.structureFingerprint, + })), + relevantFiles: fileRefs, + relevantTests: testRefs, + }; + + return Object.freeze({ + inputFingerprint: fingerprintObject(inputs), + inputs: Object.freeze(inputs), + targetFacts, + metrics: Object.freeze({ + repositoryReads: fileRefs.length + testRefs.length, + repositoryScans: targetFacts.length, + }), + }); +} + +function entryFingerprintPayload(entry) { + const { + entryFingerprint, + createdAt, + ...payload + } = entry; + return payload; +} + +export function validateContextCacheEntry(entry) { + if (!plainObject(entry)) throw new ContextCacheError('Context cache entry must be an object'); + if (entry.schemaVersion !== CONTEXT_CACHE_ENTRY_SCHEMA_VERSION) { + throw new ContextCacheError(`Unsupported context cache entry schemaVersion: ${entry.schemaVersion}`); + } + for (const field of ['cacheId', 'selectorFingerprint', 'inputFingerprint', 'entryFingerprint']) { + if (typeof entry[field] !== 'string' || !FINGERPRINT_PATTERN.test(entry[field])) { + throw new ContextCacheError(`${field} must be a sha256 fingerprint`); + } + } + timestamp(entry.createdAt, 'createdAt'); + if (!plainObject(entry.selector)) throw new ContextCacheError('selector must be an object'); + if (!plainObject(entry.inputs)) throw new ContextCacheError('inputs must be an object'); + if (!Array.isArray(entry.targetFacts)) throw new ContextCacheError('targetFacts must be an array'); + if (entry.entryFingerprint !== fingerprintObject(entryFingerprintPayload(entry))) { + throw new ContextCacheError('Context cache entry fingerprint does not match content'); + } + return true; +} + +export function createContextCacheEntry({ + selector, + selectorFingerprint, + snapshot, + createdAt = new Date().toISOString(), +} = {}) { + if (!plainObject(selector)) throw new ContextCacheError('selector is required'); + if (!FINGERPRINT_PATTERN.test(selectorFingerprint ?? '')) { + throw new ContextCacheError('selectorFingerprint is invalid'); + } + if (!plainObject(snapshot) || !FINGERPRINT_PATTERN.test(snapshot.inputFingerprint ?? '')) { + throw new ContextCacheError('snapshot is invalid'); + } + timestamp(createdAt, 'createdAt'); + + const payload = { + schemaVersion: CONTEXT_CACHE_ENTRY_SCHEMA_VERSION, + cacheId: fingerprintObject({ + selectorFingerprint, + inputFingerprint: snapshot.inputFingerprint, + }), + selectorFingerprint, + inputFingerprint: snapshot.inputFingerprint, + createdAt, + selector: structuredClone(selector), + inputs: structuredClone(snapshot.inputs), + targetFacts: structuredClone(snapshot.targetFacts), + }; + + const entry = { + ...payload, + entryFingerprint: fingerprintObject(entryFingerprintPayload(payload)), + }; + validateContextCacheEntry(entry); + return Object.freeze(entry); +} + +export function getContextCachePath(rootDir, selector) { + if (!plainObject(selector)) throw new ContextCacheError('selector is required'); + return cachePathFromSelector(rootDir, selector).cachePath; +} + +export function loadContextCacheEntry(cachePath, rootDir = process.cwd()) { + const safePath = assertCachePathInsideRoot(rootDir, cachePath); + if (!fs.existsSync(safePath)) return null; + let entry; + try { + entry = JSON.parse(fs.readFileSync(safePath, 'utf8')); + } catch (error) { + throw new ContextCacheError(`Context cache entry is not valid JSON: ${error.message}`); + } + validateContextCacheEntry(entry); + return entry; +} + +export function persistContextCacheEntry(entry, cachePath, rootDir = process.cwd()) { + validateContextCacheEntry(entry); + const safePath = assertCachePathInsideRoot(rootDir, cachePath); + atomicWrite(safePath, `${JSON.stringify(stable(entry), null, 2)}\n`); + return { cachePath: safePath }; +} + +export function checkContextCacheEntryStaleness({ + entry, + contract, + rootDir = process.cwd(), +} = {}) { + validateContextCacheEntry(entry); + validateDevelopmentContract(contract); + + const snapshot = captureRepositoryContextInputs({ + contract, + rootDir, + targetIds: entry.selector.targetIds, + relevantFiles: entry.selector.relevantFiles, + relevantTests: entry.selector.relevantTests, + gateProfileRevision: entry.selector.gateProfileRevision, + }); + + const changes = []; + if (entry.selector.contractId !== contract.contractId || entry.selector.taskId !== contract.taskId) { + changes.push({ code: 'CONTRACT_IDENTITY_CHANGED' }); + } + if (entry.inputFingerprint !== snapshot.inputFingerprint) { + changes.push({ + code: 'CONTEXT_CACHE_INPUT_CHANGED', + expected: entry.inputFingerprint, + actual: snapshot.inputFingerprint, + }); + } + + return Object.freeze({ + stale: changes.length > 0, + expectedInputFingerprint: entry.inputFingerprint, + currentInputFingerprint: snapshot.inputFingerprint, + changes, + metrics: snapshot.metrics, + }); +} + +export function resolveRepositoryContextCache({ + contract, + rootDir = process.cwd(), + targetIds = null, + relevantFiles = [], + relevantTests = [], + gateProfileRevision = null, + createdAt = new Date().toISOString(), +} = {}) { + validateDevelopmentContract(contract); + timestamp(createdAt, 'createdAt'); + + const targetDefinitions = selectedTargetDefinitions(contract, rootDir, targetIds); + const files = normalizeFileList(relevantFiles, 'relevantFiles'); + const tests = normalizeFileList(relevantTests, 'relevantTests'); + const gateRevision = normalizeRevision(gateProfileRevision, 'gateProfileRevision'); + const selector = buildSelector({ + contract, + targetIds: targetDefinitions.targetIds, + relevantFiles: files, + relevantTests: tests, + gateProfileRevision: gateRevision, + }); + const { selectorFingerprint, cachePath } = cachePathFromSelector(rootDir, selector); + + const snapshot = captureRepositoryContextInputs({ + contract, + rootDir, + targetIds: targetDefinitions.targetIds, + relevantFiles: files, + relevantTests: tests, + gateProfileRevision: gateRevision, + }); + + let existing = null; + let invalidationReason = null; + if (fs.existsSync(cachePath)) { + try { + existing = loadContextCacheEntry(cachePath, rootDir); + if (existing.selectorFingerprint !== selectorFingerprint) { + invalidationReason = 'SELECTOR_FINGERPRINT_CHANGED'; + existing = null; + } else if (existing.inputFingerprint !== snapshot.inputFingerprint) { + invalidationReason = 'INPUT_FINGERPRINT_CHANGED'; + existing = null; + } + } catch (error) { + invalidationReason = 'CACHE_ENTRY_INVALID'; + existing = null; + } + } + + if (existing) { + return Object.freeze({ + status: 'HIT', + cachePath, + selectorFingerprint, + entry: existing, + invalidationReason: null, + metrics: Object.freeze({ + ...snapshot.metrics, + cacheHits: 1, + cacheMisses: 0, + orientationRebuilds: 0, + }), + }); + } + + const entry = createContextCacheEntry({ + selector, + selectorFingerprint, + snapshot, + createdAt, + }); + persistContextCacheEntry(entry, cachePath, rootDir); + + return Object.freeze({ + status: 'MISS', + cachePath, + selectorFingerprint, + entry, + invalidationReason, + metrics: Object.freeze({ + ...snapshot.metrics, + cacheHits: 0, + cacheMisses: 1, + orientationRebuilds: 1, + }), + }); +} + +export function clearContextCache(rootDir = process.cwd()) { + const cacheRoot = path.join(rootDir, CONTEXT_CACHE_ROOT); + fs.rmSync(cacheRoot, { recursive: true, force: true }); +} + +export { CACHE_IGNORES }; diff --git a/runtime/orchestration/context-package.mjs b/runtime/orchestration/context-package.mjs index e6f3b328..474e854e 100644 --- a/runtime/orchestration/context-package.mjs +++ b/runtime/orchestration/context-package.mjs @@ -11,6 +11,7 @@ import { buildTokenProfile, materializeScopedContent, } from './token-efficiency.mjs'; +import { assertExecutionCapsuleFresh } from './execution-capsule.mjs'; const ROLE_PURPOSE = Object.freeze({ implementer: 'implementation', @@ -118,6 +119,7 @@ export function buildContextPackage({ capabilities = {}, separateProcess = false, externalVerifier = false, + executionCapsule = null, createdAt = new Date().toISOString(), } = {}) { validateDevelopmentContract(contract); @@ -139,6 +141,19 @@ export function buildContextPackage({ throw new ContextPackageError('Cannot build context from a stale Development Contract', staleness.changes); } + let capsuleFreshness = null; + if (executionCapsule !== null) { + try { + capsuleFreshness = assertExecutionCapsuleFresh({ + capsule: executionCapsule, + contract, + rootDir, + }); + } catch (error) { + throw new ContextPackageError(`Cannot build context from an invalid or stale Execution Capsule: ${error.message}`, error.details ?? []); + } + } + const sources = contract.authoritativeSources.map((source) => resolveSource(rootDir, source)); const needsDesign = contractNeedsDesignAuthority(contract); const designSource = sources.find((source) => source.kind === 'design-authority' || /(^|\/)design\.md$/i.test(source.path)); @@ -165,12 +180,16 @@ export function buildContextPackage({ createdAt, contract: structuredClone(contract), authoritativeSources: sources, + executionCapsule: executionCapsule === null ? null : structuredClone(executionCapsule), repositoryState: cloneObject(repositoryState, 'repositoryState'), capabilities: cloneObject(capabilities, 'capabilities'), isolationMetadata: { freshContext: isolation === 'fresh', sourceRehydrated: sources.length > 0, - repositoryReRead: true, + authoritativeSourcesReRead: true, + repositoryReRead: executionCapsule === null, + repositoryContextFromCapsule: executionCapsule !== null, + capsuleFreshnessVerified: capsuleFreshness !== null && capsuleFreshness.stale === false, implementationSummaryInherited: implementationReport !== null, separateAgentRole: purpose !== 'implementation', sameModelOrUnknown: true, diff --git a/runtime/orchestration/cost-observatory.mjs b/runtime/orchestration/cost-observatory.mjs new file mode 100644 index 00000000..639367a4 --- /dev/null +++ b/runtime/orchestration/cost-observatory.mjs @@ -0,0 +1,254 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +export const COST_RECORD_SCHEMA_VERSION = '1.0.0'; +export const COST_ESTIMATOR = 'logical-chars-div-4-v1'; + +const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; +const COUNT_FIELDS = Object.freeze([ + 'repositoryReads', + 'repositoryScans', + 'agentInvocations', + 'toolCalls', + 'verificationCommands', + 'cacheHits', + 'cacheMisses', +]); +const OPTIONAL_TOKEN_FIELDS = Object.freeze([ + 'providerInputTokens', + 'providerOutputTokens', + 'providerCachedTokens', +]); + +export class CostObservatoryError extends Error { + constructor(message) { + super(message); + this.name = 'CostObservatoryError'; + } +} + +function requireId(value, label) { + if (typeof value !== 'string' || !ID_PATTERN.test(value)) { + throw new CostObservatoryError(`${label} must be a valid identifier`); + } + return value; +} + +function optionalId(value, label) { + if (value === undefined || value === null) return null; + return requireId(value, label); +} + +function nonNegativeInteger(value, label, { nullable = false } = {}) { + if (value === undefined) return nullable ? null : 0; + if (value === null && nullable) return null; + if (!Number.isInteger(value) || value < 0) { + throw new CostObservatoryError(`${label} must be a non-negative integer${nullable ? ' or null' : ''}`); + } + return value; +} + +function timestamp(value, label) { + if (typeof value !== 'string' || Number.isNaN(Date.parse(value))) { + throw new CostObservatoryError(`${label} must be a valid timestamp`); + } + return value; +} + +export function normalizeLogicalText(value = '') { + return String(value).replace(/\r\n?/g, '\n'); +} + +export function measureLogicalText(value = '') { + const logical = normalizeLogicalText(value); + return Object.freeze({ + logicalChars: logical.length, + logicalBytes: Buffer.byteLength(logical, 'utf8'), + estimatedTokens: Math.ceil(logical.length / 4), + estimator: COST_ESTIMATOR, + }); +} + +export function createCostMeter(initial = {}) { + const counts = Object.fromEntries( + COUNT_FIELDS.map((field) => [field, nonNegativeInteger(initial[field], field)]), + ); + + return { + increment(field, amount = 1) { + if (!COUNT_FIELDS.includes(field)) throw new CostObservatoryError(`Unsupported counter: ${field}`); + const delta = nonNegativeInteger(amount, 'increment amount'); + counts[field] += delta; + return counts[field]; + }, + snapshot() { + return Object.freeze({ ...counts }); + }, + }; +} + +export function createCostRecord({ + operationId, + lifecycleId = null, + contractId = null, + taskId = null, + runId = null, + role = null, + context = '', + providerUsage = null, + counters = {}, + startedAt, + completedAt, +} = {}) { + requireId(operationId, 'operationId'); + const start = timestamp(startedAt, 'startedAt'); + const end = timestamp(completedAt, 'completedAt'); + if (Date.parse(end) < Date.parse(start)) { + throw new CostObservatoryError('completedAt may not be before startedAt'); + } + + const contextMetric = measureLogicalText(context); + const provider = providerUsage === null || providerUsage === undefined ? {} : providerUsage; + if (typeof provider !== 'object' || Array.isArray(provider)) { + throw new CostObservatoryError('providerUsage must be an object, null, or undefined'); + } + + const record = { + schemaVersion: COST_RECORD_SCHEMA_VERSION, + operationId, + lifecycleId: optionalId(lifecycleId, 'lifecycleId'), + contractId: optionalId(contractId, 'contractId'), + taskId: optionalId(taskId, 'taskId'), + runId: optionalId(runId, 'runId'), + role: role === null || role === undefined ? null : requireId(role, 'role'), + contextBytes: contextMetric.logicalBytes, + estimatedContextTokens: contextMetric.estimatedTokens, + contextEstimator: contextMetric.estimator, + providerInputTokens: nonNegativeInteger(provider.inputTokens, 'providerUsage.inputTokens', { nullable: true }), + providerOutputTokens: nonNegativeInteger(provider.outputTokens, 'providerUsage.outputTokens', { nullable: true }), + providerCachedTokens: nonNegativeInteger(provider.cachedTokens, 'providerUsage.cachedTokens', { nullable: true }), + repositoryReads: nonNegativeInteger(counters.repositoryReads, 'repositoryReads', { nullable: true }), + repositoryScans: nonNegativeInteger(counters.repositoryScans, 'repositoryScans', { nullable: true }), + agentInvocations: nonNegativeInteger(counters.agentInvocations, 'agentInvocations', { nullable: true }), + toolCalls: nonNegativeInteger(counters.toolCalls, 'toolCalls', { nullable: true }), + verificationCommands: nonNegativeInteger(counters.verificationCommands, 'verificationCommands', { nullable: true }), + cacheHits: nonNegativeInteger(counters.cacheHits, 'cacheHits'), + cacheMisses: nonNegativeInteger(counters.cacheMisses, 'cacheMisses'), + startedAt: start, + completedAt: end, + elapsedMs: Date.parse(end) - Date.parse(start), + }; + + validateCostRecord(record); + return Object.freeze(record); +} + +export function validateCostRecord(record) { + if (!record || typeof record !== 'object' || Array.isArray(record)) { + throw new CostObservatoryError('Cost record must be an object'); + } + if (record.schemaVersion !== COST_RECORD_SCHEMA_VERSION) { + throw new CostObservatoryError(`Unsupported cost record schemaVersion: ${record.schemaVersion}`); + } + requireId(record.operationId, 'operationId'); + for (const [field, label] of [ + ['lifecycleId', 'lifecycleId'], + ['contractId', 'contractId'], + ['taskId', 'taskId'], + ['runId', 'runId'], + ['role', 'role'], + ]) { + optionalId(record[field], label); + } + nonNegativeInteger(record.contextBytes, 'contextBytes'); + nonNegativeInteger(record.estimatedContextTokens, 'estimatedContextTokens'); + if (record.contextEstimator !== COST_ESTIMATOR) { + throw new CostObservatoryError(`Unsupported contextEstimator: ${record.contextEstimator}`); + } + for (const field of OPTIONAL_TOKEN_FIELDS) nonNegativeInteger(record[field], field, { nullable: true }); + for (const field of COUNT_FIELDS) { + const nullable = !['cacheHits', 'cacheMisses'].includes(field); + nonNegativeInteger(record[field], field, { nullable }); + } + timestamp(record.startedAt, 'startedAt'); + timestamp(record.completedAt, 'completedAt'); + nonNegativeInteger(record.elapsedMs, 'elapsedMs'); + if (Date.parse(record.completedAt) - Date.parse(record.startedAt) !== record.elapsedMs) { + throw new CostObservatoryError('elapsedMs does not match startedAt/completedAt'); + } + return true; +} + +export function compareCostRecords(baseline, current) { + validateCostRecord(baseline); + validateCostRecord(current); + const fields = [ + 'contextBytes', + 'estimatedContextTokens', + 'providerInputTokens', + 'providerOutputTokens', + 'providerCachedTokens', + 'repositoryReads', + 'repositoryScans', + 'agentInvocations', + 'toolCalls', + 'verificationCommands', + 'cacheHits', + 'cacheMisses', + 'elapsedMs', + ]; + + const metrics = {}; + for (const field of fields) { + const before = baseline[field]; + const after = current[field]; + if (before === null || after === null || before === undefined || after === undefined) { + metrics[field] = { before: before ?? null, after: after ?? null, delta: null, reductionPercent: null, available: false }; + continue; + } + const delta = after - before; + const reductionPercent = before === 0 + ? null + : Number((((before - after) / before) * 100).toFixed(2)); + metrics[field] = { before, after, delta, reductionPercent, available: true }; + } + + return Object.freeze({ + schemaVersion: '1.0.0', + baselineOperationId: baseline.operationId, + currentOperationId: current.operationId, + metrics, + }); +} + +export function getCostLedgerPath(rootDir = process.cwd()) { + return path.join(rootDir, '.development-kit', 'telemetry', 'cost-records.jsonl'); +} + +export function appendCostRecord(record, rootDir = process.cwd()) { + validateCostRecord(record); + const ledgerPath = getCostLedgerPath(rootDir); + fs.mkdirSync(path.dirname(ledgerPath), { recursive: true }); + fs.appendFileSync(ledgerPath, `${JSON.stringify(record)}\n`, 'utf8'); + return { ledgerPath }; +} + +export function loadCostRecords(rootDir = process.cwd()) { + const ledgerPath = getCostLedgerPath(rootDir); + if (!fs.existsSync(ledgerPath)) return []; + const content = fs.readFileSync(ledgerPath, 'utf8'); + const records = content + .split(/\r?\n/) + .filter(Boolean) + .map((line, index) => { + let record; + try { + record = JSON.parse(line); + } catch { + throw new CostObservatoryError(`Invalid cost ledger JSON at line ${index + 1}`); + } + validateCostRecord(record); + return record; + }); + return records; +} diff --git a/runtime/orchestration/development-contract.mjs b/runtime/orchestration/development-contract.mjs index 59385835..0f61891f 100644 --- a/runtime/orchestration/development-contract.mjs +++ b/runtime/orchestration/development-contract.mjs @@ -5,9 +5,14 @@ import { captureDevelopmentModeSnapshot, validateDevelopmentModeSnapshot, } from '../development-modes/integration.mjs'; +import { + checkWorkspaceTargetBindingStaleness, + resolveWorkspaceTargetBinding, + validatePersistedWorkspaceTargetBinding, +} from './workspace-targets.mjs'; -export const DEVELOPMENT_CONTRACT_SCHEMA_VERSION = '1.2.0'; -export const SUPPORTED_DEVELOPMENT_CONTRACT_SCHEMA_VERSIONS = Object.freeze(['1.0.0', '1.1.0', '1.2.0']); +export const DEVELOPMENT_CONTRACT_SCHEMA_VERSION = '1.3.0'; +export const SUPPORTED_DEVELOPMENT_CONTRACT_SCHEMA_VERSIONS = Object.freeze(['1.0.0', '1.1.0', '1.2.0', '1.3.0']); export const DEFAULT_CORRECTION_ATTEMPTS = 3; const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; @@ -35,6 +40,7 @@ const CONTRACT_KEYS = new Set([ 'approvalPolicy', 'configurationDependencies', 'developmentMode', + 'workspaceTargets', 'sourceFingerprint', ]); @@ -298,11 +304,16 @@ export function normalizeAcceptanceCriteria(criteria = []) { } const requirementId = typeof value.requirementId === 'string' && value.requirementId.trim() ? value.requirementId.trim() : null; + // Multi-requirement mapping is explicit, audited source authority: a + // single acceptance criterion can verify several distinct requirements. + const requirementIds = normalizeStringArray(value.requirementIds ?? + (requirementId ? [requirementId] : [])); return { id, statement, requirementId, + requirementIds, source, verificationType, requiredEvidence: value.requiredEvidence !== false, @@ -361,6 +372,11 @@ export function createDevelopmentContract({ throw new ContractValidationError('risk.level must be an integer from 0 to 4'); } + const workspaceTargetsInput = task.workspaceTargets ?? task.targetBinding ?? null; + const workspaceTargets = workspaceTargetsInput === null + ? null + : resolveWorkspaceTargetBinding(rootDir, workspaceTargetsInput); + const contract = { schemaVersion: DEVELOPMENT_CONTRACT_SCHEMA_VERSION, contractId: resolvedContractId, @@ -396,6 +412,10 @@ export function createDevelopmentContract({ sourceFingerprint: computeSourceFingerprint(sources), }; + if (workspaceTargets !== null) { + contract.workspaceTargets = structuredClone(workspaceTargets); + } + if (Array.isArray(task.configurationDependencies)) { contract.configurationDependencies = structuredClone(task.configurationDependencies); } @@ -476,7 +496,7 @@ export function validateDevelopmentContract(contract) { if (!Array.isArray(contract.acceptanceCriteria) || contract.acceptanceCriteria.length === 0) { errors.push('acceptanceCriteria must contain at least one criterion'); } else { - const criterionKeys = new Set(['id', 'statement', 'requirementId', 'source', 'verificationType', 'requiredEvidence']); + const criterionKeys = new Set(['id', 'statement', 'requirementId', 'requirementIds', 'source', 'verificationType', 'requiredEvidence']); const criterionIds = new Set(); for (const criterion of contract.acceptanceCriteria) { if (!isPlainObject(criterion)) { @@ -484,6 +504,22 @@ export function validateDevelopmentContract(contract) { continue; } assertNoExtraKeys(criterion, criterionKeys, 'acceptance criterion', errors); + if (criterion.requirementIds !== undefined) { + validateStringArray(criterion.requirementIds, 'acceptance criterion requirementIds', errors); + if (Array.isArray(criterion.requirementIds) && Array.isArray(contract.requirements)) { + for (const req of criterion.requirementIds) { + if (typeof req !== 'string' || !req.trim()) continue; + const matches = contract.requirements.some(item => { + if (typeof item === 'string') return item === req; + if (isPlainObject(item) && typeof item.id === 'string') return item.id === req; + return false; + }); + if (!matches) { + errors.push('Acceptance criterion references unknown requirement: ' + req); + } + } + } + } if (typeof criterion.id !== 'string' || !IDENTIFIER_PATTERN.test(criterion.id)) errors.push('Acceptance criterion id is invalid'); if (criterionIds.has(criterion.id)) errors.push(`Duplicate acceptance criterion id: ${criterion.id}`); criterionIds.add(criterion.id); @@ -525,8 +561,8 @@ export function validateDevelopmentContract(contract) { if (!isPlainObject(contract.approvalPolicy)) errors.push('approvalPolicy must be an object'); - if (contract.schemaVersion === '1.2.0' && contract.developmentMode === undefined) { - errors.push('developmentMode is required for schemaVersion 1.2.0'); + if (['1.2.0', '1.3.0'].includes(contract.schemaVersion) && contract.developmentMode === undefined) { + errors.push(`developmentMode is required for schemaVersion ${contract.schemaVersion}`); } if (contract.developmentMode !== undefined) { try { @@ -536,6 +572,14 @@ export function validateDevelopmentContract(contract) { } } + if (contract.workspaceTargets !== undefined) { + try { + validatePersistedWorkspaceTargetBinding(contract.workspaceTargets); + } catch (error) { + errors.push(`Invalid workspace target binding: ${error.message}`); + } + } + if (contract.configurationDependencies !== undefined) { if (!Array.isArray(contract.configurationDependencies)) { errors.push('configurationDependencies must be an array'); @@ -625,6 +669,18 @@ export function renderDevelopmentContractMarkdown(contract) { ] : ['- Legacy contract without a mode snapshot']), '', + ...(contract.workspaceTargets + ? [ + '## Workspace Targets', + '', + `- Registry: \`${contract.workspaceTargets.registryPath}\``, + `- Registry fingerprint: \`${contract.workspaceTargets.registryFingerprint}\``, + `- Primary: **${contract.workspaceTargets.primary}**`, + `- Affected: ${contract.workspaceTargets.affected.join(', ') || 'none'}`, + `- Verification: ${contract.workspaceTargets.verification.join(', ') || 'none'}`, + '', + ] + : []), '## Execution Safety', '', `- Resource scope: **${contract.executionSafety.resourceScope}**`, @@ -714,10 +770,24 @@ export function checkContractStaleness(contract, rootDir = process.cwd()) { }); const currentSourceFingerprint = computeSourceFingerprint(currentSources); + + let workspaceTargetState = null; + if (contract.workspaceTargets) { + workspaceTargetState = checkWorkspaceTargetBindingStaleness(contract.workspaceTargets, rootDir); + for (const change of workspaceTargetState.changes) { + changes.push({ + path: contract.workspaceTargets.registryPath, + status: 'workspace-target-change', + ...change, + }); + } + } + return { stale: changes.length > 0 || currentSourceFingerprint !== contract.sourceFingerprint, expectedSourceFingerprint: contract.sourceFingerprint, currentSourceFingerprint, + workspaceTargets: workspaceTargetState, changes, }; } diff --git a/runtime/orchestration/execution-capsule.mjs b/runtime/orchestration/execution-capsule.mjs new file mode 100644 index 00000000..628ba2b9 --- /dev/null +++ b/runtime/orchestration/execution-capsule.mjs @@ -0,0 +1,538 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { + checkContractStaleness, + validateDevelopmentContract, +} from './development-contract.mjs'; +import { + checkContextCacheEntryStaleness, + loadContextCacheEntry, + resolveRepositoryContextCache, + validateContextCacheEntry, +} from './context-cache.mjs'; +import { + getAffectedTargetClosure, + getTargetDependencyClosure, + loadWorkspaceTargetRegistry, +} from './workspace-targets.mjs'; + +export const EXECUTION_CAPSULE_SCHEMA_VERSION = '1.0.0'; + +const FINGERPRINT_PATTERN = /^sha256:[a-f0-9]{64}$/; +const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; + +export class ExecutionCapsuleError extends Error { + constructor(message, details = null) { + super(message); + this.name = 'ExecutionCapsuleError'; + this.details = details; + } +} + +function plainObject(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} + +function stable(value) { + if (Array.isArray(value)) return value.map(stable); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stable(value[key])])); +} + +function fingerprintObject(value) { + return `sha256:${createHash('sha256').update(JSON.stringify(stable(value))).digest('hex')}`; +} + +function text(value, label) { + if (typeof value !== 'string' || !value.trim()) throw new ExecutionCapsuleError(`${label} must be a non-empty string`); + return value.trim(); +} + +function optionalId(value, label) { + if (value === undefined || value === null) return null; + const normalized = text(value, label); + if (!ID_PATTERN.test(normalized)) throw new ExecutionCapsuleError(`${label} has unsupported characters`); + return normalized; +} + +function stringArray(value = [], label = 'array') { + if (!Array.isArray(value)) throw new ExecutionCapsuleError(`${label} must be an array`); + return [...new Set(value.map((item) => text(item, `${label} entry`)))].sort(); +} + +function timestamp(value, label) { + if (typeof value !== 'string' || Number.isNaN(Date.parse(value))) { + throw new ExecutionCapsuleError(`${label} must be a valid timestamp`); + } + return value; +} + +function sha(value, label) { + if (typeof value !== 'string' || !FINGERPRINT_PATTERN.test(value)) { + throw new ExecutionCapsuleError(`${label} must be a sha256 fingerprint`); + } + return value; +} + +function normalizeRelativePath(value, label = 'path') { + const raw = text(value, label).replaceAll('\\', '/'); + if (path.posix.isAbsolute(raw) || /^[A-Za-z]:\//.test(raw) || raw.startsWith('//')) { + throw new ExecutionCapsuleError(`${label} must be project-relative`); + } + const normalized = path.posix.normalize(raw).replace(/^\.\//, ''); + if (normalized === '..' || normalized.startsWith('../') || normalized.split('/').includes('..')) { + throw new ExecutionCapsuleError(`${label} may not escape the project root`); + } + return normalized || '.'; +} + +function fileRefs(value = [], label = 'fileRefs') { + if (!Array.isArray(value)) throw new ExecutionCapsuleError(`${label} must be an array`); + return value.map((ref) => { + if (!plainObject(ref)) throw new ExecutionCapsuleError(`${label} entries must be objects`); + return Object.freeze({ + path: normalizeRelativePath(ref.path, `${label}.path`), + fingerprint: sha(ref.fingerprint, `${label}.fingerprint`), + }); + }).sort((a, b) => a.path.localeCompare(b.path)); +} + +function normalizeUpstreamTasks(value = []) { + if (!Array.isArray(value)) throw new ExecutionCapsuleError('upstreamAcceptedTasks must be an array'); + return value.map((item) => { + if (!plainObject(item)) throw new ExecutionCapsuleError('upstreamAcceptedTasks entries must be objects'); + return Object.freeze({ + taskId: optionalId(item.taskId, 'upstream taskId'), + contractId: optionalId(item.contractId, 'upstream contractId'), + sourceFingerprint: sha(item.sourceFingerprint, 'upstream sourceFingerprint'), + acceptanceFingerprint: sha(item.acceptanceFingerprint, 'upstream acceptanceFingerprint'), + }); + }).sort((a, b) => `${a.taskId}:${a.contractId}`.localeCompare(`${b.taskId}:${b.contractId}`)); +} + +function normalizeRuntimeFacts(value = []) { + if (!Array.isArray(value)) throw new ExecutionCapsuleError('verifiedRuntimeFacts must be an array'); + return value.map((fact) => { + if (!plainObject(fact)) throw new ExecutionCapsuleError('verifiedRuntimeFacts entries must be objects'); + if (!plainObject(fact.provenance)) throw new ExecutionCapsuleError('Runtime fact provenance is required'); + const factId = optionalId(fact.id, 'runtime fact id'); + if (factId === null) throw new ExecutionCapsuleError('Runtime fact id is required'); + const normalized = { + id: factId, + value: structuredClone(fact.value), + observedAt: timestamp(fact.observedAt, 'runtime fact observedAt'), + authority: 'non-authoritative-observation', + provenance: { + kind: text(fact.provenance.kind, 'runtime fact provenance.kind'), + id: text(fact.provenance.id, 'runtime fact provenance.id'), + fingerprint: sha(fact.provenance.fingerprint, 'runtime fact provenance.fingerprint'), + }, + }; + JSON.stringify(normalized.value); + return Object.freeze(normalized); + }).sort((a, b) => a.id.localeCompare(b.id)); +} + +function pathInside(filePath, targetPath) { + if (targetPath === '.') return true; + return filePath === targetPath || filePath.startsWith(`${targetPath}/`); +} + +function changedFileRefs(rootDir, changedFiles = []) { + const normalized = stringArray(changedFiles, 'changedFiles').map((file) => normalizeRelativePath(file, 'changed file')); + return normalized.map((file) => { + const absolute = path.resolve(rootDir, file); + const root = path.resolve(rootDir); + const relative = path.relative(root, absolute); + if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new ExecutionCapsuleError(`Changed file escapes project root: ${file}`); + } + if (!fs.existsSync(absolute)) { + return Object.freeze({ path: file, status: 'deleted-or-missing', fingerprint: null }); + } + const stat = fs.lstatSync(absolute); + if (stat.isSymbolicLink()) { + throw new ExecutionCapsuleError(`Changed path may not be a symbolic link: ${file}`); + } + if (!stat.isFile()) { + throw new ExecutionCapsuleError(`Changed path is not a file: ${file}`); + } + const rootReal = fs.realpathSync(root); + const fileReal = fs.realpathSync(absolute); + const realRelative = path.relative(rootReal, fileReal); + if (realRelative === '..' || realRelative.startsWith(`..${path.sep}`) || path.isAbsolute(realRelative)) { + throw new ExecutionCapsuleError(`Changed file resolves outside project root: ${file}`); + } + const content = fs.readFileSync(absolute); + return Object.freeze({ path: file, status: 'present', fingerprint: `sha256:${createHash('sha256').update(content).digest('hex')}` }); + }); +} + +function resolveTargetDelta(contract, rootDir, changedFiles, cacheEntry) { + const changedRefs = changedFileRefs(rootDir, changedFiles); + + if (!contract.workspaceTargets) { + return Object.freeze({ + changedFiles: changedRefs, + changedTargets: changedRefs.length > 0 ? ['__project_root__'] : [], + affectedTargets: changedRefs.length > 0 ? ['__project_root__'] : [], + dependencyTargets: ['__project_root__'], + }); + } + + const registry = loadWorkspaceTargetRegistry(rootDir); + const bound = [...new Set([ + contract.workspaceTargets.primary, + ...contract.workspaceTargets.affected, + ...contract.workspaceTargets.verification, + ])].sort(); + + const changedTargets = new Set(); + for (const file of changedRefs) { + const candidates = bound + .filter((id) => registry.targets[id] && pathInside(file.path, registry.targets[id].path)) + .sort((a, b) => { + const pathA = registry.targets[a].path === '.' ? '' : registry.targets[a].path; + const pathB = registry.targets[b].path === '.' ? '' : registry.targets[b].path; + return pathB.length - pathA.length || a.localeCompare(b); + }); + if (candidates.length === 0) { + throw new ExecutionCapsuleError(`Changed file is outside the Development Contract target binding: ${file.path}`); + } + changedTargets.add(candidates[0]); + } + + const selectedTargets = cacheEntry.selector.targetIds; + const dependencyTargets = getTargetDependencyClosure(registry, selectedTargets); + const affectedTargets = changedTargets.size > 0 + ? getAffectedTargetClosure(registry, [...changedTargets]) + : []; + + return Object.freeze({ + changedFiles: changedRefs, + changedTargets: [...changedTargets].sort(), + affectedTargets, + dependencyTargets, + }); +} + +function normalizeAuthoritativeSources(contract) { + return contract.authoritativeSources.map((source) => Object.freeze({ + path: source.path, + kind: source.kind, + authority: source.authority, + sections: [...source.sections], + fingerprint: source.fingerprint, + })); +} + +function normalizeTargetFacts(entry) { + return entry.targetFacts.map((fact) => Object.freeze({ + targetId: fact.targetId, + root: fact.root, + kind: fact.kind ?? null, + dependencies: [...fact.dependencies], + structureFingerprint: fact.structureFingerprint, + fileCount: fact.fileCount, + directoryCount: fact.directoryCount, + })); +} + +function relativeCachePath(rootDir, cachePath) { + const root = path.resolve(rootDir); + const absolute = path.resolve(cachePath); + const relative = path.relative(root, absolute).replaceAll('\\', '/'); + if (relative === '..' || relative.startsWith('../') || path.isAbsolute(relative)) { + throw new ExecutionCapsuleError('Context cache path escapes project root'); + } + return relative; +} + +function semanticCapsulePayload(capsule) { + const { + capsuleId, + capsuleFingerprint, + createdAt, + ...semantic + } = capsule; + return semantic; +} + +export function validateExecutionCapsule(capsule) { + if (!plainObject(capsule)) throw new ExecutionCapsuleError('Execution Capsule must be an object'); + if (capsule.schemaVersion !== EXECUTION_CAPSULE_SCHEMA_VERSION) { + throw new ExecutionCapsuleError(`Unsupported Execution Capsule schemaVersion: ${capsule.schemaVersion}`); + } + if (typeof capsule.capsuleId !== 'string' || !/^CAP-[a-f0-9]{20}$/.test(capsule.capsuleId)) { + throw new ExecutionCapsuleError('capsuleId is invalid'); + } + optionalId(capsule.contractId, 'contractId'); + optionalId(capsule.taskId, 'taskId'); + optionalId(capsule.runId, 'runId'); + optionalId(capsule.lifecycleId, 'lifecycleId'); + sha(capsule.sourceFingerprint, 'sourceFingerprint'); + if (capsule.developmentModeFingerprint !== null) sha(capsule.developmentModeFingerprint, 'developmentModeFingerprint'); + timestamp(capsule.createdAt, 'createdAt'); + + if (!Array.isArray(capsule.authoritativeSources) || capsule.authoritativeSources.length === 0) { + throw new ExecutionCapsuleError('authoritativeSources must be a non-empty array'); + } + for (const source of capsule.authoritativeSources) { + normalizeRelativePath(source.path, 'authoritative source path'); + sha(source.fingerprint, 'authoritative source fingerprint'); + } + + if (!plainObject(capsule.repositoryContext)) throw new ExecutionCapsuleError('repositoryContext is required'); + for (const field of ['cacheId', 'selectorFingerprint', 'inputFingerprint', 'entryFingerprint']) { + sha(capsule.repositoryContext[field], `repositoryContext.${field}`); + } + normalizeRelativePath(capsule.repositoryContext.cachePath, 'repositoryContext.cachePath'); + fileRefs(capsule.repositoryContext.relevantFiles, 'repositoryContext.relevantFiles'); + fileRefs(capsule.repositoryContext.relevantTests, 'repositoryContext.relevantTests'); + if (!Array.isArray(capsule.repositoryContext.targets)) throw new ExecutionCapsuleError('repositoryContext.targets must be an array'); + + if (!plainObject(capsule.targetDelta)) throw new ExecutionCapsuleError('targetDelta is required'); + stringArray(capsule.targetDelta.changedTargets, 'targetDelta.changedTargets'); + stringArray(capsule.targetDelta.affectedTargets, 'targetDelta.affectedTargets'); + stringArray(capsule.targetDelta.dependencyTargets, 'targetDelta.dependencyTargets'); + if (!Array.isArray(capsule.targetDelta.changedFiles)) throw new ExecutionCapsuleError('targetDelta.changedFiles must be an array'); + + normalizeUpstreamTasks(capsule.upstreamAcceptedTasks); + normalizeRuntimeFacts(capsule.verifiedRuntimeFacts); + stringArray(capsule.requiredGateProfileIds, 'requiredGateProfileIds'); + + const expectedFingerprint = fingerprintObject(semanticCapsulePayload(capsule)); + if (capsule.capsuleFingerprint !== expectedFingerprint) { + throw new ExecutionCapsuleError('Execution Capsule fingerprint does not match content'); + } + if (capsule.capsuleId !== `CAP-${expectedFingerprint.slice('sha256:'.length, 'sha256:'.length + 20)}`) { + throw new ExecutionCapsuleError('Execution Capsule ID does not match fingerprint'); + } + return true; +} + +export function createExecutionCapsule({ + contract, + rootDir = process.cwd(), + cacheResolution, + runId = null, + lifecycleId = null, + changedFiles = [], + upstreamAcceptedTasks = [], + verifiedRuntimeFacts = [], + requiredGateProfileIds = [], + createdAt = new Date().toISOString(), +} = {}) { + validateDevelopmentContract(contract); + if (!plainObject(cacheResolution) || !plainObject(cacheResolution.entry)) { + throw new ExecutionCapsuleError('cacheResolution with a validated cache entry is required'); + } + timestamp(createdAt, 'createdAt'); + + const entry = cacheResolution.entry; + try { + validateContextCacheEntry(entry); + } catch (error) { + throw new ExecutionCapsuleError(`cacheResolution entry is invalid: ${error.message}`); + } + const targetDelta = resolveTargetDelta(contract, rootDir, changedFiles, entry); + const upstream = normalizeUpstreamTasks(upstreamAcceptedTasks); + const runtimeFacts = normalizeRuntimeFacts(verifiedRuntimeFacts); + const gateIds = stringArray(requiredGateProfileIds, 'requiredGateProfileIds'); + + const capsuleBase = { + schemaVersion: EXECUTION_CAPSULE_SCHEMA_VERSION, + contractId: contract.contractId, + taskId: contract.taskId, + runId: optionalId(runId, 'runId'), + lifecycleId: optionalId(lifecycleId, 'lifecycleId'), + sourceFingerprint: contract.sourceFingerprint, + developmentModeFingerprint: contract.developmentMode?.fingerprint ?? null, + workspaceTargets: contract.workspaceTargets ? structuredClone(contract.workspaceTargets) : null, + authoritativeSources: normalizeAuthoritativeSources(contract), + repositoryContext: { + cacheId: entry.cacheId, + selectorFingerprint: entry.selectorFingerprint, + inputFingerprint: entry.inputFingerprint, + entryFingerprint: entry.entryFingerprint, + cachePath: relativeCachePath(rootDir, cacheResolution.cachePath), + targets: normalizeTargetFacts(entry), + relevantFiles: fileRefs(entry.inputs.relevantFiles, 'cache relevantFiles'), + relevantTests: fileRefs(entry.inputs.relevantTests, 'cache relevantTests'), + }, + targetDelta, + upstreamAcceptedTasks: upstream, + verifiedRuntimeFacts: runtimeFacts, + requiredGateProfileIds: gateIds, + createdAt, + }; + + const capsuleFingerprint = fingerprintObject(semanticCapsulePayload(capsuleBase)); + const capsule = { + ...capsuleBase, + capsuleId: `CAP-${capsuleFingerprint.slice('sha256:'.length, 'sha256:'.length + 20)}`, + capsuleFingerprint, + }; + + validateExecutionCapsule(capsule); + return Object.freeze(capsule); +} + +export function prepareExecutionCapsule({ + contract, + rootDir = process.cwd(), + targetIds = null, + relevantFiles = [], + relevantTests = [], + gateProfileRevision = null, + runId = null, + lifecycleId = null, + changedFiles = [], + upstreamAcceptedTasks = [], + verifiedRuntimeFacts = [], + requiredGateProfileIds = [], + createdAt = new Date().toISOString(), +} = {}) { + const changedRefs = changedFileRefs(rootDir, changedFiles); + const changedPresent = changedRefs + .filter((ref) => ref.status === 'present') + .map((ref) => ref.path); + const effectiveRelevantFiles = [...new Set([ + ...stringArray(relevantFiles, 'relevantFiles').map((file) => normalizeRelativePath(file, 'relevant file')), + ...changedPresent, + ])].sort(); + + const cacheResolution = resolveRepositoryContextCache({ + contract, + rootDir, + targetIds, + relevantFiles: effectiveRelevantFiles, + relevantTests, + gateProfileRevision, + createdAt, + }); + + const capsule = createExecutionCapsule({ + contract, + rootDir, + cacheResolution, + runId, + lifecycleId, + changedFiles, + upstreamAcceptedTasks, + verifiedRuntimeFacts, + requiredGateProfileIds, + createdAt, + }); + + return Object.freeze({ + capsule, + cache: Object.freeze({ + status: cacheResolution.status, + cachePath: cacheResolution.cachePath, + invalidationReason: cacheResolution.invalidationReason, + metrics: cacheResolution.metrics, + }), + }); +} + +export function checkExecutionCapsuleStaleness({ + capsule, + contract, + rootDir = process.cwd(), +} = {}) { + validateExecutionCapsule(capsule); + validateDevelopmentContract(contract); + + const changes = []; + + if (capsule.contractId !== contract.contractId || capsule.taskId !== contract.taskId) { + changes.push({ code: 'CAPSULE_CONTRACT_IDENTITY_CHANGED' }); + } + if (capsule.sourceFingerprint !== contract.sourceFingerprint) { + changes.push({ + code: 'CAPSULE_SOURCE_FINGERPRINT_CHANGED', + expected: capsule.sourceFingerprint, + actual: contract.sourceFingerprint, + }); + } + if (capsule.developmentModeFingerprint !== (contract.developmentMode?.fingerprint ?? null)) { + changes.push({ code: 'CAPSULE_DEVELOPMENT_MODE_CHANGED' }); + } + + const expectedWorkspaceFingerprint = capsule.workspaceTargets?.registryFingerprint ?? null; + const currentWorkspaceFingerprint = contract.workspaceTargets?.registryFingerprint ?? null; + if (expectedWorkspaceFingerprint !== currentWorkspaceFingerprint) { + changes.push({ + code: 'CAPSULE_WORKSPACE_BINDING_CHANGED', + expected: expectedWorkspaceFingerprint, + actual: currentWorkspaceFingerprint, + }); + } + + const contractStaleness = checkContractStaleness(contract, rootDir); + if (contractStaleness.stale) { + changes.push({ + code: 'CAPSULE_CONTRACT_STALE', + detail: contractStaleness.changes, + }); + } + + const cacheAbsolute = path.resolve(rootDir, capsule.repositoryContext.cachePath); + const root = path.resolve(rootDir); + const cacheRelative = path.relative(root, cacheAbsolute); + if (cacheRelative === '..' || cacheRelative.startsWith(`..${path.sep}`) || path.isAbsolute(cacheRelative)) { + changes.push({ code: 'CAPSULE_CACHE_PATH_ESCAPE' }); + } else if (!fs.existsSync(cacheAbsolute)) { + changes.push({ code: 'CAPSULE_CACHE_ENTRY_MISSING' }); + } else { + try { + const entry = loadContextCacheEntry(cacheAbsolute, rootDir); + if (entry.entryFingerprint !== capsule.repositoryContext.entryFingerprint) { + changes.push({ + code: 'CAPSULE_CACHE_ENTRY_CHANGED', + expected: capsule.repositoryContext.entryFingerprint, + actual: entry.entryFingerprint, + }); + } else { + const cacheStaleness = checkContextCacheEntryStaleness({ + entry, + contract, + rootDir, + }); + if (cacheStaleness.stale) { + changes.push({ + code: 'CAPSULE_CACHE_STALE', + detail: cacheStaleness.changes, + }); + } + } + } catch (error) { + changes.push({ + code: 'CAPSULE_CACHE_INVALID', + message: error.message, + }); + } + } + + return Object.freeze({ + stale: changes.length > 0, + capsuleId: capsule.capsuleId, + capsuleFingerprint: capsule.capsuleFingerprint, + changes, + }); +} + +export function assertExecutionCapsuleFresh({ + capsule, + contract, + rootDir = process.cwd(), +} = {}) { + const report = checkExecutionCapsuleStaleness({ capsule, contract, rootDir }); + if (report.stale) { + throw new ExecutionCapsuleError('Execution Capsule is stale', report.changes); + } + return report; +} diff --git a/runtime/orchestration/index.mjs b/runtime/orchestration/index.mjs index 0da1d037..745b7ff2 100644 --- a/runtime/orchestration/index.mjs +++ b/runtime/orchestration/index.mjs @@ -58,7 +58,7 @@ export function prepareTaskRun({ return { contract, run }; } -export function createRoleContext({ contract, role, rootDir = process.cwd(), repositoryState, implementationReport, capabilities } = {}) { +export function createRoleContext({ contract, role, rootDir = process.cwd(), repositoryState, implementationReport, capabilities, executionCapsule = null } = {}) { return buildContextPackage({ contract, role, @@ -66,6 +66,7 @@ export function createRoleContext({ contract, role, rootDir = process.cwd(), rep repositoryState, implementationReport, capabilities, + executionCapsule, contextIsolation: role === 'implementation-agent' || role === 'implementer' ? 'fresh' : 'rehydrated', }); } @@ -143,3 +144,13 @@ export * from './idea-workflow.mjs'; export * from './decision-menu.mjs'; export * from './suggestion-promotion.mjs'; export * from './configuration-readiness.mjs'; + +export * from './cost-observatory.mjs'; + +export * from './workspace-targets.mjs'; + +export * from './context-cache.mjs'; +export * from './execution-capsule.mjs'; + +export * from './state-engine-v2.mjs'; +export * from './state-engine-migration.mjs'; diff --git a/runtime/orchestration/orchestration-run.mjs b/runtime/orchestration/orchestration-run.mjs index f4c156b2..898d5376 100644 --- a/runtime/orchestration/orchestration-run.mjs +++ b/runtime/orchestration/orchestration-run.mjs @@ -5,6 +5,13 @@ import { validateDevelopmentContract } from './development-contract.mjs'; import { selectRequiredGates } from './gate-selector.mjs'; import { selectExecutionStrategy } from './host-capabilities.mjs'; import { getRunDirectory } from './evidence-store.mjs'; +import { + appendEntityState, + canonicalStateJson, + hasCompletedStateMigration, + isStateEngineV2Active, + loadEntityState, +} from './state-engine-v2.mjs'; const RUN_STATES = Object.freeze([ 'READY', @@ -56,6 +63,28 @@ function persistImmutable(filePath, value, label) { return { created: true, path: filePath }; } +function runEntityId(contractId, runId) { + return `${contractId}/${runId}`; +} + +function legacyRunStateExists(contractId, runId, rootDir) { + const runDir = getRunDirectory(rootDir, contractId, runId); + const revisionDir = path.join(runDir, 'state-revisions'); + if (fs.existsSync(path.join(runDir, 'current-state.json'))) return true; + if (!fs.existsSync(revisionDir)) return false; + return fs.readdirSync(revisionDir).some((name) => /^\d{8}\.json$/.test(name)); +} + +function v2RunState(contractId, runId, rootDir) { + if (!isStateEngineV2Active(rootDir)) return null; + return loadEntityState('orchestration-run', runEntityId(contractId, runId), rootDir); +} + +function shouldUseV2RunState(contractId, runId, rootDir) { + if (!legacyRunStateExists(contractId, runId, rootDir)) return true; + return hasCompletedStateMigration(rootDir) && v2RunState(contractId, runId, rootDir) !== null; +} + export function createOrchestrationRun({ contract, runId, @@ -154,19 +183,57 @@ export function persistRunManifest(run, rootDir = process.cwd()) { export function persistRunStateRevision(run, rootDir = process.cwd()) { validateOrchestrationRun(run); - const runDir = getRunDirectory(rootDir, run.contractId, run.runId); - const revisionName = `${String(run.stateRevision).padStart(8, '0')}.json`; - const revisionPath = path.join(runDir, 'state-revisions', revisionName); - const persisted = persistImmutable(revisionPath, run, 'orchestration run state revision'); - const pointer = { - schemaVersion: '1.0.0', - contractId: run.contractId, - runId: run.runId, - stateRevision: run.stateRevision, - revisionPath: `state-revisions/${revisionName}`, + + if (!shouldUseV2RunState(run.contractId, run.runId, rootDir)) { + const runDir = getRunDirectory(rootDir, run.contractId, run.runId); + const revisionName = `${String(run.stateRevision).padStart(8, '0')}.json`; + const revisionPath = path.join(runDir, 'state-revisions', revisionName); + const persisted = persistImmutable(revisionPath, run, 'orchestration run state revision'); + const pointer = { + schemaVersion: '1.0.0', + contractId: run.contractId, + runId: run.runId, + stateRevision: run.stateRevision, + revisionPath: `state-revisions/${revisionName}`, + }; + atomicWrite(path.join(runDir, 'current-state.json'), stableContent(pointer)); + return { ...persisted, pointerPath: path.join(runDir, 'current-state.json'), stateEngine: 'legacy' }; + } + + const existing = v2RunState(run.contractId, run.runId, rootDir); + if (existing) { + if (run.stateRevision < existing.stateRevision) { + throw new OrchestrationRunError('Refusing orchestration run stateRevision regression'); + } + if (run.stateRevision === existing.stateRevision) { + if (canonicalStateJson(run) === canonicalStateJson(existing)) { + return { created: false, stateEngine: 'v2', path: null, pointerPath: null }; + } + throw new OrchestrationRunError('Refusing to overwrite orchestration run state revision'); + } + } + + const result = appendEntityState({ + rootDir, + entityType: 'orchestration-run', + entityId: runEntityId(run.contractId, run.runId), + state: run, + refs: { + contractId: run.contractId, + taskId: run.taskId, + runId: run.runId, + }, + actorClass: 'orchestration', + timestamp: run.updatedAt, + }); + + return { + created: result.changed, + stateEngine: 'v2', + path: null, + pointerPath: null, + eventId: result.event?.eventId ?? null, }; - atomicWrite(path.join(runDir, 'current-state.json'), stableContent(pointer)); - return { ...persisted, pointerPath: path.join(runDir, 'current-state.json') }; } export function persistFinalRunState(run, rootDir = process.cwd()) { @@ -185,6 +252,17 @@ export function loadRunManifest(contractId, runId, rootDir = process.cwd()) { } export function loadCurrentRunState(contractId, runId, rootDir = process.cwd()) { + if (shouldUseV2RunState(contractId, runId, rootDir)) { + const v2 = v2RunState(contractId, runId, rootDir); + if (v2) { + validateOrchestrationRun(v2); + if (v2.contractId !== contractId || v2.runId !== runId) { + throw new OrchestrationRunError('State Engine V2 run identity mismatch'); + } + return v2; + } + } + const runDir = getRunDirectory(rootDir, contractId, runId); const pointerPath = path.join(runDir, 'current-state.json'); if (!fs.existsSync(pointerPath)) return loadRunManifest(contractId, runId, rootDir); diff --git a/runtime/orchestration/state-engine-migration.mjs b/runtime/orchestration/state-engine-migration.mjs new file mode 100644 index 00000000..7ffcc4ad --- /dev/null +++ b/runtime/orchestration/state-engine-migration.mjs @@ -0,0 +1,940 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { validateWorkflowState } from '../autopilot/validators.mjs'; +import { validateOrchestrationRun } from './orchestration-run.mjs'; +import { + appendEntityState, + appendMetadataEvent, + canonicalStateJson, + ensureStateEngineLayout, + getStateEnginePaths, + loadCanonicalEvents, + loadEntityState, + loadStateSnapshot, + stateSha256, + verifyStateEngineIntegrity, +} from './state-engine-v2.mjs'; + +export const LEGACY_BACKUP_SCHEMA_VERSION = '1.0.0'; +export const LEGACY_MIGRATION_SCHEMA_VERSION = '1.0.0'; +export const LEGACY_BACKUP_FILE = 'legacy-backup.json'; + +export class StateMigrationError extends Error { + constructor(message, details = null) { + super(message); + this.name = 'StateMigrationError'; + this.details = details; + } +} + +function plainObject(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} + +function stable(value) { + if (Array.isArray(value)) return value.map(stable); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stable(value[key])])); +} + +function stablePretty(value) { + return `${JSON.stringify(stable(value), null, 2)}\n`; +} + +function sha256(value) { + return `sha256:${crypto.createHash('sha256').update(value).digest('hex')}`; +} + +function readJson(filePath, label) { + try { + return JSON.parse(fs.readFileSync(filePath, 'utf8')); + } catch (error) { + throw new StateMigrationError(`${label} is invalid JSON: ${error.message}`); + } +} + +function projectRelative(rootDir, absolutePath) { + const root = path.resolve(rootDir); + const absolute = path.resolve(absolutePath); + const relative = path.relative(root, absolute).replaceAll('\\', '/'); + if (relative === '..' || relative.startsWith('../') || path.isAbsolute(relative)) { + throw new StateMigrationError('Legacy state path escapes project root'); + } + return relative; +} + +function resolveProjectPath(rootDir, relativePath) { + const root = path.resolve(rootDir); + const absolute = path.resolve(root, relativePath); + const relative = path.relative(root, absolute); + if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new StateMigrationError(`Backup path escapes project root: ${relativePath}`); + } + return absolute; +} + +function atomicWrite(filePath, content) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + const temp = `${filePath}.tmp-${process.pid}-${crypto.randomUUID()}`; + const fd = fs.openSync(temp, 'wx'); + try { + fs.writeFileSync(fd, content, 'utf8'); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + fs.renameSync(temp, filePath); +} + +function listFilesRecursive(directory) { + if (!fs.existsSync(directory)) return []; + const files = []; + function walk(current) { + const entries = fs.readdirSync(current, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)); + for (const entry of entries) { + const full = path.join(current, entry.name); + if (entry.isDirectory()) walk(full); + else if (entry.isFile()) files.push(full); + } + } + walk(directory); + return files; +} + +function collectLegacyBackupFiles(rootDir) { + const files = new Set(); + const autopilotState = path.join(rootDir, '.development-kit', 'autopilot', 'state'); + for (const file of listFilesRecursive(autopilotState)) { + const name = path.basename(file); + if (name === 'current.json' || /^revision-\d{6}\.json$/.test(name)) files.add(file); + } + + const runsRoot = path.join(rootDir, '.development-kit', 'runs'); + for (const file of listFilesRecursive(runsRoot)) { + const name = path.basename(file); + const parent = path.basename(path.dirname(file)); + if ( + name === 'manifest.json' + || name === 'current-state.json' + || name === 'final-state.json' + || (parent === 'state-revisions' && /^\d{8}\.json$/.test(name)) + ) files.add(file); + } + + return [...files].sort((a, b) => projectRelative(rootDir, a).localeCompare(projectRelative(rootDir, b))); +} + +function buildBackupBundle(rootDir, files) { + const entries = files.map((file) => { + const content = fs.readFileSync(file); + return { + path: projectRelative(rootDir, file), + fingerprint: stateSha256(content), + bytes: content.length, + contentBase64: content.toString('base64'), + }; + }); + const manifest = entries.map(({ path: entryPath, fingerprint, bytes }) => ({ path: entryPath, fingerprint, bytes })); + const sourceFingerprint = sha256(JSON.stringify(stable(manifest))); + return { + schemaVersion: LEGACY_BACKUP_SCHEMA_VERSION, + sourceFingerprint, + fileCount: entries.length, + files: entries, + }; +} + +function validateBackupBundle(bundle) { + if (!plainObject(bundle) || bundle.schemaVersion !== LEGACY_BACKUP_SCHEMA_VERSION) { + throw new StateMigrationError('Legacy backup bundle schema is invalid'); + } + if (typeof bundle.sourceFingerprint !== 'string' || !/^sha256:[a-f0-9]{64}$/.test(bundle.sourceFingerprint)) { + throw new StateMigrationError('Legacy backup sourceFingerprint is invalid'); + } + if (!Array.isArray(bundle.files) || bundle.fileCount !== bundle.files.length) { + throw new StateMigrationError('Legacy backup file list is invalid'); + } + const manifest = []; + for (const entry of bundle.files) { + if (!plainObject(entry) || typeof entry.path !== 'string' || typeof entry.contentBase64 !== 'string') { + throw new StateMigrationError('Legacy backup entry is invalid'); + } + // Node substitutes U+FFFD for unpaired UTF-16 surrogates when mapping + // JavaScript paths to filesystem names. Distinct malicious strings can + // otherwise resolve to the same on-disk file and silently overwrite. + if (Buffer.from(entry.path, 'utf8').toString('utf8') !== entry.path) { + throw new StateMigrationError('Legacy backup path contains ill-formed Unicode'); + } + const pathParts = entry.path.split(/[/\\]+/).filter(Boolean); + if (pathParts.some((part) => part !== '.' && part !== '..' && (part.endsWith('.') || part.endsWith(' ')))) { + throw new StateMigrationError('Legacy backup path contains trailing dot or space: ' + entry.path); + } + const content = Buffer.from(entry.contentBase64, 'base64'); + if (content.length !== entry.bytes || stateSha256(content) !== entry.fingerprint) { + throw new StateMigrationError(`Legacy backup entry integrity failed: ${entry.path}`); + } + manifest.push({ path: entry.path, fingerprint: entry.fingerprint, bytes: entry.bytes }); + } + const actual = sha256(JSON.stringify(stable(manifest))); + if (actual !== bundle.sourceFingerprint) { + throw new StateMigrationError('Legacy backup sourceFingerprint does not match entries'); + } + return true; +} + +function persistLegacyBackup(rootDir, bundle) { + validateBackupBundle(bundle); + const paths = ensureStateEngineLayout(rootDir); + const backupPath = path.join(paths.stateRoot, LEGACY_BACKUP_FILE); + if (fs.existsSync(backupPath)) { + const existing = readJson(backupPath, 'Existing legacy backup'); + validateBackupBundle(existing); + if (existing.sourceFingerprint !== bundle.sourceFingerprint) { + throw new StateMigrationError('Refusing to overwrite legacy backup with different source state'); + } + return { backupPath, created: false }; + } + atomicWrite(backupPath, stablePretty(bundle)); + return { backupPath, created: true }; +} + +function autopilotHistory(rootDir) { + const stateDir = path.join(rootDir, '.development-kit', 'autopilot', 'state'); + if (!fs.existsSync(stateDir)) return null; + + const revisionFiles = fs.readdirSync(stateDir) + .filter((name) => /^revision-\d{6}\.json$/.test(name)) + .sort(); + + if (revisionFiles.length === 0) return null; + const currentPath = path.join(stateDir, 'current.json'); + if (!fs.existsSync(currentPath)) throw new StateMigrationError('Legacy Autopilot current.json is missing'); + + const pointer = readJson(currentPath, 'Legacy Autopilot current pointer'); + if (!Number.isInteger(pointer.currentRevision) || pointer.currentRevision < 1) { + throw new StateMigrationError('Legacy Autopilot currentRevision is invalid'); + } + const expectedCurrent = `revision-${String(pointer.currentRevision).padStart(6, '0')}.json`; + if (pointer.currentRevisionFile !== expectedCurrent) { + throw new StateMigrationError('Legacy Autopilot current pointer path is inconsistent'); + } + if (!revisionFiles.includes(expectedCurrent)) { + throw new StateMigrationError('Legacy Autopilot current revision file is missing'); + } + + for (let revision = 1; revision <= pointer.currentRevision; revision += 1) { + const expected = `revision-${String(revision).padStart(6, '0')}.json`; + if (!revisionFiles.includes(expected)) { + throw new StateMigrationError(`Legacy Autopilot revision chain is missing ${expected}`); + } + } + + const states = []; + for (const name of revisionFiles) { + const number = Number(name.match(/(\d{6})/)[1]); + if (number > pointer.currentRevision) { + throw new StateMigrationError('Legacy Autopilot contains revisions newer than current pointer'); + } + const state = readJson(path.join(stateDir, name), `Legacy Autopilot ${name}`); + try { validateWorkflowState(state); } catch (error) { + throw new StateMigrationError(`Legacy Autopilot state is invalid: ${name}: ${error.message}`); + } + if (state.stateRevision !== number) { + throw new StateMigrationError(`Legacy Autopilot revision filename/stateRevision mismatch: ${name}`); + } + if (state.workflowId !== pointer.workflowId) { + throw new StateMigrationError(`Legacy Autopilot workflow identity changes within revision chain: ${name}`); + } + states.push(state); + } + + const current = states.at(-1); + if (current.stateRevision !== pointer.currentRevision || current.workflowId !== pointer.workflowId) { + throw new StateMigrationError('Legacy Autopilot current state does not match pointer identity'); + } + + return { + entityType: 'autopilot-workflow', + entityId: current.workflowId, + refs: {}, + states, + current, + }; +} + +function orchestrationHistories(rootDir) { + const runsRoot = path.join(rootDir, '.development-kit', 'runs'); + if (!fs.existsSync(runsRoot)) return []; + const histories = []; + + const contractDirs = fs.readdirSync(runsRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .sort((a, b) => a.name.localeCompare(b.name)); + + for (const contractEntry of contractDirs) { + const contractDir = path.join(runsRoot, contractEntry.name); + const runDirs = fs.readdirSync(contractDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .sort((a, b) => a.name.localeCompare(b.name)); + + for (const runEntry of runDirs) { + const runDir = path.join(contractDir, runEntry.name); + const manifestPath = path.join(runDir, 'manifest.json'); + if (!fs.existsSync(manifestPath)) continue; + const manifest = readJson(manifestPath, 'Legacy orchestration manifest'); + try { validateOrchestrationRun(manifest); } catch (error) { + throw new StateMigrationError(`Legacy orchestration manifest is invalid: ${error.message}`); + } + + const revisionDir = path.join(runDir, 'state-revisions'); + let states = []; + if (fs.existsSync(revisionDir)) { + const names = fs.readdirSync(revisionDir) + .filter((name) => /^\d{8}\.json$/.test(name)) + .sort(); + + if (names.length > 0) { + const pointerPath = path.join(runDir, 'current-state.json'); + if (!fs.existsSync(pointerPath)) throw new StateMigrationError('Legacy orchestration current-state.json is missing'); + const pointer = readJson(pointerPath, 'Legacy orchestration current pointer'); + if ( + pointer.contractId !== manifest.contractId + || pointer.runId !== manifest.runId + || !Number.isInteger(pointer.stateRevision) + || pointer.stateRevision < 1 + ) { + throw new StateMigrationError('Legacy orchestration current pointer identity is invalid'); + } + const expectedName = `${String(pointer.stateRevision).padStart(8, '0')}.json`; + if (pointer.revisionPath !== `state-revisions/${expectedName}` || !names.includes(expectedName)) { + throw new StateMigrationError('Legacy orchestration current pointer path is inconsistent'); + } + for (let revision = 1; revision <= pointer.stateRevision; revision += 1) { + const expected = `${String(revision).padStart(8, '0')}.json`; + if (!names.includes(expected)) { + throw new StateMigrationError(`Legacy orchestration revision chain is missing ${expected}`); + } + } + + for (const name of names) { + const number = Number(name.replace('.json', '')); + if (number > pointer.stateRevision) { + throw new StateMigrationError('Legacy orchestration contains revisions newer than current pointer'); + } + const state = readJson(path.join(revisionDir, name), `Legacy orchestration revision ${name}`); + try { validateOrchestrationRun(state); } catch (error) { + throw new StateMigrationError(`Legacy orchestration revision is invalid: ${name}: ${error.message}`); + } + if (state.stateRevision !== number) { + throw new StateMigrationError(`Legacy orchestration filename/stateRevision mismatch: ${name}`); + } + if (state.contractId !== manifest.contractId || state.runId !== manifest.runId) { + throw new StateMigrationError('Legacy orchestration revision identity does not match manifest'); + } + states.push(state); + } + if (canonicalStateJson(states[0]) !== canonicalStateJson(manifest)) { + throw new StateMigrationError('Legacy orchestration revision 1 differs from immutable manifest'); + } + } + } + + if (states.length === 0) states = [manifest]; + const current = states.at(-1); + + const finalPath = path.join(runDir, 'final-state.json'); + if (fs.existsSync(finalPath)) { + const finalState = readJson(finalPath, 'Legacy orchestration final-state'); + try { validateOrchestrationRun(finalState); } catch (error) { + throw new StateMigrationError(`Legacy orchestration final-state is invalid: ${error.message}`); + } + if (canonicalStateJson(finalState) !== canonicalStateJson(current)) { + throw new StateMigrationError('Legacy orchestration final-state differs from current state'); + } + } + + histories.push({ + entityType: 'orchestration-run', + entityId: `${manifest.contractId}/${manifest.runId}`, + refs: { + contractId: manifest.contractId, + taskId: manifest.taskId, + runId: manifest.runId, + }, + states, + current, + }); + } + } + + return histories; +} + +export function inspectLegacyState(rootDir = process.cwd()) { + const backupFiles = collectLegacyBackupFiles(rootDir); + const backup = buildBackupBundle(rootDir, backupFiles); + const histories = []; + const autopilot = autopilotHistory(rootDir); + if (autopilot) histories.push(autopilot); + histories.push(...orchestrationHistories(rootDir)); + + return Object.freeze({ + sourceFingerprint: backup.sourceFingerprint, + legacyFileCount: backup.fileCount, + backup, + histories, + entityCount: histories.length, + }); +} + +function migrationId(sourceFingerprint) { + return `mig-v2-${sourceFingerprint.slice('sha256:'.length, 'sha256:'.length + 24)}`; +} + +function completedMigration(sourceFingerprint, rootDir) { + const events = loadCanonicalEvents(rootDir); + return events.find((event) => ( + event.eventType === 'MIGRATION_COMPLETED' + && event.payload?.sourceFingerprint === sourceFingerprint + && event.payload?.semanticEquivalence === true + )) ?? null; +} + +function migrateHistory(history, rootDir) { + for (const legacyState of history.states) { + const current = loadEntityState(history.entityType, history.entityId, rootDir); + if (current) { + if (current.stateRevision > legacyState.stateRevision) continue; + if (current.stateRevision === legacyState.stateRevision) { + if (canonicalStateJson(current) !== canonicalStateJson(legacyState)) { + throw new StateMigrationError( + `Existing State Engine entity differs from legacy state at revision ${legacyState.stateRevision}: ${history.entityType}/${history.entityId}`, + ); + } + continue; + } + } + + appendEntityState({ + rootDir, + entityType: history.entityType, + entityId: history.entityId, + state: legacyState, + refs: history.refs, + actorClass: 'legacy-migration', + timestamp: legacyState.updatedAt ?? legacyState.createdAt ?? new Date().toISOString(), + }); + } +} + +function verifySemanticEquivalence(histories, rootDir) { + const mismatches = []; + for (const history of histories) { + const current = loadEntityState(history.entityType, history.entityId, rootDir); + if (canonicalStateJson(current) !== canonicalStateJson(history.current)) { + mismatches.push({ + entityType: history.entityType, + entityId: history.entityId, + expectedRevision: history.current.stateRevision, + actualRevision: current?.stateRevision ?? null, + }); + } + } + return { + equivalent: mismatches.length === 0, + mismatches, + }; +} + +export function migrateLegacyStateToV2({ + rootDir = process.cwd(), + completedAt = new Date().toISOString(), +} = {}) { + const inspection = inspectLegacyState(rootDir); + ensureStateEngineLayout(rootDir); + + const existingCompletion = completedMigration(inspection.sourceFingerprint, rootDir); + if (existingCompletion) { + verifyStateEngineIntegrity(rootDir); + const backupPath = path.join(getStateEnginePaths(rootDir).stateRoot, LEGACY_BACKUP_FILE); + if (!fs.existsSync(backupPath)) { + throw new StateMigrationError('Completed migration is missing its recoverable legacy backup'); + } + const backup = readJson(backupPath, 'Existing legacy backup'); + validateBackupBundle(backup); + if (backup.sourceFingerprint !== inspection.sourceFingerprint) { + throw new StateMigrationError('Completed migration backup no longer matches the legacy source fingerprint'); + } + + return Object.freeze({ + migrated: false, + idempotent: true, + migrationId: existingCompletion.payload.migrationId, + sourceFingerprint: inspection.sourceFingerprint, + legacyFileCount: inspection.legacyFileCount, + entityCount: inspection.entityCount, + semanticEquivalence: true, + backupPath, + }); + } + + const backupResult = persistLegacyBackup(rootDir, inspection.backup); + + for (const history of inspection.histories) migrateHistory(history, rootDir); + + const equivalence = verifySemanticEquivalence(inspection.histories, rootDir); + if (!equivalence.equivalent) { + throw new StateMigrationError('Legacy migration semantic-equivalence check failed', equivalence.mismatches); + } + + const id = migrationId(inspection.sourceFingerprint); + appendMetadataEvent({ + rootDir, + eventType: 'MIGRATION_COMPLETED', + actorClass: 'legacy-migration', + timestamp: completedAt, + payload: { + schemaVersion: LEGACY_MIGRATION_SCHEMA_VERSION, + migrationId: id, + sourceFingerprint: inspection.sourceFingerprint, + entityCount: inspection.entityCount, + legacyFileCount: inspection.legacyFileCount, + semanticEquivalence: true, + legacyRetained: true, + backupFile: LEGACY_BACKUP_FILE, + }, + }); + + verifyStateEngineIntegrity(rootDir); + const snapshot = loadStateSnapshot(rootDir, { rebuildIfNeeded: false }); + const migration = snapshot.migrations.find((item) => item.migrationId === id); + if (!migration?.semanticEquivalence) { + throw new StateMigrationError('Migration completion is missing from reconstructed snapshot'); + } + + return Object.freeze({ + migrated: true, + idempotent: false, + migrationId: id, + sourceFingerprint: inspection.sourceFingerprint, + legacyFileCount: inspection.legacyFileCount, + entityCount: inspection.entityCount, + semanticEquivalence: true, + legacyRetained: true, + backupPath: backupResult.backupPath, + }); +} + +function resolveSafeRestoreDestination(targetRoot, relativePath, { createMissing = false } = {}) { + const root = path.resolve(targetRoot); + const destination = resolveProjectPath(root, relativePath); + if (createMissing) fs.mkdirSync(root, { recursive: true }); + if (!fs.existsSync(root)) return destination; + if (fs.lstatSync(root).isSymbolicLink()) { + throw new StateMigrationError('Legacy backup restore root must not be a symbolic link'); + } + const rootReal = fs.realpathSync(root); + const components = path.relative(root, destination).split(path.sep); + if (components.some((part) => !part || part === '.' || part === '..' || part.endsWith('.') || part.endsWith(' '))) { + throw new StateMigrationError('Invalid legacy backup restore path'); + } + let current = root; + for (const component of components.slice(0, -1)) { + current = path.join(current, component); + try { + const stat = fs.lstatSync(current); + if (stat.isSymbolicLink()) { + throw new StateMigrationError('Legacy backup restore path contains a symbolic link or junction: ' + relativePath); + } + if (!stat.isDirectory()) throw new StateMigrationError('Legacy backup restore parent is not a directory: ' + relativePath); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + if (createMissing) fs.mkdirSync(current); + else continue; + } + const real = fs.realpathSync(current); + const rel = path.relative(rootReal, real); + if (rel === '..' || rel.startsWith('..' + path.sep) || path.isAbsolute(rel)) { + throw new StateMigrationError('Legacy backup restore escapes its target root: ' + relativePath); + } + } + try { + const existing = fs.lstatSync(destination); + if (!existing.isFile() || existing.isSymbolicLink()) { + throw new StateMigrationError('Legacy backup restore destination must be a regular file: ' + relativePath); + } + if (existing.nlink > 1) { + throw new StateMigrationError('Legacy backup restore destination has multiple hard links: ' + relativePath); + } + } catch (error) { + if (error.code !== 'ENOENT') throw error; + } + return destination; +} + + +/** A deterministic sibling journal lets offline recovery find interrupted root promotion. */ +function restoreJournalPath(targetRoot) { + const root = path.resolve(targetRoot); + const digest = crypto.createHash('sha256').update(root).digest('hex').slice(0, 24); + return path.join(path.dirname(root), '.dk-restore-journal-' + digest + '.json'); +} + +/** + * Directory-entry ordering barrier for the restore transaction. + * + * POSIX: fsync(2) on the parent directory descriptor. + * Windows: libuv opens directories with FILE_FLAG_BACKUP_SEMANTICS, and + * fs.fsyncSync calls FlushFileBuffers. FlushFileBuffers needs a write-capable + * handle ('r+'); a read-only handle fails EPERM. Microsoft documents a normal + * (flags = 0) flush as writing cached file data *and metadata* and + * synchronizing the storage cache (NtFlushBuffersFileEx; only + * FLUSH_FLAGS_FILE_DATA_SYNC_ONLY is documented invalid for directory + * handles). See docs/04-architecture/dkf-t04-windows-restore-durability-protocol.md + * for the exact guarantee relied on and its limits. + * + * Any failure is fatal: callers never mutate or acknowledge past a barrier + * that did not complete, so the journal and witnesses are retained. + */ +function syncRestoreParent(parent) { + let fd; + try { + fd = fs.openSync(parent, process.platform === 'win32' ? 'r+' : 'r'); + fs.fsyncSync(fd); + } catch (error) { + throw new StateMigrationError('Restore directory flush barrier failed; recovery evidence retained', { + code: 'RESTORE_DIRECTORY_FLUSH_FAILED', directory: parent, cause: error.code || error.message, + }); + } finally { + if (fd !== undefined) fs.closeSync(fd); + } +} + +function durableRestoreRollbackRename(from, to) { + fs.renameSync(from, to); + syncRestoreParent(path.dirname(to)); +} + +/** + * Sync the staged file bytes and (on POSIX) every containing directory before + * the promotion journal becomes durable. A rename alone does not durably + * commit nested directory entries after sudden power loss. + */ +function syncRestoreTree(directory) { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const item = path.join(directory, entry.name); + if (entry.isSymbolicLink()) continue; // retain links as links, never follow + if (entry.isDirectory()) { + syncRestoreTree(item); + } else if (entry.isFile()) { + // FlushFileBuffers on Windows requires a write-capable handle; a + // read-only descriptor can fail EPERM even when the file is writable. + const fd = fs.openSync(item, process.platform === 'win32' ? 'r+' : 'r'); + try { fs.fsyncSync(fd); } finally { fs.closeSync(fd); } + } else { + throw new StateMigrationError('Legacy restore stage contains an unsupported special file'); + } + } + syncRestoreParent(directory); +} + +function createRestoreJournal(file, data) { + const fd = fs.openSync(file, 'wx', 0o600); + try { + fs.writeFileSync(fd, stablePretty(data)); + fs.fsyncSync(fd); + } finally { fs.closeSync(fd); } + syncRestoreParent(path.dirname(file)); +} + +/** + * Recover only an interrupted OFFLINE restore for the exact target root. + * Reject unexpected names, inode witnesses, or concurrent target mutations. + */ +export function recoverLegacyRestore({ targetRoot, confirmOffline = false } = {}) { + if (typeof targetRoot !== 'string' || !targetRoot) { + throw new StateMigrationError('targetRoot is required for restore recovery'); + } + if (confirmOffline !== true) { + throw new StateMigrationError('Explicit offline confirmation is required for restore recovery'); + } + const root = path.resolve(targetRoot); + const parent = path.dirname(root); + const journalPath = restoreJournalPath(root); + if (!fs.existsSync(journalPath)) return Object.freeze({ recovered: false }); + const tx = readJson(journalPath, 'Legacy restore journal'); + const expectedStage = '.dk-legacy-restore-stage-' + tx.transactionId; + const stage = path.join(parent, expectedStage); + const displaced = stage + '-original'; + if (!/^[a-f0-9]{32}$/.test(tx.transactionId || '') || + tx.targetRoot !== root || tx.schemaVersion !== '1.0.0' || + tx.stageName !== expectedStage || tx.existed !== true && tx.existed !== false || + !/^\d+$/.test(tx.stageDev || '') || !/^\d+$/.test(tx.stageIno || '') || + (tx.existed && (!/^\d+$/.test(tx.originalDev || '') || !/^\d+$/.test(tx.originalIno || '')))) { + throw new StateMigrationError('Legacy restore journal contains an invalid transaction witness'); + } + // The recovery runs in an explicitly quiescent trusted workspace. All + // location/inode witnesses are rechecked before any cleanup. + const inspect = file => { + try { return fs.lstatSync(file, { bigint: true }); } catch (e) { if (e.code === 'ENOENT') return null; throw e; } + }; + const atRoot = inspect(root); + const inStage = inspect(stage); + const inDisplaced = inspect(displaced); + const equal = (stat, dev, ino) => stat && stat.isDirectory() && !stat.isSymbolicLink() && + stat.dev.toString() === dev && stat.ino.toString() === ino; + if (inStage && !equal(inStage, tx.stageDev, tx.stageIno)) { + throw new StateMigrationError('Unrecognized staged restore directory; refusing recovery'); + } + if (inDisplaced && (!tx.existed || !equal(inDisplaced, tx.originalDev, tx.originalIno))) { + throw new StateMigrationError('Displaced restore directory differs from original witness'); + } + + let outcome; + // Pre-mutation capability probe (fails closed with every witness intact). + syncRestoreParent(parent); + if (inDisplaced && !atRoot) { + // The restored root directory entry must be durable before removing the + // only journal that permits redoing this rollback after another power loss. + durableRestoreRollbackRename(displaced, root); // crash after displacement + outcome = 'rolled-back-original'; + } else if (inDisplaced && equal(atRoot, tx.stageDev, tx.stageIno)) { + // Promotion completed; preserve original in displaced directory for + // explicit/manual audit rather than silently deleting recovery evidence. + outcome = 'promoted-original-retained'; + } else if (!inDisplaced && (!atRoot && !tx.existed || + (tx.existed && equal(atRoot, tx.originalDev, tx.originalIno)))) { + outcome = 'uncommitted-original-unchanged'; + } else if (!tx.existed && equal(atRoot, tx.stageDev, tx.stageIno)) { + outcome = 'promoted-new-root'; + } else { + throw new StateMigrationError('Ambiguous restore journal state; refusing automatic mutation'); + } + + if (inStage) { + fs.rmSync(stage, { recursive: true, force: true }); + syncRestoreParent(parent); + } + // Keep any promoted displaced original as a recovery asset: never auto-delete. + fs.unlinkSync(journalPath); + syncRestoreParent(parent); + return Object.freeze({ recovered: true, outcome, displacedOriginal: outcome === 'promoted-original-retained' ? displaced : null }); +} + +/** + * Snapshot descendant metadata before copying: cpSync recreates ownership + * under the maintenance identity and may update source directory atimes. + * Post-order records let directories receive their final times only after + * all nested file handles have been visited. + */ +function collectRestoreMetadata(source, relative = '') { + const stat = fs.lstatSync(source); + const children = stat.isDirectory() + ? fs.readdirSync(source).flatMap(name => collectRestoreMetadata( + path.join(source, name), relative ? path.join(relative, name) : name, + )) + : []; + return [...children, { + relative, mode: stat.mode & 0o7777, + uid: stat.uid, gid: stat.gid, + atime: stat.atime, mtime: stat.mtime, + kind: stat.isDirectory() ? 'directory' : stat.isFile() ? 'file' : stat.isSymbolicLink() ? 'symlink' : 'unsupported', + }]; +} + +function applyRestoreMetadata(stage, records) { + for (const entry of records) { + const destination = entry.relative ? path.join(stage, entry.relative) : stage; + const current = fs.lstatSync(destination); + const sameKind = entry.kind === 'directory' ? current.isDirectory() + : entry.kind === 'file' ? current.isFile() + : entry.kind === 'symlink' ? current.isSymbolicLink() : false; + if (!sameKind) throw new StateMigrationError('Staged restore object type differs from original metadata'); + if (process.platform !== 'win32' && + (current.uid !== entry.uid || current.gid !== entry.gid)) { + if (entry.kind === 'symlink') fs.lchownSync(destination, entry.uid, entry.gid); + else fs.chownSync(destination, entry.uid, entry.gid); + } + if (entry.kind === 'symlink') continue; // never follow + fs.chmodSync(destination, entry.mode); + fs.utimesSync(destination, entry.atime, entry.mtime); + // Persist restored metadata, and avoid readdir once parent atime is set. + if (entry.kind === 'directory') { + syncRestoreParent(destination); + } else if (process.platform !== 'win32') { + const fd = fs.openSync(destination, 'r'); + try { fs.fsyncSync(fd); } finally { fs.closeSync(fd); } + } + } +} + +export function restoreLegacyBackup({ + backupPath, + targetRoot, + overwrite = false, + confirmOffline = false, +} = {}) { + if (typeof backupPath !== 'string' || !backupPath) throw new StateMigrationError('backupPath is required'); + if (typeof targetRoot !== 'string' || !targetRoot) throw new StateMigrationError('targetRoot is required'); + if (confirmOffline !== true) { + throw new StateMigrationError('Legacy backup restoration requires explicit offline confirmation and an exclusively controlled local workspace'); + } + if (fs.existsSync(restoreJournalPath(targetRoot))) { + throw new StateMigrationError('Incomplete legacy restore exists; run recoverLegacyRestore with offline confirmation first'); + } + const bundle = readJson(backupPath, 'Legacy backup'); + validateBackupBundle(bundle); + + // Complete a full destination security and overwrite preflight before writing + // even one byte; a later malicious backup path cannot leave a partial restore. + const destinations = bundle.files.map((entry) => ({ + entry, + destination: resolveSafeRestoreDestination(targetRoot, entry.path), + })); + const normalizedDestinations = new Set(); + for (const { entry, destination } of destinations) { + // Conservative case/Unicode folding and Win32 trailing-dot/space removal + // prevents duplicate-target aliases on APFS/HFS+, NTFS, and FAT. + // Refusing distinct-case or aliased names on a case-sensitive mount is intentional: + // restore cannot safely assume a particular target filesystem policy. + const key = destination + .normalize('NFC') + .toLowerCase() + .split(/[\\/]+/) + .map((part) => part.replace(/[. ]+$/, '')) + .join('/'); + if (normalizedDestinations.has(key)) { + throw new StateMigrationError('Duplicate normalized backup restore destination: ' + entry.path); + } + normalizedDestinations.add(key); + } + for (const { entry, destination } of destinations) { + if (fs.existsSync(destination) && !overwrite) { + throw new StateMigrationError('Refusing to overwrite restored legacy path: ' + entry.path); + } + } + + // Path-based preflight + O_NOFOLLOW cannot secure an overwrite against a + // concurrent parent-directory swap. Build and fsync a complete replacement + // tree in an owner-only sibling directory. Publish by renaming the root + // directory itself; no restored byte is ever opened through an attacker- + // mutable path *inside* targetRoot. + const resolvedRoot = path.resolve(targetRoot); + const parent = path.dirname(resolvedRoot); + const originalRootExists = fs.existsSync(resolvedRoot); + const rootStat = originalRootExists ? fs.lstatSync(resolvedRoot) : null; + const originalMetadata = originalRootExists ? collectRestoreMetadata(resolvedRoot) : []; + const rootIdentity = originalRootExists ? fs.lstatSync(resolvedRoot, { bigint: true }) : null; + if (rootStat && (!rootStat.isDirectory() || rootStat.isSymbolicLink())) { + throw new StateMigrationError('Legacy backup restore root must be an ordinary directory'); + } + const transactionId = crypto.randomBytes(16).toString('hex'); + // Pre-mutation capability probe: if this volume cannot complete the + // directory flush barrier, refuse before creating the stage or journal. + syncRestoreParent(parent); + const stage = path.join(parent, '.dk-legacy-restore-stage-' + transactionId); + fs.mkdirSync(stage, { mode: 0o700 }); + const displaced = stage + '-original'; + let promoted = false; + let displacedOriginal = false; + try { + fs.chmodSync(stage, 0o700); + if (originalRootExists) { + fs.cpSync(resolvedRoot, stage, { recursive: true, force: true, dereference: false, verbatimSymlinks: true }); + } + const restored = []; + for (const { entry } of destinations) { + const destination = resolveSafeRestoreDestination(stage, entry.path, { createMissing: true }); + const content = Buffer.from(entry.contentBase64, 'base64'); + const tempFile = destination + '.tmp-' + crypto.randomUUID(); + const fd = fs.openSync(tempFile, 'wx', 0o600); + try { + fs.writeFileSync(fd, content); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + // An atomic rename replaces the *directory entry* even if the prior + // staging entry is a link; it never truncates an external hardlink inode. + fs.renameSync(tempFile, destination); + if (stateSha256(fs.readFileSync(destination)) !== entry.fingerprint) { + throw new StateMigrationError('Staged legacy file failed fingerprint verification: ' + entry.path); + } + restored.push(entry.path); + } + + // Seal and flush staged file contents before metadata restoration. + // Metadata is replayed post-order so source file owners and descendant + // traversal modes do not drift and historical parent atimes are applied + // after every directory traversal has finished. + syncRestoreTree(stage); + if (rootStat) applyRestoreMetadata(stage, originalMetadata); + + // Durable journal precedes *both* root-directory renames. A crash after + // either rename is recoverable by an explicit offline recovery invocation. + const stageStat = fs.lstatSync(stage, { bigint: true }); + const journal = restoreJournalPath(resolvedRoot); + createRestoreJournal(journal, { + schemaVersion: '1.0.0', + transactionId, + stageName: path.basename(stage), + targetRoot: resolvedRoot, + existed: originalRootExists, + originalDev: rootIdentity?.dev.toString() ?? null, + originalIno: rootIdentity?.ino.toString() ?? null, + stageDev: stageStat.dev.toString(), + stageIno: stageStat.ino.toString(), + }); + + // Moving the existing root out of the way does not follow a symlink. + // Verify that the directory moved is still the original preflighted inode + // before publishing the staged replacement. + if (originalRootExists) { + fs.renameSync(resolvedRoot, displaced); + syncRestoreParent(parent); + displacedOriginal = true; + const moved = fs.lstatSync(displaced); + if (moved.dev !== rootStat.dev || moved.ino !== rootStat.ino) { + throw new StateMigrationError('Legacy backup restore root changed during staging'); + } + } + fs.renameSync(stage, resolvedRoot); + syncRestoreParent(parent); + promoted = true; + displacedOriginal = false; + // Preserve original root as a retained rollback asset. Only after the + // new root is durably visible should the journal be acknowledged. + fs.unlinkSync(journal); + syncRestoreParent(parent); + // Promotion is now acknowledged and durable. An old tree kept in the + // displaced slot is not needed for this completed transaction. + if (originalRootExists) { + try { fs.rmSync(displaced, { recursive: true, force: true }); } catch { + // Retain rather than damage the newly promoted root if cleanup fails. + } + } + return Object.freeze({ + restoredFiles: restored.sort(), + sourceFingerprint: bundle.sourceFingerprint, + }); + } finally { + if (displacedOriginal) { + try { + if (!fs.existsSync(resolvedRoot)) { + durableRestoreRollbackRename(displaced, resolvedRoot); + } + } catch {} + } + if (!promoted) { + try { fs.rmSync(stage, { recursive: true, force: true }); } catch {} + } + } +} + +export function getLegacyBackupPath(rootDir = process.cwd()) { + return path.join(getStateEnginePaths(rootDir).stateRoot, LEGACY_BACKUP_FILE); +} + +export { validateBackupBundle }; diff --git a/runtime/orchestration/state-engine-v2.mjs b/runtime/orchestration/state-engine-v2.mjs new file mode 100644 index 00000000..b6fed615 --- /dev/null +++ b/runtime/orchestration/state-engine-v2.mjs @@ -0,0 +1,1560 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +export const STATE_ENGINE_SCHEMA_VERSION = '2.0.0'; +export const STATE_EVENT_SCHEMA_VERSION = '1.0.0'; +export const STATE_SNAPSHOT_SCHEMA_VERSION = '2.0.0'; +export const STATE_INDEX_SCHEMA_VERSION = '1.0.0'; + +export const STATE_ENGINE_ROOT = '.development-kit/state'; +export const STATE_EVENTS_FILE = 'events.jsonl'; +export const STATE_SNAPSHOT_FILE = 'snapshot.json'; +export const STATE_INDEX_FILE = 'index.db'; +export const STATE_SCHEMA_FILE = 'schema.json'; + +const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/; +const EVENT_TYPE_PATTERN = /^[A-Z][A-Z0-9_]{1,95}$/; +const ENTITY_TYPE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const MAX_ENTITY_ID_LENGTH = 512; +// Bounded blocking sleeps avoid starving a contended local state writer. +const STATE_LOCK_SLEEP = new Int32Array(new SharedArrayBuffer(4)); +function waitStateLock(ms = 25) { Atomics.wait(STATE_LOCK_SLEEP, 0, 0, ms); } + +export class StateEngineError extends Error { + constructor(message, details = null) { + super(message); + this.name = 'StateEngineError'; + this.details = details; + } +} + +function plainObject(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} + +function stable(value) { + if (Array.isArray(value)) return value.map(stable); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stable(value[key])])); +} + +function stableJson(value) { + return JSON.stringify(stable(value)); +} + +function stablePretty(value) { + return `${JSON.stringify(stable(value), null, 2)}\n`; +} + +function sha256(value) { + return `sha256:${crypto.createHash('sha256').update(value).digest('hex')}`; +} + +function requireText(value, label) { + if (typeof value !== 'string' || !value.trim()) { + throw new StateEngineError(`${label} must be a non-empty string`); + } + return value.trim(); +} + +function optionalText(value, label) { + if (value === undefined || value === null) return null; + return requireText(value, label); +} + +function entityType(value) { + const normalized = requireText(value, 'entityType'); + if (!ENTITY_TYPE_PATTERN.test(normalized)) { + throw new StateEngineError('entityType contains unsupported characters'); + } + return normalized; +} + +function entityId(value) { + const normalized = requireText(value, 'entityId'); + if (normalized.length > MAX_ENTITY_ID_LENGTH || /[\r\n\0]/.test(normalized)) { + throw new StateEngineError('entityId is invalid'); + } + return normalized; +} + +function eventType(value) { + const normalized = requireText(value, 'eventType'); + if (!EVENT_TYPE_PATTERN.test(normalized)) { + throw new StateEngineError('eventType is invalid'); + } + return normalized; +} + +function timestamp(value, label = 'timestamp') { + if (typeof value !== 'string' || Number.isNaN(Date.parse(value))) { + throw new StateEngineError(`${label} must be a valid timestamp`); + } + return value; +} + +function stateRevision(value, { allowNull = false } = {}) { + if (allowNull && (value === undefined || value === null)) return null; + if (!Number.isInteger(value) || value < 1) { + throw new StateEngineError('stateRevision must be a positive integer'); + } + return value; +} + +function statePaths(rootDir = process.cwd()) { + const root = path.resolve(rootDir); + const stateRoot = path.resolve(root, STATE_ENGINE_ROOT); + const relative = path.relative(root, stateRoot); + if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new StateEngineError('State Engine path escapes project root'); + } + const paths = Object.freeze({ + root, + stateRoot, + events: path.join(stateRoot, STATE_EVENTS_FILE), + snapshot: path.join(stateRoot, STATE_SNAPSHOT_FILE), + index: path.join(stateRoot, STATE_INDEX_FILE), + schema: path.join(stateRoot, STATE_SCHEMA_FILE), + lock: path.join(stateRoot, 'state.lock'), + pending: path.join(stateRoot, 'pending-commit.json'), + }); + if (fs.existsSync(stateRoot)) { + assertStateRootRealpathSafe(paths); + assertStateFilesNotLinks(paths); + } + return paths; +} + +function assertStateRootRealpathSafe(paths) { + const rootReal = fs.realpathSync(paths.root); + const stateReal = fs.realpathSync(paths.stateRoot); + const relative = path.relative(rootReal, stateReal); + if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new StateEngineError('State Engine root resolves outside project root'); + } + return stateReal; +} + +function assertStateFilesNotLinks(paths) { + for (const field of ['events', 'snapshot', 'index', 'schema', 'pending', 'lock']) { + try { + const stat = fs.lstatSync(paths[field]); + if (stat.isSymbolicLink() || !stat.isFile()) { + throw new StateEngineError('State Engine internal file is not a regular file: ' + field); + } + if (stat.nlink > 1) { + throw new StateEngineError('State Engine internal file has multiple hard links: ' + field); + } + } catch (error) { + if (error.code !== 'ENOENT') throw error; + } + } +} + +/** + * Persist directory namespace updates before journal acknowledgement. + * POSIX uses the pinned directory FD; Windows must open a write-capable + * directory FD for fsync/FlushFileBuffers. Unavailable barriers fail closed. + */ +function flushStateDirectory(directory, pinnedDirFd = null) { + if (process.platform !== 'win32' && pinnedDirFd !== null) { + fs.fsyncSync(pinnedDirFd); + return; + } + let fd; + try { + fd = fs.openSync(directory, process.platform === 'win32' ? 'r+' : 'r'); + const stat = fs.fstatSync(fd); + if (!stat.isDirectory()) { + throw new StateEngineError('State Engine directory flush target is not a directory'); + } + if (pinnedDirFd !== null) { + const pinned = fs.fstatSync(pinnedDirFd); + if (!pinned.isDirectory() || pinned.dev !== stat.dev || pinned.ino !== stat.ino) { + throw new StateEngineError('State Engine directory changed before durability barrier'); + } + } + fs.fsyncSync(fd); + } catch (error) { + throw new StateEngineError('State Engine directory durability barrier failed; refusing acknowledgement', { + code: 'STATE_DIRECTORY_FLUSH_FAILED', + directory, + cause: error.code || error.message, + }); + } finally { + if (fd !== undefined) fs.closeSync(fd); + } +} + +function fsyncParentDirectory(directory) { + flushStateDirectory(directory); +} + +function pinnedUnlinkAndSync(filePath, pinnedDirFd = null) { + const directory = path.dirname(filePath); + const file = path.basename(filePath); + const ownsDirFd = !pinnedDirFd; + const dirFd = pinnedDirFd ?? fs.openSync(directory, 'r'); + try { + const pinned = fs.fstatSync(dirFd); + const observed = fs.lstatSync(directory); + if (!pinned.isDirectory() || !observed.isDirectory() || observed.isSymbolicLink() || + pinned.dev !== observed.dev || pinned.ino !== observed.ino) { + throw new StateEngineError('State Engine directory changed before journal cleanup'); + } + const rootDir = path.dirname(path.dirname(directory)); + assertStateRootRealpathSafe({ root: rootDir, stateRoot: directory }); + const target = process.platform === 'linux' ? path.join('/proc/self/fd/' + dirFd, file) : filePath; + const preStat = fs.lstatSync(target); + if (preStat.isSymbolicLink() || !preStat.isFile() || preStat.nlink > 1) { + throw new StateEngineError('Target journal file is not a regular unlinked file'); + } + // Trust model (T04, trusted/exclusive local runtime): two directory entries + // cannot be checked-and-unlinked atomically against a hostile same-UID + // process, so no ledger-binding check is attempted here. Ledger authority is + // enforced instead on every load, which rejects any snapshot/index that is + // inconsistent with canonical history (see loadStateSnapshot). + // Recheck directory immediately before unlink on platforms without descriptor-relative paths + const latest = fs.lstatSync(directory); + if (!latest.isDirectory() || latest.isSymbolicLink() || + latest.dev !== pinned.dev || latest.ino !== pinned.ino) { + throw new StateEngineError('State Engine directory was replaced before journal cleanup'); + } + fs.unlinkSync(target); + flushStateDirectory(directory, dirFd); + } finally { + if (ownsDirFd) fs.closeSync(dirFd); + } +} + +function atomicWrite(filePath, content, pinnedDirFd = null) { + const directory = path.dirname(filePath); + fs.mkdirSync(directory, { recursive: true }); + // Pin the already-validated State Engine directory before creating a + // journal, snapshot or index. Linux exposes descriptor-relative paths via + // /proc/self/fd; renames there cannot follow a subsequently swapped + // workspace directory symlink into an external target. + const ownsDirFd = !pinnedDirFd; + const dirFd = pinnedDirFd ?? fs.openSync(directory, 'r'); + const file = path.basename(filePath); + const pinnedDir = process.platform === 'linux' ? '/proc/self/fd/' + dirFd : directory; + const tempPath = path.join(pinnedDir, file + '.tmp-' + process.pid + '-' + crypto.randomUUID()); + const destination = path.join(pinnedDir, file); + let published = false; + try { + const pinned = fs.fstatSync(dirFd); + const observed = fs.lstatSync(directory); + if (!pinned.isDirectory() || !observed.isDirectory() || observed.isSymbolicLink() || + pinned.dev !== observed.dev || pinned.ino !== observed.ino) { + throw new StateEngineError('State Engine directory changed during durable write preparation'); + } + const rootDir = path.dirname(path.dirname(directory)); + assertStateRootRealpathSafe({ root: rootDir, stateRoot: directory }); + const fd = fs.openSync(tempPath, 'wx', 0o600); + try { + fs.writeFileSync(fd, content, 'utf8'); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + // On non-Linux platforms without handle-relative filesystem primitives, + // revalidate the directory identity immediately before publication. + // Hostile same-UID concurrent processes remain unsupported outside + // a trusted offline maintenance environment on those hosts. + const latest = fs.lstatSync(directory); + if (!latest.isDirectory() || latest.isSymbolicLink() || + latest.dev !== pinned.dev || latest.ino !== pinned.ino) { + throw new StateEngineError('State Engine directory was replaced before durable publication'); + } + fs.renameSync(tempPath, destination); + published = true; + flushStateDirectory(directory, dirFd); + return filePath; + } finally { + if (!published) { + try { fs.unlinkSync(tempPath); } catch {} + } + if (ownsDirFd) fs.closeSync(dirFd); + } +} + +function acquireStateLock(rootDir, timeoutMs = 12000) { + const paths = statePaths(rootDir); + fs.mkdirSync(paths.stateRoot, { recursive: true }); + assertStateRootRealpathSafe(paths); + const owner = 'state-lock-' + crypto.randomUUID(); + const started = Date.now(); + + while (Date.now() - started < timeoutMs) { + try { + const fd = fs.openSync(paths.lock, 'wx'); + try { + fs.writeFileSync(fd, JSON.stringify({ + owner, + pid: process.pid, + hostname: os.hostname(), + acquiredAt: new Date().toISOString(), + }), 'utf8'); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + return { path: paths.lock, owner }; + } catch (error) { + if (error.code !== 'EEXIST') { + // Windows can briefly deny create/open while another process closes or + // removes this exact lock file. Retry a bounded transient sharing + // violation; never interpret it as permission to steal the lock. + if (process.platform !== 'win32' || !['EPERM', 'EACCES'].includes(error.code)) { + throw new StateEngineError('Unable to acquire State Engine lock: ' + error.message); + } + waitStateLock(25); + continue; + } + let stat; + try { + stat = fs.statSync(paths.lock); + } catch (readError) { + if (readError.code === 'ENOENT') continue; + throw new StateEngineError('Cannot inspect existing State Engine lock: ' + readError.message); + } + if (Date.now() - stat.mtimeMs > 15_000) { + let prior; + try { + prior = JSON.parse(fs.readFileSync(paths.lock, 'utf8')); + } catch { + // Corrupt or incomplete lock metadata must never authorize lock stealing. + prior = null; + } + // A live local PID owns the lock regardless of its age. Remote and legacy + // owners are treated as unknown and cannot be stolen automatically. + if (prior && prior.hostname === os.hostname() && Number.isSafeInteger(prior.pid) && prior.pid > 0) { + let alive = true; + try { + process.kill(prior.pid, 0); + } catch (probeError) { + if (probeError.code === 'ESRCH') alive = false; + } + if (!alive) { + // Pathname compare-then-unlink is unsafe: another reclaimer may + // install a fresh live lock between the check and unlink. mkdir + // gives each local stale-lock reclaimer exclusive ownership of a + // recovery guard, so other reclaimers cannot race its unlink. + const reclaimGuard = paths.lock + '.reclaim'; + let guardOwned = false; + let guardMarker = null; + try { + try { + fs.mkdirSync(reclaimGuard, { mode: 0o700 }); + guardOwned = true; + guardMarker = path.join(reclaimGuard, 'owner.json'); + const guardFd = fs.openSync(guardMarker, 'wx', 0o600); + try { + fs.writeFileSync(guardFd, JSON.stringify({ + pid: process.pid, + hostname: os.hostname(), + owner, + acquiredAt: new Date().toISOString(), + })); + fs.fsyncSync(guardFd); + } finally { + fs.closeSync(guardFd); + } + } catch (guardError) { + if (guardError.code !== 'EEXIST') throw guardError; + } + if (guardOwned) { + let latest; + try { + latest = fs.statSync(paths.lock); + } catch (raceError) { + if (raceError.code !== 'ENOENT') throw raceError; + } + if (latest && latest.ino === stat.ino && latest.mtimeMs === stat.mtimeMs && latest.size === stat.size) { + let current; + try { + current = JSON.parse(fs.readFileSync(paths.lock, 'utf8')); + } catch (readError) { + if (readError.code !== 'ENOENT') throw readError; + } + if (current?.owner === prior.owner && current?.pid === prior.pid && current?.hostname === prior.hostname) { + // Verify ownership is still dead inside the reclaim guard. + let confirmedDead = false; + try { + process.kill(current.pid, 0); + } catch (probeError) { + confirmedDead = probeError.code === 'ESRCH'; + } + if (confirmedDead) { + try { + fs.unlinkSync(paths.lock); + } catch (removeError) { + if (removeError.code !== 'ENOENT') throw removeError; + } + } + } + } + } + } finally { + if (guardOwned) { + if (guardMarker && fs.existsSync(guardMarker)) fs.unlinkSync(guardMarker); + fs.rmdirSync(reclaimGuard); + } + } + if (guardOwned) continue; + } + } + } + // Use bounded, synchronous waiting so Node >=18 and current callers remain compatible. + waitStateLock(25); + } + } + throw new StateEngineError('State Engine lock acquisition timed out'); +} + +/** + * Explicit OFFLINE maintenance operation. Never reclaim an orphaned guard in + * the normal commit path: proving the reclaimer is gone requires inspecting + * owner metadata and ruling out an active writer. The caller must take the + * project out of service before acknowledging recovery. + */ +export function recoverAbandonedStateLock({ rootDir = process.cwd(), confirmOffline = false } = {}) { + if (confirmOffline !== true) { + throw new StateEngineError('Explicit offline confirmation is required for abandoned-lock recovery'); + } + const paths = statePaths(rootDir); + const guard = paths.lock + '.reclaim'; + if (!fs.existsSync(guard)) return Object.freeze({ recovered: false, reason: 'no guard' }); + const stat = fs.lstatSync(guard); + if (!stat.isDirectory() || stat.isSymbolicLink()) { + throw new StateEngineError('Abandoned lock guard is not a regular directory'); + } + if (Date.now() - stat.mtimeMs < 60_000) { + throw new StateEngineError('Refusing to reclaim a recently active State Engine guard'); + } + const marker = path.join(guard, 'owner.json'); + if (fs.existsSync(marker)) { + const owner = JSON.parse(fs.readFileSync(marker, 'utf8')); + if (owner.hostname !== os.hostname() || !Number.isSafeInteger(owner.pid) || owner.pid <= 0) { + throw new StateEngineError('Cannot establish local guard ownership for offline recovery'); + } + try { + process.kill(owner.pid, 0); + throw new StateEngineError('Refusing to remove a live State Engine recovery guard'); + } catch (probe) { + if (probe.code !== 'ESRCH') throw probe; + } + } + if (fs.existsSync(paths.lock)) { + const lock = JSON.parse(fs.readFileSync(paths.lock, 'utf8')); + if (lock.hostname !== os.hostname() || !Number.isSafeInteger(lock.pid) || lock.pid <= 0) { + throw new StateEngineError('Offline guard recovery requires a provably local dead state lock'); + } + try { + process.kill(lock.pid, 0); + throw new StateEngineError('Refusing to recover a guard while state lock owner is alive'); + } catch (probe) { + if (probe.code !== 'ESRCH') throw probe; + } + } + if (fs.existsSync(marker)) fs.unlinkSync(marker); + fs.rmdirSync(guard); + return Object.freeze({ recovered: true, mode: 'confirmed-offline' }); +} + +function releaseStateLock(lock) { + if (!lock?.path) return; + try { + if (!fs.existsSync(lock.path)) return; + const current = JSON.parse(fs.readFileSync(lock.path, 'utf8')); + if (current.owner === lock.owner) fs.unlinkSync(lock.path); + } catch {} +} + +function schemaDocument() { + return Object.freeze({ + schemaVersion: STATE_ENGINE_SCHEMA_VERSION, + authority: { + canonicalHistory: STATE_EVENTS_FILE, + materializedSnapshot: STATE_SNAPSHOT_FILE, + disposableIndex: STATE_INDEX_FILE, + }, + contracts: { + event: STATE_EVENT_SCHEMA_VERSION, + snapshot: STATE_SNAPSHOT_SCHEMA_VERSION, + index: STATE_INDEX_SCHEMA_VERSION, + }, + invariants: [ + 'events.jsonl is canonical runtime state history', + 'snapshot.json is derived from events.jsonl', + 'index.db is disposable and rebuildable', + 'event sequence is monotonic', + 'event hashes form a single integrity chain', + ], + }); +} + +export function ensureStateEngineLayout(rootDir = process.cwd()) { + const paths = statePaths(rootDir); + fs.mkdirSync(paths.stateRoot, { recursive: true }); + assertStateRootRealpathSafe(paths); + + const expectedSchema = stablePretty(schemaDocument()); + if (fs.existsSync(paths.schema)) { + const existing = fs.readFileSync(paths.schema, 'utf8'); + if (existing !== expectedSchema) { + throw new StateEngineError('State Engine schema.json does not match the runtime schema contract'); + } + } else { + atomicWrite(paths.schema, expectedSchema); + } + + if (!fs.existsSync(paths.events)) { + if (fs.existsSync(paths.pending)) { + throw new StateEngineError('State Engine events.jsonl is missing while pending-commit.json is present; state integrity compromised'); + } + atomicWrite(paths.events, ''); + } + if (!fs.existsSync(paths.snapshot)) atomicWrite(paths.snapshot, stablePretty(emptySnapshot())); + if (!fs.existsSync(paths.index)) atomicWrite(paths.index, stablePretty(buildStateIndex(emptySnapshot()))); + + return paths; +} + +export function isStateEngineV2Active(rootDir = process.cwd()) { + const paths = statePaths(rootDir); + if (!fs.existsSync(paths.stateRoot)) return false; + + const schemaExists = fs.existsSync(paths.schema); + const eventsExist = fs.existsSync(paths.events); + if (!schemaExists && !eventsExist) return false; + if (!schemaExists || !eventsExist) { + throw new StateEngineError('State Engine V2 layout is incomplete'); + } + + const expectedSchema = stablePretty(schemaDocument()); + const actualSchema = fs.readFileSync(paths.schema, 'utf8'); + if (actualSchema !== expectedSchema) { + throw new StateEngineError('State Engine schema.json does not match the runtime schema contract'); + } + return true; +} + +function emptySnapshot() { + return { + schemaVersion: STATE_SNAPSHOT_SCHEMA_VERSION, + engineSchemaVersion: STATE_ENGINE_SCHEMA_VERSION, + lastEventSequence: 0, + lastEventHash: null, + entities: {}, + migrations: [], + }; +} + +function eventHashPayload(event) { + const { eventHash, ...payload } = event; + return payload; +} + +function computeEventId(base) { + const digest = crypto.createHash('sha256').update(stableJson(base)).digest('hex').slice(0, 20); + return `evt-${String(base.sequence).padStart(10, '0')}-${digest}`; +} + +function computeEventHash(event) { + return sha256(stableJson(eventHashPayload(event))); +} + +export function validateStateEvent(event, previous = null) { + if (!plainObject(event)) throw new StateEngineError('State event must be an object'); + if (event.schemaVersion !== STATE_EVENT_SCHEMA_VERSION) { + throw new StateEngineError(`Unsupported event schemaVersion: ${event.schemaVersion}`); + } + if (!Number.isInteger(event.sequence) || event.sequence < 1) throw new StateEngineError('Event sequence must be a positive integer'); + if (typeof event.eventId !== 'string' || !/^evt-\d{10}-[a-f0-9]{20}$/.test(event.eventId)) { + throw new StateEngineError('Event eventId is invalid'); + } + timestamp(event.timestamp, 'event.timestamp'); + requireText(event.actorClass, 'event.actorClass'); + eventType(event.eventType); + + if (event.entityType !== null) entityType(event.entityType); + if (event.entityId !== null) entityId(event.entityId); + for (const field of ['lifecycleId', 'taskId', 'contractId', 'runId']) optionalText(event[field], field); + if (!plainObject(event.payload)) throw new StateEngineError('Event payload must be an object'); + + if (previous === null) { + if (event.sequence !== 1) throw new StateEngineError('First state event sequence must be 1'); + if (event.previousEventHash !== null) throw new StateEngineError('First state event previousEventHash must be null'); + } else { + if (event.sequence !== previous.sequence + 1) throw new StateEngineError('Event sequence is not contiguous'); + if (event.previousEventHash !== previous.eventHash) throw new StateEngineError('Event previousEventHash does not match chain'); + } + + if (typeof event.eventHash !== 'string' || !SHA256_PATTERN.test(event.eventHash)) { + throw new StateEngineError('Event eventHash is invalid'); + } + if (event.eventHash !== computeEventHash(event)) { + throw new StateEngineError(`Event hash mismatch at sequence ${event.sequence}`); + } + + const base = { ...eventHashPayload(event) }; + delete base.eventId; + if (event.eventId !== computeEventId(base)) { + throw new StateEngineError(`Event ID mismatch at sequence ${event.sequence}`); + } + + return true; +} + +function parseLedger(content) { + if (!content) return []; + const lines = content.split(/\r?\n/); + if (lines.at(-1) === '') lines.pop(); + const events = []; + let previous = null; + + for (let i = 0; i < lines.length; i += 1) { + if (!lines[i].trim()) throw new StateEngineError(`Canonical state ledger contains a blank record at line ${i + 1}`); + let event; + try { + event = JSON.parse(lines[i]); + } catch (error) { + throw new StateEngineError(`Canonical state ledger JSON is invalid at line ${i + 1}: ${error.message}`); + } + validateStateEvent(event, previous); + events.push(event); + previous = event; + } + return events; +} + +export function loadCanonicalEvents(rootDir = process.cwd()) { + const paths = statePaths(rootDir); + if (!fs.existsSync(paths.events)) { + if (fs.existsSync(paths.pending)) { + throw new StateEngineError('State Engine events.jsonl is missing while pending-commit.json is present; state integrity compromised'); + } + return []; + } + if (fs.existsSync(paths.pending)) { + return withStateLock(rootDir, () => parseLedger(fs.readFileSync(paths.events, 'utf8'))); + } + const content = fs.readFileSync(paths.events, 'utf8'); + if (fs.existsSync(paths.pending)) { + return withStateLock(rootDir, () => parseLedger(fs.readFileSync(paths.events, 'utf8'))); + } + return parseLedger(content); +} + +function normalizePathSegments(value) { + if (!Array.isArray(value) || value.some((segment) => typeof segment !== 'string' || !segment || /[\r\n\0]/.test(segment))) { + throw new StateEngineError('Patch operation path must be a non-empty string array'); + } + return value; +} + +function cloneSerializable(value, label) { + try { + const cloned = structuredClone(value); + JSON.stringify(cloned); + return cloned; + } catch { + throw new StateEngineError(`${label} must be JSON-serializable`); + } +} + +function diffValues(before, after, pathSegments = []) { + if (stableJson(before) === stableJson(after)) return []; + + if (plainObject(before) && plainObject(after)) { + const operations = []; + const keys = [...new Set([...Object.keys(before), ...Object.keys(after)])].sort(); + for (const key of keys) { + const childPath = [...pathSegments, key]; + if (!(key in after)) { + operations.push({ op: 'delete', path: childPath }); + } else if (!(key in before)) { + operations.push({ op: 'set', path: childPath, value: cloneSerializable(after[key], 'Patch value') }); + } else { + operations.push(...diffValues(before[key], after[key], childPath)); + } + } + return operations; + } + + return [{ op: 'set', path: pathSegments, value: cloneSerializable(after, 'Patch value') }]; +} + +function setAtPath(target, segments, value) { + if (segments.length === 0) return cloneSerializable(value, 'Patch root value'); + let cursor = target; + for (let i = 0; i < segments.length - 1; i += 1) { + const key = segments[i]; + if (!plainObject(cursor[key])) cursor[key] = {}; + cursor = cursor[key]; + } + cursor[segments.at(-1)] = cloneSerializable(value, 'Patch value'); + return target; +} + +function deleteAtPath(target, segments) { + if (segments.length === 0) throw new StateEngineError('Cannot delete the entity-state root'); + let cursor = target; + for (let i = 0; i < segments.length - 1; i += 1) { + const key = segments[i]; + if (!plainObject(cursor[key])) return target; + cursor = cursor[key]; + } + delete cursor[segments.at(-1)]; + return target; +} + +function applyOperations(previousState, operations) { + if (!Array.isArray(operations)) throw new StateEngineError('State event operations must be an array'); + let next = cloneSerializable(previousState, 'Previous state'); + for (const operation of operations) { + if (!plainObject(operation)) throw new StateEngineError('Patch operation must be an object'); + const segments = normalizePathSegments(operation.path); + if (operation.op === 'set') { + next = setAtPath(next, segments, operation.value); + } else if (operation.op === 'delete') { + next = deleteAtPath(next, segments); + } else { + throw new StateEngineError(`Unsupported patch operation: ${operation.op}`); + } + } + return next; +} + +function ensureEntityBucket(snapshot, type) { + if (!snapshot.entities[type]) snapshot.entities[type] = {}; + return snapshot.entities[type]; +} + +export function rebuildSnapshotFromEvents(events) { + if (!Array.isArray(events)) throw new StateEngineError('events must be an array'); + const snapshot = emptySnapshot(); + let previous = null; + + for (const event of events) { + validateStateEvent(event, previous); + + if (event.eventType === 'ENTITY_STATE_INITIALIZED') { + if (event.entityType === null || event.entityId === null) throw new StateEngineError('State initialization event requires entity identity'); + const bucket = ensureEntityBucket(snapshot, event.entityType); + if (bucket[event.entityId]) throw new StateEngineError(`Entity is initialized more than once: ${event.entityType}/${event.entityId}`); + const state = cloneSerializable(event.payload.state, 'Initialized state'); + const revision = stateRevision(event.payload.stateRevision); + bucket[event.entityId] = { + stateRevision: revision, + state, + lastEventSequence: event.sequence, + lastEventHash: event.eventHash, + refs: { + lifecycleId: event.lifecycleId, + taskId: event.taskId, + contractId: event.contractId, + runId: event.runId, + }, + }; + } else if (event.eventType === 'ENTITY_STATE_UPDATED') { + if (event.entityType === null || event.entityId === null) throw new StateEngineError('State update event requires entity identity'); + const bucket = ensureEntityBucket(snapshot, event.entityType); + const current = bucket[event.entityId]; + if (!current) throw new StateEngineError(`State update references unknown entity: ${event.entityType}/${event.entityId}`); + const revision = stateRevision(event.payload.stateRevision); + if (revision < current.stateRevision) { + throw new StateEngineError(`Entity stateRevision regressed for ${event.entityType}/${event.entityId}`); + } + current.state = applyOperations(current.state, event.payload.operations); + current.stateRevision = revision; + current.lastEventSequence = event.sequence; + current.lastEventHash = event.eventHash; + current.refs = { + lifecycleId: event.lifecycleId ?? current.refs.lifecycleId, + taskId: event.taskId ?? current.refs.taskId, + contractId: event.contractId ?? current.refs.contractId, + runId: event.runId ?? current.refs.runId, + }; + } else if (event.eventType === 'MIGRATION_COMPLETED') { + const migrationId = requireText(event.payload.migrationId, 'migrationId'); + const sourceFingerprint = requireText(event.payload.sourceFingerprint, 'sourceFingerprint'); + if (!SHA256_PATTERN.test(sourceFingerprint)) throw new StateEngineError('Migration sourceFingerprint is invalid'); + if (!snapshot.migrations.some((migration) => migration.migrationId === migrationId)) { + snapshot.migrations.push({ + migrationId, + sourceFingerprint, + completedAt: event.timestamp, + entityCount: Number(event.payload.entityCount ?? 0), + semanticEquivalence: event.payload.semanticEquivalence === true, + }); + } + } + + snapshot.lastEventSequence = event.sequence; + snapshot.lastEventHash = event.eventHash; + previous = event; + } + + snapshot.migrations.sort((a, b) => a.migrationId.localeCompare(b.migrationId)); + return snapshot; +} + +export function validateStateSnapshot(snapshot) { + if (!plainObject(snapshot)) throw new StateEngineError('State snapshot must be an object'); + if (snapshot.schemaVersion !== STATE_SNAPSHOT_SCHEMA_VERSION) throw new StateEngineError('Unsupported snapshot schemaVersion'); + if (snapshot.engineSchemaVersion !== STATE_ENGINE_SCHEMA_VERSION) throw new StateEngineError('Snapshot engine schema mismatch'); + if (!Number.isInteger(snapshot.lastEventSequence) || snapshot.lastEventSequence < 0) throw new StateEngineError('Snapshot lastEventSequence is invalid'); + if (snapshot.lastEventHash !== null && !SHA256_PATTERN.test(snapshot.lastEventHash)) throw new StateEngineError('Snapshot lastEventHash is invalid'); + if (!plainObject(snapshot.entities)) throw new StateEngineError('Snapshot entities must be an object'); + if (!Array.isArray(snapshot.migrations)) throw new StateEngineError('Snapshot migrations must be an array'); + + for (const [type, bucket] of Object.entries(snapshot.entities)) { + entityType(type); + if (!plainObject(bucket)) throw new StateEngineError(`Snapshot entity bucket is invalid: ${type}`); + for (const [id, record] of Object.entries(bucket)) { + entityId(id); + if (!plainObject(record)) throw new StateEngineError('Snapshot entity record must be an object'); + stateRevision(record.stateRevision); + if (!Number.isInteger(record.lastEventSequence) || record.lastEventSequence < 1) throw new StateEngineError('Entity lastEventSequence is invalid'); + if (!SHA256_PATTERN.test(record.lastEventHash)) throw new StateEngineError('Entity lastEventHash is invalid'); + cloneSerializable(record.state, 'Entity state'); + } + } + return true; +} + +function indexArrayPush(map, key, value) { + if (!key) return; + if (!map[key]) map[key] = []; + if (!map[key].includes(value)) map[key].push(value); +} + +export function buildStateIndex(snapshot) { + validateStateSnapshot(snapshot); + const records = []; + const byContract = {}; + const byTask = {}; + const byRun = {}; + const byLifecycle = {}; + + for (const [type, bucket] of Object.entries(snapshot.entities).sort(([a], [b]) => a.localeCompare(b))) { + for (const [id, record] of Object.entries(bucket).sort(([a], [b]) => a.localeCompare(b))) { + const key = `${type}::${id}`; + records.push({ + key, + entityType: type, + entityId: id, + stateRevision: record.stateRevision, + lastEventSequence: record.lastEventSequence, + lastEventHash: record.lastEventHash, + }); + indexArrayPush(byContract, record.refs?.contractId, key); + indexArrayPush(byTask, record.refs?.taskId, key); + indexArrayPush(byRun, record.refs?.runId, key); + indexArrayPush(byLifecycle, record.refs?.lifecycleId, key); + } + } + + for (const map of [byContract, byTask, byRun, byLifecycle]) { + for (const values of Object.values(map)) values.sort(); + } + + return { + schemaVersion: STATE_INDEX_SCHEMA_VERSION, + sourceSequence: snapshot.lastEventSequence, + sourceHash: snapshot.lastEventHash, + recordCount: records.length, + records, + byContract, + byTask, + byRun, + byLifecycle, + }; +} + +export function validateStateIndex(index, snapshot = null) { + if (!plainObject(index)) throw new StateEngineError('State index must be an object'); + if (index.schemaVersion !== STATE_INDEX_SCHEMA_VERSION) throw new StateEngineError('Unsupported index schemaVersion'); + if (!Number.isInteger(index.sourceSequence) || index.sourceSequence < 0) throw new StateEngineError('Index sourceSequence is invalid'); + if (index.sourceHash !== null && !SHA256_PATTERN.test(index.sourceHash)) throw new StateEngineError('Index sourceHash is invalid'); + if (!Array.isArray(index.records)) throw new StateEngineError('Index records must be an array'); + if (index.recordCount !== index.records.length) throw new StateEngineError('Index recordCount mismatch'); + for (const field of ['byContract', 'byTask', 'byRun', 'byLifecycle']) { + if (!plainObject(index[field])) throw new StateEngineError(`Index ${field} must be an object`); + } + if (snapshot) { + validateStateSnapshot(snapshot); + if (index.sourceSequence !== snapshot.lastEventSequence || index.sourceHash !== snapshot.lastEventHash) { + throw new StateEngineError('State index is stale relative to snapshot'); + } + } + return true; +} + +function persistDerived(snapshot, rootDir, pinnedDirFd = null) { + validateStateSnapshot(snapshot); + const paths = pinnedDirFd ? statePaths(rootDir) : ensureStateEngineLayout(rootDir); + const index = buildStateIndex(snapshot); + atomicWrite(paths.snapshot, stablePretty(snapshot), pinnedDirFd); + atomicWrite(paths.index, stablePretty(index), pinnedDirFd); + return { snapshot, index }; +} + +function repairDerivedLocked(rootDir) { + // Snapshot repair replays the *current* ledger under the writer lock, never a + // stale cached history from before another writer committed a later event. + const paths = ensureStateEngineLayout(rootDir); + const canonical = rebuildSnapshotFromEvents(parseLedger(fs.readFileSync(paths.events, 'utf8'))); + if (fs.existsSync(paths.snapshot)) { + let witness = null; + try { + witness = JSON.parse(fs.readFileSync(paths.snapshot, 'utf8')); + validateStateSnapshot(witness); + } catch { + witness = null; + } + if (witness && witness.lastEventSequence > canonical.lastEventSequence) { + throw new StateEngineError('Canonical state history appears truncated relative to the materialized snapshot'); + } + if (witness && witness.lastEventSequence === canonical.lastEventSequence + && witness.lastEventHash !== canonical.lastEventHash) { + throw new StateEngineError('Canonical state history hash differs from the materialized snapshot integrity witness'); + } + } + persistDerived(canonical, rootDir); + return canonical; +} + +export function rebuildStateSnapshot(rootDir = process.cwd()) { + return withStateLock(rootDir, () => repairDerivedLocked(rootDir)); +} + +export function loadStateSnapshot(rootDir = process.cwd(), { rebuildIfNeeded = true } = {}) { + const paths = statePaths(rootDir); + if (!fs.existsSync(paths.events)) { + if (fs.existsSync(paths.pending)) { + throw new StateEngineError('State Engine events.jsonl is missing while pending-commit.json is present; state integrity compromised'); + } + return null; + } + const canonical = rebuildSnapshotFromEvents(loadCanonicalEvents(rootDir)); + + if (!fs.existsSync(paths.snapshot)) { + if (!rebuildIfNeeded) return null; + return withStateLock(rootDir, () => repairDerivedLocked(rootDir)); + } + + let current; + try { + current = JSON.parse(fs.readFileSync(paths.snapshot, 'utf8')); + validateStateSnapshot(current); + } catch (error) { + if (!rebuildIfNeeded) throw error; + return withStateLock(rootDir, () => repairDerivedLocked(rootDir)); + } + + if (current.lastEventSequence > canonical.lastEventSequence) { + // A legitimate writer can advance the snapshot between a reader's ledger + // and snapshot reads. Check the latest canonical history before declaring + // truncation, without silencing a genuine witnessed history regression. + if (rebuildIfNeeded) { + const refreshed = rebuildSnapshotFromEvents(loadCanonicalEvents(rootDir)); + if (refreshed.lastEventSequence > current.lastEventSequence + || (refreshed.lastEventSequence === current.lastEventSequence + && refreshed.lastEventHash === current.lastEventHash)) { + return refreshed; + } + } + throw new StateEngineError( + 'Canonical state history appears truncated relative to the materialized snapshot', + { + snapshotSequence: current.lastEventSequence, + canonicalSequence: canonical.lastEventSequence, + snapshotHash: current.lastEventHash, + canonicalHash: canonical.lastEventHash, + }, + ); + } + if (current.lastEventSequence === canonical.lastEventSequence + && current.lastEventHash !== canonical.lastEventHash) { + throw new StateEngineError( + 'Canonical state history hash differs from the materialized snapshot integrity witness', + { + sequence: current.lastEventSequence, + snapshotHash: current.lastEventHash, + canonicalHash: canonical.lastEventHash, + }, + ); + } + if (stableJson(current) !== stableJson(canonical)) { + if (!rebuildIfNeeded) throw new StateEngineError('Materialized snapshot does not match canonical history'); + return withStateLock(rootDir, () => repairDerivedLocked(rootDir)); + } + return current; +} + +export function rebuildStateIndex(rootDir = process.cwd()) { + return withStateLock(rootDir, () => { + const snapshot = loadStateSnapshot(rootDir, { rebuildIfNeeded: true }) ?? emptySnapshot(); + const paths = ensureStateEngineLayout(rootDir); + const index = buildStateIndex(snapshot); + atomicWrite(paths.index, stablePretty(index)); + return index; + }); +} + +export function loadStateIndex(rootDir = process.cwd(), { rebuildIfNeeded = true } = {}) { + const snapshot = loadStateSnapshot(rootDir, { rebuildIfNeeded: true }) ?? emptySnapshot(); + const paths = statePaths(rootDir); + if (!fs.existsSync(paths.index)) { + return rebuildIfNeeded ? rebuildStateIndex(rootDir) : null; + } + try { + const index = JSON.parse(fs.readFileSync(paths.index, 'utf8')); + validateStateIndex(index, snapshot); + return index; + } catch (error) { + if (!rebuildIfNeeded) throw error; + return rebuildStateIndex(rootDir); + } +} + +function buildEvent({ + sequence, + previousEventHash, + timestamp: at, + actorClass, + type, + id: entityIdentifier = null, + entity: entityKind = null, + refs = {}, + payload, +}) { + const base = { + schemaVersion: STATE_EVENT_SCHEMA_VERSION, + sequence, + timestamp: timestamp(at, 'event timestamp'), + actorClass: requireText(actorClass, 'actorClass'), + eventType: eventType(type), + entityType: entityKind === null ? null : entityType(entityKind), + entityId: entityIdentifier === null ? null : entityId(entityIdentifier), + lifecycleId: optionalText(refs.lifecycleId, 'lifecycleId'), + taskId: optionalText(refs.taskId, 'taskId'), + contractId: optionalText(refs.contractId, 'contractId'), + runId: optionalText(refs.runId, 'runId'), + payload: cloneSerializable(payload, 'Event payload'), + previousEventHash: previousEventHash ?? null, + }; + const eventId = computeEventId(base); + const event = { + ...base, + eventId, + }; + event.eventHash = computeEventHash(event); + return event; +} + +function appendDurably(filePath, appendedText, expectedIdentity = null, pinnedDirFd = null, heldLedgerFd = null) { + const directory = path.dirname(filePath); + const file = path.basename(filePath); + const ownsDirFd = !pinnedDirFd; + const dirFd = pinnedDirFd ?? fs.openSync(directory, 'r'); + try { + const pinned = fs.fstatSync(dirFd); + const observed = fs.lstatSync(directory); + if (!pinned.isDirectory() || !observed.isDirectory() || observed.isSymbolicLink() || + pinned.dev !== observed.dev || pinned.ino !== observed.ino) { + throw new StateEngineError('State Engine directory changed before durable append'); + } + const rootDir = path.dirname(path.dirname(directory)); + assertStateRootRealpathSafe({ root: rootDir, stateRoot: directory }); + const target = process.platform === 'linux' ? path.join('/proc/self/fd/' + dirFd, file) : filePath; + // O_NOFOLLOW rejects a final-component symlink on supported POSIX hosts. + // On Windows, verify the opened inode and current directory entry before + // writing because path-only preflight cannot prevent a replacement race. + const flags = fs.constants.O_WRONLY | fs.constants.O_APPEND | (fs.constants.O_NOFOLLOW ?? 0); + const fd = fs.openSync(target, flags); + try { + const stat = fs.fstatSync(fd); + const entry = fs.lstatSync(target); + if (!stat.isFile() || stat.nlink !== 1 || entry.isSymbolicLink() || + !entry.isFile() || entry.nlink !== 1 || + entry.dev !== stat.dev || entry.ino !== stat.ino) { + throw new StateEngineError('Canonical event file symlink/inode mismatch or multiple hard links'); + } + if (heldLedgerFd !== null) { + const heldStat = fs.fstatSync(heldLedgerFd); + if (heldStat.dev !== stat.dev || heldStat.ino !== stat.ino) { + throw new StateEngineError('Canonical ledger inode changed while holding preparation descriptor'); + } + } + if (expectedIdentity && (stat.dev !== expectedIdentity.dev || + stat.ino !== expectedIdentity.ino || stat.size !== expectedIdentity.size)) { + throw new StateEngineError('Canonical event inode changed between event preparation and append'); + } + // Recheck directory immediately before write on platforms without descriptor-relative paths + const latest = fs.lstatSync(directory); + if (!latest.isDirectory() || latest.isSymbolicLink() || + latest.dev !== pinned.dev || latest.ino !== pinned.ino) { + throw new StateEngineError('State Engine directory was replaced before durable append'); + } + const bytes = Buffer.from(appendedText, 'utf8'); + let written = 0; + while (written < bytes.length) { + const amount = fs.writeSync(fd, bytes, written, bytes.length - written); + if (amount <= 0) throw new StateEngineError('Canonical event append made no forward progress'); + written += amount; + } + fs.fsyncSync(fd); + // Re-verify the directory entry after fsync to guarantee the canonical + // path still references this exact inode and was not unlinked or swapped + // during write (PRRC_kwDOTd-YNM79WYGR). + const finalEntry = fs.lstatSync(target); + const finalStat = fs.fstatSync(fd); + if (!finalEntry.isFile() || finalEntry.isSymbolicLink() || finalEntry.nlink !== 1 || + finalEntry.dev !== finalStat.dev || finalEntry.ino !== finalStat.ino) { + throw new StateEngineError('Canonical ledger directory entry was displaced during durable append'); + } + } finally { + fs.closeSync(fd); + } + flushStateDirectory(directory, dirFd); + } finally { + if (ownsDirFd) fs.closeSync(dirFd); + } +} + +function recoverPendingCommitLocked(rootDir) { + const paths = ensureStateEngineLayout(rootDir); + const dirFd = fs.openSync(paths.stateRoot, 'r'); + try { + const pinned = fs.fstatSync(dirFd); + const observed = fs.lstatSync(paths.stateRoot); + if (!pinned.isDirectory() || !observed.isDirectory() || observed.isSymbolicLink() || + pinned.dev !== observed.dev || pinned.ino !== observed.ino) { + throw new StateEngineError('State Engine directory changed before recovery preparation'); + } + const pendingTarget = process.platform === 'linux' + ? path.join('/proc/self/fd/' + dirFd, path.basename(paths.pending)) + : paths.pending; + const eventsTarget = process.platform === 'linux' + ? path.join('/proc/self/fd/' + dirFd, path.basename(paths.events)) + : paths.events; + const snapshotTarget = process.platform === 'linux' + ? path.join('/proc/self/fd/' + dirFd, path.basename(paths.snapshot)) + : paths.snapshot; + + if (!fs.existsSync(pendingTarget)) return; + if (!fs.existsSync(eventsTarget)) { + throw new StateEngineError('Cannot recover pending commit: canonical ledger events.jsonl is missing'); + } + let pending; + try { + pending = JSON.parse(fs.readFileSync(pendingTarget, 'utf8')); + } catch (error) { + throw new StateEngineError('Pending State Engine commit is invalid: ' + error.message); + } + if ( + pending.schemaVersion !== '1.0.0' + || !Number.isSafeInteger(pending.previousByteLength) + || pending.previousByteLength < 0 + || !SHA256_PATTERN.test(pending.previousHash) + || typeof pending.suffix !== 'string' + || !pending.suffix.endsWith('\n') + ) { + throw new StateEngineError('Pending State Engine commit metadata is invalid'); + } + // Bind every recovery operation to the same verified canonical descriptor. + // Pathname truncation can otherwise modify a hardlinked external file when + // an attacker replaces events.jsonl after the prefix has been inspected. + const flags = fs.constants.O_RDWR | (fs.constants.O_NOFOLLOW ?? 0); + const fd = fs.openSync(eventsTarget, flags); + let canonical; + let canonicalStat; + try { + const opened = fs.fstatSync(fd); + canonicalStat = opened; + const entry = fs.lstatSync(eventsTarget); + if (!opened.isFile() || !entry.isFile() || entry.isSymbolicLink() || + opened.nlink !== 1 || entry.nlink !== 1 || + opened.dev !== entry.dev || opened.ino !== entry.ino) { + throw new StateEngineError('Recovery canonical ledger inode/symlink/hardlink integrity failed'); + } + const bytes = Buffer.alloc(opened.size); + let read = 0; + while (read < bytes.length) { + const amount = fs.readSync(fd, bytes, read, bytes.length - read, read); + if (amount <= 0) throw new StateEngineError('Canonical ledger changed while recovering pending commit'); + read += amount; + } + if (bytes.length < pending.previousByteLength) { + throw new StateEngineError('Canonical state history was truncated during a pending commit'); + } + if (sha256(bytes.subarray(0, pending.previousByteLength)) !== pending.previousHash) { + throw new StateEngineError('Canonical state prefix differs from pending commit integrity witness'); + } + const existingSuffix = bytes.subarray(pending.previousByteLength); + const expectedSuffix = Buffer.from(pending.suffix, 'utf8'); + if (!expectedSuffix.subarray(0, existingSuffix.length).equals(existingSuffix)) { + throw new StateEngineError('Canonical state suffix conflicts with pending commit; refusing recovery'); + } + if (existingSuffix.length < expectedSuffix.length) { + // Truncate and repair through this FD, never a separately resolved path. + fs.ftruncateSync(fd, pending.previousByteLength); + let written = 0; + while (written < expectedSuffix.length) { + const amount = fs.writeSync(fd, expectedSuffix, written, + expectedSuffix.length - written, pending.previousByteLength + written); + if (amount <= 0) throw new StateEngineError('Pending recovery append made no forward progress'); + written += amount; + } + } + // Recovery can observe a fully written suffix that was never fsynced by + // the interrupted writer. Flush the canonical ledger unconditionally + // before publishing derived views or acknowledging the pending journal. + fs.fsyncSync(fd); + // Re-verify the directory entry after fsync to ensure the canonical path + // still names the repaired inode before publishing derived views (PRRC_kwDOTd-YNM79WYGR). + const finalEntry = fs.lstatSync(eventsTarget); + const finalStat = fs.fstatSync(fd); + if (!finalEntry.isFile() || finalEntry.isSymbolicLink() || finalEntry.nlink !== 1 || + finalEntry.dev !== finalStat.dev || finalEntry.ino !== finalStat.ino) { + throw new StateEngineError('Canonical ledger directory entry was displaced during pending recovery'); + } + canonical = rebuildSnapshotFromEvents( + parseLedger(Buffer.concat([bytes.subarray(0, pending.previousByteLength), expectedSuffix]).toString('utf8')), + ); + // Reconstruct both derived views before acknowledging the durable commit. + let witness = null; + try { + witness = JSON.parse(fs.readFileSync(snapshotTarget, 'utf8')); + validateStateSnapshot(witness); + } catch { + witness = null; + } + if (witness && witness.lastEventSequence > canonical.lastEventSequence) { + throw new StateEngineError('Cannot recover a pending commit over a later snapshot integrity witness'); + } + if (witness && witness.lastEventSequence === canonical.lastEventSequence + && witness.lastEventHash !== canonical.lastEventHash) { + throw new StateEngineError('Pending recovery conflicts with snapshot integrity witness'); + } + persistDerived(canonical, rootDir, dirFd); + pinnedUnlinkAndSync(paths.pending, dirFd); + } finally { + fs.closeSync(fd); + } + } finally { + fs.closeSync(dirFd); + } +} + +const ACTIVE_STATE_LOCKS = new Set(); +function withStateLock(rootDir, task) { + const key = path.resolve(rootDir); + if (ACTIVE_STATE_LOCKS.has(key)) return task(); + const lock = acquireStateLock(rootDir); + ACTIVE_STATE_LOCKS.add(key); + try { + recoverPendingCommitLocked(rootDir); + return task(); + } finally { + ACTIVE_STATE_LOCKS.delete(key); + releaseStateLock(lock); + } +} + +function commitPreparedEventsLocked(newEvents, rootDir = process.cwd()) { + if (!Array.isArray(newEvents) || newEvents.length === 0) { + const snapshot = loadStateSnapshot(rootDir, { rebuildIfNeeded: true }) ?? emptySnapshot(); + return { events: [], snapshot, index: loadStateIndex(rootDir, { rebuildIfNeeded: true }) }; + } + + const paths = ensureStateEngineLayout(rootDir); + const dirFd = fs.openSync(paths.stateRoot, 'r'); + try { + const pinned = fs.fstatSync(dirFd); + const observed = fs.lstatSync(paths.stateRoot); + if (!pinned.isDirectory() || !observed.isDirectory() || observed.isSymbolicLink() || + pinned.dev !== observed.dev || pinned.ino !== observed.ino) { + throw new StateEngineError('State Engine directory changed before durable write preparation'); + } + const rootDirParent = path.dirname(path.dirname(paths.stateRoot)); + assertStateRootRealpathSafe({ root: rootDirParent, stateRoot: paths.stateRoot }); + + const ledgerPath = process.platform === 'linux' + ? path.join('/proc/self/fd/' + dirFd, path.basename(paths.events)) + : paths.events; + + // Open ledger with descriptor-bound flags and read through this descriptor + // so preparation and identity witness come from the exact same descriptor. + // Retain this descriptor open across atomicWrite and appendDurably so the inode + // cannot be unlinked and recycled while the commit is in flight (PRRT_kwDOTd-YNM6ri6bH). + const readFlags = fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0); + const readFd = fs.openSync(ledgerPath, readFlags); + let existingContent; + let expectedIdentity; + try { + const openedStat = fs.fstatSync(readFd); + const entryStat = fs.lstatSync(ledgerPath); + if (!openedStat.isFile() || openedStat.nlink !== 1 || entryStat.isSymbolicLink() || + !entryStat.isFile() || entryStat.nlink !== 1 || + openedStat.dev !== entryStat.dev || openedStat.ino !== entryStat.ino) { + throw new StateEngineError('Canonical event file symlink/inode mismatch or multiple hard links'); + } + expectedIdentity = { dev: openedStat.dev, ino: openedStat.ino, size: openedStat.size }; + const bytes = Buffer.alloc(openedStat.size); + let read = 0; + while (read < bytes.length) { + const amount = fs.readSync(readFd, bytes, read, bytes.length - read, read); + if (amount <= 0 && read < bytes.length) { + throw new StateEngineError('Canonical ledger changed while reading during preparation'); + } + if (amount <= 0) break; + read += amount; + } + existingContent = bytes.toString('utf8'); + + const existing = parseLedger(existingContent); + let previous = existing.at(-1) ?? null; + for (const event of newEvents) { + validateStateEvent(event, previous); + previous = event; + } + const allEvents = [...existing, ...newEvents]; + // Validate and reconstruct before beginning a durable state mutation. + const nextSnapshot = rebuildSnapshotFromEvents(allEvents); + // A valid last event may lack the optional terminating newline. Preserve + // record separation when appending, including in the crash-recovery journal. + const separator = existingContent.length > 0 && !existingContent.endsWith('\n') ? '\n' : ''; + const suffix = separator + newEvents.map((event) => JSON.stringify(stable(event))).join('\n') + '\n'; + const pending = { + schemaVersion: '1.0.0', + previousByteLength: Buffer.byteLength(existingContent, 'utf8'), + previousHash: sha256(Buffer.from(existingContent, 'utf8')), + suffix, + }; + + atomicWrite(paths.pending, stablePretty(pending), dirFd); + // Physical append avoids O(history) rewrites; the pending journal permits + // deterministic repair after a torn write or interrupted snapshot persist. + appendDurably(paths.events, suffix, expectedIdentity, dirFd, readFd); + const { index } = persistDerived(nextSnapshot, rootDir, dirFd); + pinnedUnlinkAndSync(paths.pending, dirFd); + return { events: newEvents, snapshot: nextSnapshot, index }; + } finally { + fs.closeSync(readFd); + } + } finally { + fs.closeSync(dirFd); + } +} + +function nextEventContext(rootDir) { + const events = loadCanonicalEvents(rootDir); + const previous = events.at(-1) ?? null; + return { + sequence: previous ? previous.sequence + 1 : 1, + previousEventHash: previous?.eventHash ?? null, + previous, + }; +} + +export function appendEntityState({ + rootDir = process.cwd(), + entityType: type, + entityId: id, + state, + refs = {}, + actorClass = 'runtime', + timestamp: at = new Date().toISOString(), +} = {}) { + const normalizedType = entityType(type); + const normalizedId = entityId(id); + const nextState = cloneSerializable(state, 'Entity state'); + const revision = stateRevision(nextState.stateRevision); + + return withStateLock(rootDir, () => { + const currentSnapshot = loadStateSnapshot(rootDir, { rebuildIfNeeded: true }) ?? emptySnapshot(); + const current = currentSnapshot.entities[normalizedType]?.[normalizedId] ?? null; + + if (current && revision < current.stateRevision) { + throw new StateEngineError('Refusing stateRevision regression for ' + normalizedType + '/' + normalizedId); + } + if (current && stableJson(current.state) === stableJson(nextState)) { + return Object.freeze({ changed: false, event: null, snapshot: currentSnapshot }); + } + + const context = nextEventContext(rootDir); + const payload = current + ? { stateRevision: revision, operations: diffValues(current.state, nextState) } + : { stateRevision: revision, state: nextState }; + const event = buildEvent({ + sequence: context.sequence, + previousEventHash: context.previousEventHash, + timestamp: at, + actorClass, + type: current ? 'ENTITY_STATE_UPDATED' : 'ENTITY_STATE_INITIALIZED', + id: normalizedId, + entity: normalizedType, + refs, + payload, + }); + const committed = commitPreparedEventsLocked([event], rootDir); + return Object.freeze({ changed: true, event, snapshot: committed.snapshot }); + }); +} + +export function appendMetadataEvent({ + rootDir = process.cwd(), + eventType: type, + payload, + refs = {}, + actorClass = 'runtime', + timestamp: at = new Date().toISOString(), +} = {}) { + return withStateLock(rootDir, () => { + const context = nextEventContext(rootDir); + const event = buildEvent({ + sequence: context.sequence, + previousEventHash: context.previousEventHash, + timestamp: at, + actorClass, + type, + id: null, + entity: null, + refs, + payload, + }); + const committed = commitPreparedEventsLocked([event], rootDir); + return Object.freeze({ event, snapshot: committed.snapshot }); + }); +} + +export function loadEntityState(type, id, rootDir = process.cwd()) { + const normalizedType = entityType(type); + const normalizedId = entityId(id); + const snapshot = loadStateSnapshot(rootDir, { rebuildIfNeeded: true }); + const record = snapshot?.entities?.[normalizedType]?.[normalizedId]; + return record ? structuredClone(record.state) : null; +} + +export function listEntityRecords(type, rootDir = process.cwd()) { + const normalizedType = entityType(type); + const snapshot = loadStateSnapshot(rootDir, { rebuildIfNeeded: true }); + const bucket = snapshot?.entities?.[normalizedType] ?? {}; + return Object.entries(bucket) + .map(([id, record]) => ({ + entityId: id, + stateRevision: record.stateRevision, + lastEventSequence: record.lastEventSequence, + refs: structuredClone(record.refs), + })) + .sort((a, b) => a.entityId.localeCompare(b.entityId)); +} + +export function loadLatestEntityState(type, rootDir = process.cwd()) { + const normalizedType = entityType(type); + const snapshot = loadStateSnapshot(rootDir, { rebuildIfNeeded: true }); + const bucket = snapshot?.entities?.[normalizedType] ?? {}; + const records = Object.entries(bucket) + .map(([id, record]) => ({ id, record })) + .sort((a, b) => b.record.lastEventSequence - a.record.lastEventSequence || a.id.localeCompare(b.id)); + if (records.length === 0) return null; + return structuredClone(records[0].record.state); +} + +export function hasCompletedStateMigration(rootDir = process.cwd(), sourceFingerprint = null) { + const snapshot = loadStateSnapshot(rootDir, { rebuildIfNeeded: true }); + const migrations = snapshot?.migrations ?? []; + if (sourceFingerprint === null || sourceFingerprint === undefined) { + return migrations.some((migration) => migration.semanticEquivalence === true); + } + return migrations.some((migration) => ( + migration.semanticEquivalence === true + && migration.sourceFingerprint === sourceFingerprint + )); +} + +export function queryStateIndex({ + rootDir = process.cwd(), + contractId = null, + taskId = null, + runId = null, + lifecycleId = null, +} = {}) { + const index = loadStateIndex(rootDir, { rebuildIfNeeded: true }); + const filters = [ + ['byContract', contractId], + ['byTask', taskId], + ['byRun', runId], + ['byLifecycle', lifecycleId], + ].filter(([, value]) => value !== null && value !== undefined); + + if (filters.length === 0) return structuredClone(index.records); + + let keys = null; + for (const [field, value] of filters) { + const current = new Set(index[field][String(value)] ?? []); + keys = keys === null ? current : new Set([...keys].filter((key) => current.has(key))); + } + return index.records.filter((record) => keys.has(record.key)).map((record) => structuredClone(record)); +} + +export function verifyStateEngineIntegrity(rootDir = process.cwd()) { + const events = loadCanonicalEvents(rootDir); + const canonical = rebuildSnapshotFromEvents(events); + const snapshot = loadStateSnapshot(rootDir, { rebuildIfNeeded: false }); + const index = loadStateIndex(rootDir, { rebuildIfNeeded: false }); + + if (snapshot === null) throw new StateEngineError('State snapshot is missing'); + if (stableJson(snapshot) !== stableJson(canonical)) { + throw new StateEngineError('State snapshot is not semantically equivalent to canonical history'); + } + validateStateIndex(index, snapshot); + const rebuiltIndex = buildStateIndex(snapshot); + if (stableJson(index) !== stableJson(rebuiltIndex)) { + throw new StateEngineError('State index is not semantically equivalent to rebuilt index'); + } + + return Object.freeze({ + valid: true, + eventCount: events.length, + entityCount: Object.values(snapshot.entities).reduce((sum, bucket) => sum + Object.keys(bucket).length, 0), + lastEventSequence: snapshot.lastEventSequence, + lastEventHash: snapshot.lastEventHash, + }); +} + +export function cleanupOrphanedStateTemps(rootDir = process.cwd()) { + const paths = statePaths(rootDir); + if (!fs.existsSync(paths.stateRoot)) return []; + const removed = []; + for (const name of fs.readdirSync(paths.stateRoot)) { + if (!name.includes('.tmp-')) continue; + const candidate = path.join(paths.stateRoot, name); + if (fs.statSync(candidate).isFile()) { + fs.unlinkSync(candidate); + removed.push(candidate); + } + } + return removed; +} + +export function getStateEnginePaths(rootDir = process.cwd()) { + return statePaths(rootDir); +} + +export function countStateEngineFiles(rootDir = process.cwd()) { + const paths = statePaths(rootDir); + if (!fs.existsSync(paths.stateRoot)) return 0; + return fs.readdirSync(paths.stateRoot, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name !== 'state.lock' && entry.name !== 'pending-commit.json' && !entry.name.includes('.tmp-')) + .length; +} + +export { stableJson as canonicalStateJson, sha256 as stateSha256 }; diff --git a/runtime/orchestration/workspace-targets.mjs b/runtime/orchestration/workspace-targets.mjs new file mode 100644 index 00000000..f860d87a --- /dev/null +++ b/runtime/orchestration/workspace-targets.mjs @@ -0,0 +1,558 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +export const WORKSPACE_TARGET_REGISTRY_SCHEMA_VERSION = 1; +export const WORKSPACE_TARGET_REGISTRY_PATH = '.development-kit/workspace.json'; + +const TARGET_ID_PATTERN = /^[A-Za-z0-9@][A-Za-z0-9._:@/-]{0,127}$/; +const COMMAND_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const DISCOVERY_MARKERS = Object.freeze([ + 'package.json', + 'pyproject.toml', + 'Cargo.toml', + 'go.mod', + 'pom.xml', + 'build.gradle', + 'build.gradle.kts', +]); +const DISCOVERY_IGNORES = Object.freeze([ + '.git', + '.development-kit', + 'node_modules', + 'vendor', + 'dist', + 'build', + 'coverage', +]); + +export class WorkspaceTargetError extends Error { + constructor(message, details = null) { + super(message); + this.name = 'WorkspaceTargetError'; + this.details = details; + } +} + +function plainObject(value) { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function stable(value) { + if (Array.isArray(value)) return value.map(stable); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stable(value[key])])); +} + +function stableContent(value) { + return `${JSON.stringify(stable(value), null, 2)}\n`; +} + +function hash(value) { + return createHash('sha256').update(value).digest('hex'); +} + +function targetId(value, label = 'targetId') { + if (typeof value !== 'string' || !TARGET_ID_PATTERN.test(value)) { + throw new WorkspaceTargetError(`${label} must match ${TARGET_ID_PATTERN}`); + } + return value; +} + +function commandId(value, label = 'command name') { + if (typeof value !== 'string' || !COMMAND_ID_PATTERN.test(value)) { + throw new WorkspaceTargetError(`${label} must match ${COMMAND_ID_PATTERN}`); + } + return value; +} + +function text(value, label) { + if (typeof value !== 'string' || !value.trim()) { + throw new WorkspaceTargetError(`${label} must be a non-empty string`); + } + return value.trim(); +} + +function stringArray(value = [], label = 'array') { + if (!Array.isArray(value)) throw new WorkspaceTargetError(`${label} must be an array`); + return [...new Set(value.map((item) => text(item, `${label} entry`)))].sort(); +} + +function targetIdArray(value = [], label = 'target IDs') { + if (!Array.isArray(value)) throw new WorkspaceTargetError(`${label} must be an array`); + return [...new Set(value.map((item) => targetId(item, `${label} entry`)))].sort(); +} + +function normalizeRelativePath(value) { + const raw = text(value, 'target.path').replaceAll('\\', '/'); + if (raw === '.') return '.'; + if (path.posix.isAbsolute(raw) || /^[A-Za-z]:\//.test(raw) || raw.startsWith('//')) { + throw new WorkspaceTargetError(`target.path must be project-relative: ${value}`); + } + const normalized = path.posix.normalize(raw).replace(/^\.\//, '').replace(/\/$/, ''); + if (!normalized || normalized === '.') return '.'; + if (normalized === '..' || normalized.startsWith('../') || normalized.split('/').includes('..')) { + throw new WorkspaceTargetError(`target.path may not escape the project root: ${value}`); + } + return normalized; +} + +function resolveTargetPath(rootDir, targetPath) { + const root = path.resolve(rootDir); + const resolved = path.resolve(root, targetPath === '.' ? '' : targetPath); + const relative = path.relative(root, resolved); + if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new WorkspaceTargetError(`target.path escapes the project root: ${targetPath}`); + } + return resolved; +} + +function normalizeCommands(value = {}) { + if (!plainObject(value)) throw new WorkspaceTargetError('target.commands must be an object'); + return Object.fromEntries(Object.entries(value) + .map(([name, command]) => [commandId(name), text(command, `command ${name}`)]) + .sort(([a], [b]) => a.localeCompare(b))); +} + +function normalizeAdapter(value) { + if (value === undefined || value === null) return null; + if (!plainObject(value)) throw new WorkspaceTargetError('target.adapter must be an object or null'); + return structuredClone(value); +} + +function normalizeTarget(value, id) { + if (!plainObject(value)) throw new WorkspaceTargetError(`Target ${id} must be an object`); + const supported = new Set(['path', 'kind', 'dependencies', 'commands', 'capabilities', 'verification', 'adapter']); + for (const key of Object.keys(value)) { + if (!supported.has(key)) throw new WorkspaceTargetError(`Target ${id} contains unsupported property: ${key}`); + } + + const commands = normalizeCommands(value.commands ?? {}); + const verification = stringArray(value.verification ?? [], `${id}.verification`).map((name) => commandId(name, `${id}.verification entry`)); + for (const name of verification) { + if (!(name in commands)) { + throw new WorkspaceTargetError(`Target ${id} verification entry references missing command: ${name}`); + } + } + + const kind = value.kind === undefined || value.kind === null ? null : text(value.kind, `${id}.kind`); + + return { + path: normalizeRelativePath(value.path ?? '.'), + kind, + dependencies: targetIdArray(value.dependencies ?? [], `${id}.dependencies`), + commands, + capabilities: stringArray(value.capabilities ?? [], `${id}.capabilities`), + verification, + adapter: normalizeAdapter(value.adapter), + }; +} + +function assertDependencyGraph(targets) { + const ids = new Set(Object.keys(targets)); + const visiting = new Set(); + const visited = new Set(); + const stack = []; + + function visit(id) { + if (visiting.has(id)) { + const start = stack.indexOf(id); + const cycle = [...stack.slice(start), id]; + throw new WorkspaceTargetError(`Workspace target dependency cycle: ${cycle.join(' -> ')}`, { cycle }); + } + if (visited.has(id)) return; + visiting.add(id); + stack.push(id); + for (const dependency of targets[id].dependencies) { + if (!ids.has(dependency)) { + throw new WorkspaceTargetError(`Target ${id} depends on unknown target: ${dependency}`); + } + if (dependency === id) { + throw new WorkspaceTargetError(`Target ${id} may not depend on itself`); + } + visit(dependency); + } + stack.pop(); + visiting.delete(id); + visited.add(id); + } + + for (const id of ids) visit(id); +} + +export function normalizeWorkspaceTargetRegistry(registry, { rootDir = null } = {}) { + if (!plainObject(registry)) throw new WorkspaceTargetError('Workspace target registry must be an object'); + const supported = new Set(['schemaVersion', 'targets']); + for (const key of Object.keys(registry)) { + if (!supported.has(key)) throw new WorkspaceTargetError(`Workspace target registry contains unsupported property: ${key}`); + } + if (registry.schemaVersion !== WORKSPACE_TARGET_REGISTRY_SCHEMA_VERSION) { + throw new WorkspaceTargetError(`Unsupported workspace target registry schemaVersion: ${registry.schemaVersion}`); + } + if (!plainObject(registry.targets) || Object.keys(registry.targets).length === 0) { + throw new WorkspaceTargetError('Workspace target registry must contain at least one target'); + } + + const targets = {}; + for (const [id, value] of Object.entries(registry.targets).sort(([a], [b]) => a.localeCompare(b))) { + targetId(id); + targets[id] = normalizeTarget(value, id); + } + assertDependencyGraph(targets); + + if (rootDir !== null) { + for (const [id, target] of Object.entries(targets)) { + const resolved = resolveTargetPath(rootDir, target.path); + if (!fs.existsSync(resolved) || !fs.statSync(resolved).isDirectory()) { + throw new WorkspaceTargetError(`Target ${id} path is not an existing directory: ${target.path}`); + } + } + } + + return { + schemaVersion: WORKSPACE_TARGET_REGISTRY_SCHEMA_VERSION, + targets, + }; +} + +export function fingerprintWorkspaceTargetRegistry(registry) { + const normalized = normalizeWorkspaceTargetRegistry(registry); + return `sha256:${hash(JSON.stringify(stable(normalized)))}`; +} + +export function getWorkspaceTargetRegistryPath(rootDir = process.cwd()) { + return path.join(rootDir, WORKSPACE_TARGET_REGISTRY_PATH); +} + +export function loadWorkspaceTargetRegistry(rootDir = process.cwd(), { required = true } = {}) { + const registryPath = getWorkspaceTargetRegistryPath(rootDir); + if (!fs.existsSync(registryPath)) { + if (!required) return null; + throw new WorkspaceTargetError(`Workspace target registry is missing: ${WORKSPACE_TARGET_REGISTRY_PATH}`); + } + let parsed; + try { + parsed = JSON.parse(fs.readFileSync(registryPath, 'utf8')); + } catch (error) { + throw new WorkspaceTargetError(`Workspace target registry is not valid JSON: ${error.message}`); + } + return normalizeWorkspaceTargetRegistry(parsed, { rootDir }); +} + +function atomicWrite(filePath, content) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + const tempPath = `${filePath}.tmp-${process.pid}-${Date.now()}`; + fs.writeFileSync(tempPath, content, 'utf8'); + fs.renameSync(tempPath, filePath); +} + +export function persistWorkspaceTargetRegistry(registry, rootDir = process.cwd(), { expectedFingerprint = null } = {}) { + const normalized = normalizeWorkspaceTargetRegistry(registry, { rootDir }); + const filePath = getWorkspaceTargetRegistryPath(rootDir); + + if (fs.existsSync(filePath) && expectedFingerprint !== null) { + const current = loadWorkspaceTargetRegistry(rootDir); + const currentFingerprint = fingerprintWorkspaceTargetRegistry(current); + if (currentFingerprint !== expectedFingerprint) { + throw new WorkspaceTargetError('Workspace target registry changed since it was read', { + expectedFingerprint, + currentFingerprint, + }); + } + } + + atomicWrite(filePath, stableContent(normalized)); + return { + filePath, + fingerprint: fingerprintWorkspaceTargetRegistry(normalized), + registry: normalized, + }; +} + +export function registerWorkspaceTarget(registry, id, target) { + const normalized = normalizeWorkspaceTargetRegistry(registry); + const nextId = targetId(id); + if (normalized.targets[nextId]) { + throw new WorkspaceTargetError(`Target already exists: ${nextId}`); + } + return normalizeWorkspaceTargetRegistry({ + schemaVersion: normalized.schemaVersion, + targets: { + ...normalized.targets, + [nextId]: target, + }, + }); +} + +function ensureTarget(registry, id) { + const normalized = normalizeWorkspaceTargetRegistry(registry); + const resolvedId = targetId(id); + const value = normalized.targets[resolvedId]; + if (!value) throw new WorkspaceTargetError(`Unknown workspace target: ${resolvedId}`); + return { registry: normalized, id: resolvedId, target: value }; +} + +export function getDirectTargetDependencies(registry, id) { + const resolved = ensureTarget(registry, id); + return [...resolved.target.dependencies]; +} + +export function getReverseTargetDependants(registry, id) { + const resolved = ensureTarget(registry, id); + return Object.entries(resolved.registry.targets) + .filter(([, target]) => target.dependencies.includes(resolved.id)) + .map(([candidate]) => candidate) + .sort(); +} + +function closure(seedIds, next) { + const queue = [...seedIds]; + const seen = new Set(); + while (queue.length > 0) { + const id = queue.shift(); + if (seen.has(id)) continue; + seen.add(id); + for (const candidate of next(id)) { + if (!seen.has(candidate)) queue.push(candidate); + } + } + return [...seen].sort(); +} + +export function getTargetDependencyClosure(registry, targetIds) { + const normalized = normalizeWorkspaceTargetRegistry(registry); + const seeds = targetIdArray(targetIds, 'targetIds'); + for (const id of seeds) ensureTarget(normalized, id); + return closure(seeds, (id) => normalized.targets[id].dependencies); +} + +export function getAffectedTargetClosure(registry, targetIds) { + const normalized = normalizeWorkspaceTargetRegistry(registry); + const seeds = targetIdArray(targetIds, 'targetIds'); + for (const id of seeds) ensureTarget(normalized, id); + return closure(seeds, (id) => getReverseTargetDependants(normalized, id)); +} + +export function getVerificationTargetClosure(registry, targetIds) { + const affected = getAffectedTargetClosure(registry, targetIds); + return getTargetDependencyClosure(registry, affected); +} + +export function resolveTargetCommand(registry, targetIdValue, commandName, { rootDir = null } = {}) { + const resolved = ensureTarget(registry, targetIdValue); + const name = commandId(commandName); + const command = resolved.target.commands[name]; + if (!command) { + throw new WorkspaceTargetError(`Target ${resolved.id} does not declare command: ${name}`); + } + return Object.freeze({ + targetId: resolved.id, + commandName: name, + command, + cwd: rootDir === null ? resolved.target.path : resolveTargetPath(rootDir, resolved.target.path), + capabilities: [...resolved.target.capabilities], + adapter: resolved.target.adapter === null ? null : structuredClone(resolved.target.adapter), + }); +} + +export function resolveTargetsByCapability(registry, capability, { commandName = null } = {}) { + const normalized = normalizeWorkspaceTargetRegistry(registry); + const wanted = text(capability, 'capability'); + const requiredCommand = commandName === null ? null : commandId(commandName); + return Object.entries(normalized.targets) + .filter(([, target]) => target.capabilities.includes(wanted)) + .filter(([, target]) => requiredCommand === null || Boolean(target.commands[requiredCommand])) + .map(([id]) => id) + .sort(); +} + +export function resolveCapabilityRoute(registry, { + capability, + commandName, + preferredTargets = [], + rootDir = null, +} = {}) { + const candidates = resolveTargetsByCapability(registry, capability, { commandName }); + if (candidates.length === 0) { + throw new WorkspaceTargetError(`No workspace target declares capability ${capability} with command ${commandName}`); + } + const preferred = targetIdArray(preferredTargets, 'preferredTargets'); + const selectedId = preferred.find((id) => candidates.includes(id)) ?? candidates[0]; + return resolveTargetCommand(registry, selectedId, commandName, { rootDir }); +} + +export function createWorkspaceTargetBinding(registry, { + primary, + affected = null, + verification = null, +} = {}) { + const normalized = normalizeWorkspaceTargetRegistry(registry); + const primaryId = targetId(primary, 'primary target'); + ensureTarget(normalized, primaryId); + + const affectedIds = affected === null + ? getAffectedTargetClosure(normalized, [primaryId]) + : targetIdArray(affected, 'affected targets'); + const affectedWithPrimary = [...new Set([primaryId, ...affectedIds])].sort(); + for (const id of affectedWithPrimary) ensureTarget(normalized, id); + + const verificationIds = verification === null + ? getVerificationTargetClosure(normalized, affectedWithPrimary) + : targetIdArray(verification, 'verification targets'); + for (const id of verificationIds) ensureTarget(normalized, id); + + return Object.freeze({ + primary: primaryId, + affected: affectedWithPrimary, + verification: verificationIds, + }); +} + +export function resolveWorkspaceTargetBinding(rootDir, binding) { + if (!plainObject(binding)) throw new WorkspaceTargetError('workspace target binding must be an object'); + const registry = loadWorkspaceTargetRegistry(rootDir); + const normalizedBinding = createWorkspaceTargetBinding(registry, binding); + return Object.freeze({ + registryPath: WORKSPACE_TARGET_REGISTRY_PATH, + registryFingerprint: fingerprintWorkspaceTargetRegistry(registry), + ...normalizedBinding, + }); +} + +export function validatePersistedWorkspaceTargetBinding(binding) { + if (!plainObject(binding)) throw new WorkspaceTargetError('workspace target binding must be an object'); + const supported = new Set(['registryPath', 'registryFingerprint', 'primary', 'affected', 'verification']); + for (const key of Object.keys(binding)) { + if (!supported.has(key)) throw new WorkspaceTargetError(`workspace target binding contains unsupported property: ${key}`); + } + if (binding.registryPath !== WORKSPACE_TARGET_REGISTRY_PATH) { + throw new WorkspaceTargetError(`workspace target registry path must be ${WORKSPACE_TARGET_REGISTRY_PATH}`); + } + if (typeof binding.registryFingerprint !== 'string' || !/^sha256:[a-f0-9]{64}$/.test(binding.registryFingerprint)) { + throw new WorkspaceTargetError('workspace target registry fingerprint is invalid'); + } + targetId(binding.primary, 'primary target'); + const affected = targetIdArray(binding.affected, 'affected targets'); + const verification = targetIdArray(binding.verification, 'verification targets'); + if (!affected.includes(binding.primary)) { + throw new WorkspaceTargetError('affected targets must include the primary target'); + } + return true; +} + +export function checkWorkspaceTargetBindingStaleness(binding, rootDir = process.cwd()) { + validatePersistedWorkspaceTargetBinding(binding); + let registry; + try { + registry = loadWorkspaceTargetRegistry(rootDir); + } catch (error) { + return { + stale: true, + expectedRegistryFingerprint: binding.registryFingerprint, + currentRegistryFingerprint: null, + changes: [{ code: 'WORKSPACE_REGISTRY_UNAVAILABLE', message: error.message }], + }; + } + + const currentRegistryFingerprint = fingerprintWorkspaceTargetRegistry(registry); + const changes = []; + if (currentRegistryFingerprint !== binding.registryFingerprint) { + changes.push({ + code: 'WORKSPACE_REGISTRY_CHANGED', + expected: binding.registryFingerprint, + actual: currentRegistryFingerprint, + }); + } + + for (const [role, ids] of [ + ['primary', [binding.primary]], + ['affected', binding.affected], + ['verification', binding.verification], + ]) { + for (const id of ids) { + if (!registry.targets[id]) changes.push({ code: 'WORKSPACE_TARGET_MISSING', role, targetId: id }); + } + } + + return { + stale: changes.length > 0, + expectedRegistryFingerprint: binding.registryFingerprint, + currentRegistryFingerprint, + changes, + }; +} + +function discoveryId(relativePath, used) { + const base = relativePath === '.' + ? 'root' + : relativePath.replaceAll('\\', '/').replace(/[^A-Za-z0-9._-]+/g, '-').replace(/^-+|-+$/g, '') || 'target'; + let candidate = base; + let suffix = 1; + while (used.has(candidate)) { + suffix += 1; + candidate = `${base}-${suffix}`; + } + used.add(candidate); + return candidate; +} + +export function discoverWorkspaceTargetRegistry(rootDir = process.cwd(), { + maxDepth = 6, + markers = DISCOVERY_MARKERS, + ignoredDirectories = DISCOVERY_IGNORES, +} = {}) { + if (!Number.isInteger(maxDepth) || maxDepth < 0) throw new WorkspaceTargetError('maxDepth must be a non-negative integer'); + const root = path.resolve(rootDir); + if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) { + throw new WorkspaceTargetError('Workspace discovery root must be an existing directory'); + } + + const markerSet = new Set(stringArray(markers, 'markers')); + const ignored = new Set(stringArray(ignoredDirectories, 'ignoredDirectories')); + const found = []; + + function walk(directory, depth) { + const entries = fs.readdirSync(directory, { withFileTypes: true }); + const markerNames = entries.filter((entry) => entry.isFile() && markerSet.has(entry.name)).map((entry) => entry.name).sort(); + if (markerNames.length > 0) { + const relative = path.relative(root, directory).replaceAll('\\', '/') || '.'; + found.push({ relative, markers: markerNames }); + } + if (depth >= maxDepth) return; + for (const entry of entries.filter((entry) => entry.isDirectory()).sort((a, b) => a.name.localeCompare(b.name))) { + if (ignored.has(entry.name)) continue; + walk(path.join(directory, entry.name), depth + 1); + } + } + + walk(root, 0); + if (found.length === 0) found.push({ relative: '.', markers: [] }); + + const used = new Set(); + const targets = {}; + for (const boundary of found.sort((a, b) => a.relative.localeCompare(b.relative))) { + const id = discoveryId(boundary.relative, used); + targets[id] = { + path: boundary.relative, + kind: 'discovered-boundary', + dependencies: [], + commands: {}, + capabilities: [], + verification: [], + adapter: { + discovery: { + markers: boundary.markers, + }, + }, + }; + } + + return normalizeWorkspaceTargetRegistry({ + schemaVersion: WORKSPACE_TARGET_REGISTRY_SCHEMA_VERSION, + targets, + }, { rootDir }); +} + +export { DISCOVERY_MARKERS, DISCOVERY_IGNORES, TARGET_ID_PATTERN }; diff --git a/schemas/context-cache-entry.schema.json b/schemas/context-cache-entry.schema.json new file mode 100644 index 00000000..ae14ffdc --- /dev/null +++ b/schemas/context-cache-entry.schema.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://development-kit.dev/schemas/context-cache-entry.schema.json", + "title": "Development Kit Repository Context Cache Entry", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "cacheId", + "selectorFingerprint", + "inputFingerprint", + "createdAt", + "selector", + "inputs", + "targetFacts", + "entryFingerprint" + ], + "properties": { + "schemaVersion": { + "const": "1.0.0" + }, + "cacheId": { + "$ref": "#/$defs/sha256" + }, + "selectorFingerprint": { + "$ref": "#/$defs/sha256" + }, + "inputFingerprint": { + "$ref": "#/$defs/sha256" + }, + "createdAt": { + "type": "string", + "format": "date-time" + }, + "selector": { + "type": "object" + }, + "inputs": { + "type": "object" + }, + "targetFacts": { + "type": "array", + "items": { + "type": "object" + } + }, + "entryFingerprint": { + "$ref": "#/$defs/sha256" + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + } + } +} diff --git a/schemas/cost-record.schema.json b/schemas/cost-record.schema.json new file mode 100644 index 00000000..83eef653 --- /dev/null +++ b/schemas/cost-record.schema.json @@ -0,0 +1,158 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://development-kit.dev/schemas/cost-record.schema.json", + "title": "Development Kit Cost Record", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "operationId", + "lifecycleId", + "contractId", + "taskId", + "runId", + "role", + "contextBytes", + "estimatedContextTokens", + "contextEstimator", + "providerInputTokens", + "providerOutputTokens", + "providerCachedTokens", + "repositoryReads", + "repositoryScans", + "agentInvocations", + "toolCalls", + "verificationCommands", + "cacheHits", + "cacheMisses", + "startedAt", + "completedAt", + "elapsedMs" + ], + "properties": { + "schemaVersion": { + "const": "1.0.0" + }, + "operationId": { + "type": "string", + "minLength": 1 + }, + "lifecycleId": { + "type": [ + "string", + "null" + ] + }, + "contractId": { + "type": [ + "string", + "null" + ] + }, + "taskId": { + "type": [ + "string", + "null" + ] + }, + "runId": { + "type": [ + "string", + "null" + ] + }, + "role": { + "type": [ + "string", + "null" + ] + }, + "contextBytes": { + "type": "integer", + "minimum": 0 + }, + "estimatedContextTokens": { + "type": "integer", + "minimum": 0 + }, + "contextEstimator": { + "const": "logical-chars-div-4-v1" + }, + "providerInputTokens": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "providerOutputTokens": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "providerCachedTokens": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "repositoryReads": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "repositoryScans": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "agentInvocations": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "toolCalls": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "verificationCommands": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "cacheHits": { + "type": "integer", + "minimum": 0 + }, + "cacheMisses": { + "type": "integer", + "minimum": 0 + }, + "startedAt": { + "type": "string", + "format": "date-time" + }, + "completedAt": { + "type": "string", + "format": "date-time" + }, + "elapsedMs": { + "type": "integer", + "minimum": 0 + } + } +} diff --git a/schemas/development-contract.schema.json b/schemas/development-contract.schema.json index 32e947bf..dda1bddb 100644 --- a/schemas/development-contract.schema.json +++ b/schemas/development-contract.schema.json @@ -32,7 +32,8 @@ "enum": [ "1.0.0", "1.1.0", - "1.2.0" + "1.2.0", + "1.3.0" ] }, "contractId": { @@ -143,6 +144,15 @@ }, "requiredEvidence": { "type": "boolean" + }, + "requirementId": { + "type": [ + "string", + "null" + ] + }, + "requirementIds": { + "$ref": "#/$defs/stringArray" } } } @@ -609,6 +619,46 @@ } } } + }, + "workspaceTargets": { + "type": "object", + "additionalProperties": false, + "required": [ + "registryPath", + "registryFingerprint", + "primary", + "affected", + "verification" + ], + "properties": { + "registryPath": { + "const": ".development-kit/workspace.json" + }, + "registryFingerprint": { + "$ref": "#/$defs/sha256" + }, + "primary": { + "type": "string", + "pattern": "^[A-Za-z0-9@][A-Za-z0-9._:@/-]{0,127}$" + }, + "affected": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z0-9@][A-Za-z0-9._:@/-]{0,127}$" + }, + "uniqueItems": true, + "minItems": 1 + }, + "verification": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z0-9@][A-Za-z0-9._:@/-]{0,127}$" + }, + "uniqueItems": true + } + } } }, "$defs": { @@ -648,7 +698,10 @@ "if": { "properties": { "schemaVersion": { - "const": "1.2.0" + "enum": [ + "1.2.0", + "1.3.0" + ] } }, "required": [ diff --git a/schemas/execution-capsule.schema.json b/schemas/execution-capsule.schema.json new file mode 100644 index 00000000..97f0b1e5 --- /dev/null +++ b/schemas/execution-capsule.schema.json @@ -0,0 +1,214 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://development-kit.dev/schemas/execution-capsule.schema.json", + "title": "Development Kit Execution Capsule", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "capsuleId", + "contractId", + "taskId", + "runId", + "lifecycleId", + "sourceFingerprint", + "developmentModeFingerprint", + "workspaceTargets", + "authoritativeSources", + "repositoryContext", + "targetDelta", + "upstreamAcceptedTasks", + "verifiedRuntimeFacts", + "requiredGateProfileIds", + "createdAt", + "capsuleFingerprint" + ], + "properties": { + "schemaVersion": { + "const": "1.0.0" + }, + "capsuleId": { + "type": "string", + "pattern": "^CAP-[a-f0-9]{20}$" + }, + "contractId": { + "type": "string", + "minLength": 1 + }, + "taskId": { + "type": "string", + "minLength": 1 + }, + "runId": { + "type": [ + "string", + "null" + ] + }, + "lifecycleId": { + "type": [ + "string", + "null" + ] + }, + "sourceFingerprint": { + "$ref": "#/$defs/sha256" + }, + "developmentModeFingerprint": { + "anyOf": [ + { + "$ref": "#/$defs/sha256" + }, + { + "type": "null" + } + ] + }, + "workspaceTargets": { + "type": [ + "object", + "null" + ] + }, + "authoritativeSources": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "kind", + "authority", + "sections", + "fingerprint" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "kind": { + "type": "string", + "minLength": 1 + }, + "authority": { + "enum": [ + "required", + "supporting" + ] + }, + "sections": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "fingerprint": { + "$ref": "#/$defs/sha256" + } + } + } + }, + "repositoryContext": { + "type": "object", + "additionalProperties": false, + "required": [ + "cacheId", + "selectorFingerprint", + "inputFingerprint", + "entryFingerprint", + "cachePath", + "targets", + "relevantFiles", + "relevantTests" + ], + "properties": { + "cacheId": { + "$ref": "#/$defs/sha256" + }, + "selectorFingerprint": { + "$ref": "#/$defs/sha256" + }, + "inputFingerprint": { + "$ref": "#/$defs/sha256" + }, + "entryFingerprint": { + "$ref": "#/$defs/sha256" + }, + "cachePath": { + "type": "string", + "minLength": 1 + }, + "targets": { + "type": "array", + "items": { + "type": "object" + } + }, + "relevantFiles": { + "$ref": "#/$defs/fileRefs" + }, + "relevantTests": { + "$ref": "#/$defs/fileRefs" + } + } + }, + "targetDelta": { + "type": "object" + }, + "upstreamAcceptedTasks": { + "type": "array", + "items": { + "type": "object" + } + }, + "verifiedRuntimeFacts": { + "type": "array", + "items": { + "type": "object" + } + }, + "requiredGateProfileIds": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "createdAt": { + "type": "string", + "format": "date-time" + }, + "capsuleFingerprint": { + "$ref": "#/$defs/sha256" + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + }, + "fileRefs": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "fingerprint" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "fingerprint": { + "$ref": "#/$defs/sha256" + } + } + } + } + } +} diff --git a/schemas/state-engine-event.schema.json b/schemas/state-engine-event.schema.json new file mode 100644 index 00000000..97b77069 --- /dev/null +++ b/schemas/state-engine-event.schema.json @@ -0,0 +1,107 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://development-kit.dev/schemas/state-engine-event.schema.json", + "title": "Development Kit State Engine V2 Event", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "sequence", + "eventId", + "timestamp", + "actorClass", + "eventType", + "entityType", + "entityId", + "lifecycleId", + "taskId", + "contractId", + "runId", + "payload", + "previousEventHash", + "eventHash" + ], + "properties": { + "schemaVersion": { + "const": "1.0.0" + }, + "sequence": { + "type": "integer", + "minimum": 1 + }, + "eventId": { + "type": "string", + "pattern": "^evt-[0-9]{10}-[a-f0-9]{20}$" + }, + "timestamp": { + "type": "string", + "format": "date-time" + }, + "actorClass": { + "type": "string", + "minLength": 1 + }, + "eventType": { + "type": "string", + "pattern": "^[A-Z][A-Z0-9_]{1,95}$" + }, + "entityType": { + "type": [ + "string", + "null" + ] + }, + "entityId": { + "type": [ + "string", + "null" + ] + }, + "lifecycleId": { + "type": [ + "string", + "null" + ] + }, + "taskId": { + "type": [ + "string", + "null" + ] + }, + "contractId": { + "type": [ + "string", + "null" + ] + }, + "runId": { + "type": [ + "string", + "null" + ] + }, + "payload": { + "type": "object" + }, + "previousEventHash": { + "anyOf": [ + { + "$ref": "#/$defs/sha256" + }, + { + "type": "null" + } + ] + }, + "eventHash": { + "$ref": "#/$defs/sha256" + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + } + } +} diff --git a/schemas/state-engine-index.schema.json b/schemas/state-engine-index.schema.json new file mode 100644 index 00000000..d7ed58a2 --- /dev/null +++ b/schemas/state-engine-index.schema.json @@ -0,0 +1,65 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://development-kit.dev/schemas/state-engine-index.schema.json", + "title": "Development Kit State Engine V2 Disposable Index", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "sourceSequence", + "sourceHash", + "recordCount", + "records", + "byContract", + "byTask", + "byRun", + "byLifecycle" + ], + "properties": { + "schemaVersion": { + "const": "1.0.0" + }, + "sourceSequence": { + "type": "integer", + "minimum": 0 + }, + "sourceHash": { + "anyOf": [ + { + "$ref": "#/$defs/sha256" + }, + { + "type": "null" + } + ] + }, + "recordCount": { + "type": "integer", + "minimum": 0 + }, + "records": { + "type": "array", + "items": { + "type": "object" + } + }, + "byContract": { + "type": "object" + }, + "byTask": { + "type": "object" + }, + "byRun": { + "type": "object" + }, + "byLifecycle": { + "type": "object" + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + } + } +} diff --git a/schemas/state-engine-snapshot.schema.json b/schemas/state-engine-snapshot.schema.json new file mode 100644 index 00000000..729cb03d --- /dev/null +++ b/schemas/state-engine-snapshot.schema.json @@ -0,0 +1,82 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://development-kit.dev/schemas/state-engine-snapshot.schema.json", + "title": "Development Kit State Engine V2 Snapshot", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "engineSchemaVersion", + "lastEventSequence", + "lastEventHash", + "entities", + "migrations" + ], + "properties": { + "schemaVersion": { + "const": "2.0.0" + }, + "engineSchemaVersion": { + "const": "2.0.0" + }, + "lastEventSequence": { + "type": "integer", + "minimum": 0 + }, + "lastEventHash": { + "anyOf": [ + { + "$ref": "#/$defs/sha256" + }, + { + "type": "null" + } + ] + }, + "entities": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "stateRevision", + "state", + "lastEventSequence", + "lastEventHash", + "refs" + ], + "properties": { + "stateRevision": { + "type": "integer", + "minimum": 1 + }, + "state": {}, + "lastEventSequence": { + "type": "integer", + "minimum": 1 + }, + "lastEventHash": { + "$ref": "#/$defs/sha256" + }, + "refs": { + "type": "object" + } + } + } + } + }, + "migrations": { + "type": "array", + "items": { + "type": "object" + } + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + } + } +} diff --git a/schemas/workspace-target-registry.schema.json b/schemas/workspace-target-registry.schema.json new file mode 100644 index 00000000..e5a7f3b6 --- /dev/null +++ b/schemas/workspace-target-registry.schema.json @@ -0,0 +1,88 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://development-kit.dev/schemas/workspace-target-registry.schema.json", + "title": "Development Kit Workspace Target Registry", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "targets" + ], + "properties": { + "schemaVersion": { + "const": 1 + }, + "targets": { + "type": "object", + "minProperties": 1, + "propertyNames": { + "pattern": "^[A-Za-z0-9@][A-Za-z0-9._:@/-]{0,127}$" + }, + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "kind", + "dependencies", + "commands", + "capabilities", + "verification", + "adapter" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "kind": { + "type": [ + "string", + "null" + ] + }, + "dependencies": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z0-9@][A-Za-z0-9._:@/-]{0,127}$" + }, + "uniqueItems": true + }, + "commands": { + "type": "object", + "propertyNames": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + } + }, + "capabilities": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "verification": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" + }, + "uniqueItems": true + }, + "adapter": { + "type": [ + "object", + "null" + ] + } + } + } + } + } +} diff --git a/scripts/authority-graph.test.mjs b/scripts/authority-graph.test.mjs index ef982db0..36ca9a9d 100644 --- a/scripts/authority-graph.test.mjs +++ b/scripts/authority-graph.test.mjs @@ -67,3 +67,46 @@ test('AuthorityGraph: buildAuthorityGraphFromContract converts contract and veri assert.ok(json.edges.length >= 3); }); + + +test('AuthorityGraph: explicitly maps multiple requirements to one existing criterion with evidence', () => { + const contract = { + contractId: 'INC-AUTH-MULTI', + taskId: 'TASK-MULTI', + status: 'approved', + scope: {}, + requirements: ['Alpha requirement', 'Beta requirement', 'Gamma requirement'], + acceptanceCriteria: [ + { id: 'AC-MULTI-1', requirementIds: ['Alpha requirement', 'Beta requirement'] }, + { id: 'AC-MULTI-2', requirementIds: ['Gamma requirement'] }, + ], + }; + const verified = { + verdict: 'PASS', + criteria: [ + { id: 'AC-MULTI-1', status: 'PASS', trustLevel: 'E3' }, + { id: 'AC-MULTI-2', status: 'PASS', trustLevel: 'E3' }, + ], + }; + const graph = buildAuthorityGraphFromContract({contract, verification: verified}); + assert.equal(graph.validateTraceability().complete, true); + assert.deepEqual(graph.getDownstream('Beta requirement'), ['TASK-MULTI', 'AC-MULTI-1']); + const withoutMapping = structuredClone(contract); + withoutMapping.acceptanceCriteria[0].requirementIds = ['Alpha requirement']; + const missing = buildAuthorityGraphFromContract({contract: withoutMapping, verification: verified}); + assert.equal(missing.validateTraceability().complete, false); + assert.deepEqual(missing.validateTraceability().unverifiedRequirements, ['Beta requirement']); +}); + +test('AuthorityGraph: explicit unknown requirement mapping fails closed', () => { + const contract = { + contractId: 'INC-AUTH-INVALID', + taskId: 'TASK-INVALID', + status: 'approved', + scope: {}, + requirements: ['Known requirement'], + acceptanceCriteria: [{ id: 'AC-INVALID', requirementIds: ['Nonexistent requirement'] }], + }; + assert.throws(() => buildAuthorityGraphFromContract({contract, verification: null}), + /explicitly references missing requirement/i); +}); diff --git a/scripts/autopilot.test.mjs b/scripts/autopilot.test.mjs index 22f382f6..40e16aac 100644 --- a/scripts/autopilot.test.mjs +++ b/scripts/autopilot.test.mjs @@ -89,14 +89,14 @@ test('3. Immutable Revision Persistence & Reading', () => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); -test('4. Corrupt Pointer Recovery', () => { +test('4. Corrupt Materialized Snapshot Recovery', () => { const tmpDir = createTempDir(); const state1 = createInitialState({ autonomy: 'guided-autopilot' }, tmpDir); saveStateRevision(state1, tmpDir); - // Corrupt current.json - const currentFile = path.join(tmpDir, '.development-kit', 'autopilot', 'state', 'current.json'); - fs.writeFileSync(currentFile, '{ "corrupt": true }', 'utf8'); + // State Engine V2 canonical history must rebuild a corrupt derived snapshot. + const snapshotFile = path.join(tmpDir, '.development-kit', 'state', 'snapshot.json'); + fs.writeFileSync(snapshotFile, '{ "corrupt": true }', 'utf8'); const recovered = getCurrentState(tmpDir); assert.ok(recovered); diff --git a/scripts/configuration-readiness-integration.test.mjs b/scripts/configuration-readiness-integration.test.mjs index 3b3a2161..f8c27e1a 100644 --- a/scripts/configuration-readiness-integration.test.mjs +++ b/scripts/configuration-readiness-integration.test.mjs @@ -30,6 +30,7 @@ import { } from '../runtime/orchestration/plan-validator.mjs'; import { + DEVELOPMENT_CONTRACT_SCHEMA_VERSION, createDevelopmentContract, validateDevelopmentContract, } from '../runtime/orchestration/development-contract.mjs'; @@ -150,7 +151,7 @@ test('CFG-INT-02: Current Contract schema supports configurationDependencies and createdAt: '2026-09-01T12:00:00.000Z', }); - assert.equal(contract.schemaVersion, '1.2.0'); + assert.equal(contract.schemaVersion, DEVELOPMENT_CONTRACT_SCHEMA_VERSION); assert.equal(contract.configurationDependencies.length, 1); assert.equal(contract.configurationDependencies[0].name, 'STRIPE_SECRET_KEY'); assert.equal(validateDevelopmentContract(contract), true); diff --git a/scripts/context-cache.test.mjs b/scripts/context-cache.test.mjs new file mode 100644 index 00000000..f9c787a4 --- /dev/null +++ b/scripts/context-cache.test.mjs @@ -0,0 +1,284 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + clearContextCache, + loadContextCacheEntry, + persistContextCacheEntry, + resolveRepositoryContextCache, + validateContextCacheEntry, +} from '../runtime/orchestration/context-cache.mjs'; +import { createDevelopmentContract } from '../runtime/orchestration/development-contract.mjs'; +import { persistWorkspaceTargetRegistry } from '../runtime/orchestration/workspace-targets.mjs'; + +function tempProject(t) { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dk-context-cache-')); + t.after(() => fs.rmSync(rootDir, { recursive: true, force: true })); + fs.mkdirSync(path.join(rootDir, 'docs'), { recursive: true }); + fs.mkdirSync(path.join(rootDir, 'units', 'alpha', 'tests'), { recursive: true }); + fs.mkdirSync(path.join(rootDir, 'units', 'beta'), { recursive: true }); + fs.writeFileSync(path.join(rootDir, 'docs', 'spec.md'), '# Spec\n\n## REQ-1\nBuild the approved behavior.\n', 'utf8'); + fs.writeFileSync(path.join(rootDir, 'units', 'alpha', 'index.mjs'), 'export const alpha = 1;\n', 'utf8'); + fs.writeFileSync(path.join(rootDir, 'units', 'alpha', 'tests', 'alpha.test.mjs'), 'export const pass = true;\n', 'utf8'); + fs.writeFileSync(path.join(rootDir, 'units', 'beta', 'index.mjs'), 'export const beta = 1;\n', 'utf8'); + + persistWorkspaceTargetRegistry({ + schemaVersion: 1, + targets: { + alpha: { + path: 'units/alpha', + kind: 'project-defined-a', + dependencies: [], + commands: { test: 'node tests/alpha.test.mjs' }, + capabilities: ['quality'], + verification: ['test'], + adapter: null, + }, + beta: { + path: 'units/beta', + kind: 'project-defined-b', + dependencies: [], + commands: {}, + capabilities: [], + verification: [], + adapter: null, + }, + }, + }, rootDir); + + return rootDir; +} + +function task(targetBinding = { + primary: 'alpha', + affected: ['alpha'], + verification: ['alpha'], +}) { + return { + id: 'TASK-CACHE', + projectId: 'proj-cache', + status: 'approved', + objective: 'Exercise deterministic repository context caching', + scope: { in: ['units/alpha'], out: [] }, + requirements: ['REQ-1'], + acceptanceCriteria: [{ + id: 'AC-CACHE', + statement: 'Cache preserves correctness', + source: 'REQ-1', + verificationType: ['test'], + requiredEvidence: true, + }], + architectureConstraints: [], + designConstraints: [], + securityConstraints: [], + risk: { level: 2, reasons: [] }, + requiredVerification: ['tests'], + requiredReviewers: ['code-reviewer'], + targetBinding, + }; +} + +function contract(rootDir, targetBinding) { + return createDevelopmentContract({ + rootDir, + task: task(targetBinding), + authoritativeSources: [{ + path: 'docs/spec.md', + kind: 'specification', + authority: 'required', + sections: ['REQ-1'], + }], + createdAt: '2026-10-07T19:00:00.000Z', + }); +} + +test('AC-010 cache miss then deterministic hit reuses the same semantic entry', (t) => { + const rootDir = tempProject(t); + const c = contract(rootDir); + + const first = resolveRepositoryContextCache({ + contract: c, + rootDir, + relevantFiles: ['units/alpha/index.mjs'], + relevantTests: ['units/alpha/tests/alpha.test.mjs'], + createdAt: '2026-10-07T19:01:00.000Z', + }); + assert.equal(first.status, 'MISS'); + assert.equal(first.metrics.cacheMisses, 1); + assert.equal(first.metrics.orientationRebuilds, 1); + assert.equal(validateContextCacheEntry(first.entry), true); + + const second = resolveRepositoryContextCache({ + contract: c, + rootDir, + relevantFiles: ['units/alpha/index.mjs'], + relevantTests: ['units/alpha/tests/alpha.test.mjs'], + createdAt: '2026-10-07T19:02:00.000Z', + }); + assert.equal(second.status, 'HIT'); + assert.equal(second.metrics.cacheHits, 1); + assert.equal(second.metrics.orientationRebuilds, 0); + assert.equal(second.entry.cacheId, first.entry.cacheId); + assert.equal(second.entry.inputFingerprint, first.entry.inputFingerprint); + assert.equal(second.entry.entryFingerprint, first.entry.entryFingerprint); +}); + +test('AC-010 relevant file content change invalidates the cached input deterministically', (t) => { + const rootDir = tempProject(t); + const c = contract(rootDir); + + const first = resolveRepositoryContextCache({ + contract: c, + rootDir, + relevantFiles: ['units/alpha/index.mjs'], + }); + fs.writeFileSync(path.join(rootDir, 'units', 'alpha', 'index.mjs'), 'export const alpha = 2;\n', 'utf8'); + + const second = resolveRepositoryContextCache({ + contract: c, + rootDir, + relevantFiles: ['units/alpha/index.mjs'], + }); + assert.equal(second.status, 'MISS'); + assert.equal(second.invalidationReason, 'INPUT_FINGERPRINT_CHANGED'); + assert.notEqual(second.entry.inputFingerprint, first.entry.inputFingerprint); +}); + +test('AC-010 relevant target structure change invalidates structural cache facts', (t) => { + const rootDir = tempProject(t); + const c = contract(rootDir); + + const first = resolveRepositoryContextCache({ contract: c, rootDir }); + fs.writeFileSync(path.join(rootDir, 'units', 'alpha', 'new-module.mjs'), 'export {};\n', 'utf8'); + const second = resolveRepositoryContextCache({ contract: c, rootDir }); + + assert.equal(second.status, 'MISS'); + assert.equal(second.invalidationReason, 'INPUT_FINGERPRINT_CHANGED'); + assert.notEqual( + second.entry.targetFacts[0].structureFingerprint, + first.entry.targetFacts[0].structureFingerprint, + ); +}); + +test('irrelevant target file and structure changes do not invalidate an alpha-scoped cache', (t) => { + const rootDir = tempProject(t); + const c = contract(rootDir); + + const first = resolveRepositoryContextCache({ + contract: c, + rootDir, + targetIds: ['alpha'], + relevantFiles: ['units/alpha/index.mjs'], + }); + + fs.writeFileSync(path.join(rootDir, 'units', 'beta', 'index.mjs'), 'export const beta = 99;\n', 'utf8'); + fs.writeFileSync(path.join(rootDir, 'units', 'beta', 'extra.mjs'), 'export {};\n', 'utf8'); + + const second = resolveRepositoryContextCache({ + contract: c, + rootDir, + targetIds: ['alpha'], + relevantFiles: ['units/alpha/index.mjs'], + }); + + assert.equal(second.status, 'HIT'); + assert.equal(second.entry.inputFingerprint, first.entry.inputFingerprint); + assert.equal(second.metrics.repositoryScans, 1); +}); + +test('corrupt cache content is never trusted and is rebuilt as a miss', (t) => { + const rootDir = tempProject(t); + const c = contract(rootDir); + const first = resolveRepositoryContextCache({ contract: c, rootDir }); + fs.writeFileSync(first.cachePath, '{"schemaVersion":"broken"}\n', 'utf8'); + + const rebuilt = resolveRepositoryContextCache({ contract: c, rootDir }); + assert.equal(rebuilt.status, 'MISS'); + assert.equal(rebuilt.invalidationReason, 'CACHE_ENTRY_INVALID'); + assert.equal(validateContextCacheEntry(rebuilt.entry), true); +}); + +test('cache deletion degrades to a correct miss rather than becoming authority loss', (t) => { + const rootDir = tempProject(t); + const c = contract(rootDir); + const first = resolveRepositoryContextCache({ contract: c, rootDir }); + assert.equal(first.status, 'MISS'); + + clearContextCache(rootDir); + const rebuilt = resolveRepositoryContextCache({ contract: c, rootDir }); + assert.equal(rebuilt.status, 'MISS'); + assert.equal(validateContextCacheEntry(rebuilt.entry), true); +}); + +test('cache load/write helpers reject paths outside the project context-cache root', (t) => { + const rootDir = tempProject(t); + const c = contract(rootDir); + const resolved = resolveRepositoryContextCache({ contract: c, rootDir }); + const outside = path.join(rootDir, 'outside-cache.json'); + + assert.throws( + () => persistContextCacheEntry(resolved.entry, outside, rootDir), + /escapes the cache root/, + ); + fs.writeFileSync(outside, JSON.stringify(resolved.entry), 'utf8'); + assert.throws( + () => loadContextCacheEntry(outside, rootDir), + /escapes the cache root/, + ); +}); + +test('persisted configuration-registry changes deterministically invalidate cached context', (t) => { + const rootDir = tempProject(t); + const c = contract(rootDir); + const first = resolveRepositoryContextCache({ contract: c, rootDir }); + assert.equal(first.status, 'MISS'); + + const secretsDir = path.join(rootDir, '.development-kit', 'secrets'); + fs.mkdirSync(secretsDir, { recursive: true }); + fs.writeFileSync(path.join(secretsDir, 'requirements.json'), JSON.stringify({ + schemaVersion: '1.0.0', + updatedAt: '2026-10-07T19:30:00.000Z', + requirements: [{ + id: 'CFG-001', + name: 'PROJECT_IDENTIFIER', + kind: 'identifier', + status: 'VALID', + }], + }, null, 2), 'utf8'); + + const second = resolveRepositoryContextCache({ contract: c, rootDir }); + assert.equal(second.status, 'MISS'); + assert.equal(second.invalidationReason, 'INPUT_FINGERPRINT_CHANGED'); + assert.notEqual(second.entry.inputFingerprint, first.entry.inputFingerprint); +}); + +test('legacy non-target-aware contracts remain supported through one project-root cache boundary', (t) => { + const rootDir = tempProject(t); + const legacy = createDevelopmentContract({ + rootDir, + task: { + ...task(), + id: 'TASK-LEGACY-CACHE', + targetBinding: undefined, + }, + authoritativeSources: [{ + path: 'docs/spec.md', + kind: 'specification', + authority: 'required', + sections: ['REQ-1'], + }], + createdAt: '2026-10-07T19:03:00.000Z', + }); + + const resolved = resolveRepositoryContextCache({ + contract: legacy, + rootDir, + relevantFiles: ['units/alpha/index.mjs'], + }); + assert.equal(resolved.status, 'MISS'); + assert.equal(resolved.entry.targetFacts[0].targetId, '__project_root__'); + assert.equal(resolved.entry.targetFacts[0].root, '.'); +}); diff --git a/scripts/cost-observatory.test.mjs b/scripts/cost-observatory.test.mjs new file mode 100644 index 00000000..50195208 --- /dev/null +++ b/scripts/cost-observatory.test.mjs @@ -0,0 +1,222 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + COST_ESTIMATOR, + appendCostRecord, + compareCostRecords, + createCostMeter, + createCostRecord, + loadCostRecords, + measureLogicalText, + normalizeLogicalText, + validateCostRecord, +} from '../runtime/orchestration/cost-observatory.mjs'; +import { createPolicyBoundDevelopmentContract } from '../runtime/orchestration/contract-policy.mjs'; +import { createVerificationRecord } from '../runtime/orchestration/evidence-store.mjs'; +import { createReviewResult } from '../runtime/orchestration/review-result.mjs'; +import { decideAcceptance } from '../runtime/orchestration/acceptance-engine.mjs'; + +function tempProject(t) { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dk-cost-observatory-')); + t.after(() => fs.rmSync(rootDir, { recursive: true, force: true })); + return rootDir; +} + +function record(overrides = {}) { + return createCostRecord({ + operationId: 'operation-001', + contractId: 'INC-DKF120-T01', + taskId: 'DKF120-T01', + role: 'implementation-agent', + context: 'alpha\nbeta\n', + startedAt: '2026-10-07T17:00:00.000Z', + completedAt: '2026-10-07T17:00:01.250Z', + ...overrides, + }); +} + +test('AC-003 newline-normalised logical metrics are identical across LF and CRLF', () => { + const lf = 'alpha\nbeta\ngamma\n'; + const crlf = 'alpha\r\nbeta\r\ngamma\r\n'; + + assert.equal(normalizeLogicalText(crlf), lf); + assert.deepEqual(measureLogicalText(crlf), measureLogicalText(lf)); + assert.equal(measureLogicalText(lf).estimator, COST_ESTIMATOR); +}); + +test('AC-001 cost meter records only explicit DKF-controlled counters', () => { + const meter = createCostMeter(); + meter.increment('repositoryReads', 3); + meter.increment('toolCalls', 2); + meter.increment('verificationCommands'); + meter.increment('cacheHits', 4); + + const counters = meter.snapshot(); + assert.deepEqual(counters, { + repositoryReads: 3, + repositoryScans: 0, + agentInvocations: 0, + toolCalls: 2, + verificationCommands: 1, + cacheHits: 4, + cacheMisses: 0, + }); + + const cost = record({ counters }); + assert.equal(cost.repositoryReads, 3); + assert.equal(cost.toolCalls, 2); + assert.equal(cost.verificationCommands, 1); + assert.equal(cost.cacheHits, 4); + assert.equal(validateCostRecord(cost), true); +}); + +test('AC-002 provider token fields are null unless provider telemetry is supplied', () => { + const unavailable = record(); + assert.equal(unavailable.providerInputTokens, null); + assert.equal(unavailable.providerOutputTokens, null); + assert.equal(unavailable.providerCachedTokens, null); + + const observed = record({ + operationId: 'operation-provider', + providerUsage: { + inputTokens: 1200, + outputTokens: 300, + cachedTokens: 700, + }, + }); + assert.equal(observed.providerInputTokens, 1200); + assert.equal(observed.providerOutputTokens, 300); + assert.equal(observed.providerCachedTokens, 700); +}); + +test('unknown host counters stay unavailable instead of becoming inferred zeroes', () => { + const cost = record(); + assert.equal(cost.repositoryReads, null); + assert.equal(cost.repositoryScans, null); + assert.equal(cost.agentInvocations, null); + assert.equal(cost.toolCalls, null); + assert.equal(cost.verificationCommands, null); + assert.equal(cost.cacheHits, 0); + assert.equal(cost.cacheMisses, 0); +}); + +test('invalid cost values fail closed', () => { + assert.throws(() => record({ providerUsage: { inputTokens: -1 } }), /non-negative integer/); + assert.throws(() => record({ + startedAt: '2026-10-07T17:00:02.000Z', + completedAt: '2026-10-07T17:00:01.000Z', + }), /may not be before/); + assert.throws(() => createCostMeter().increment('not-a-counter'), /Unsupported counter/); +}); + +test('cost ledger uses one append-only JSONL file and validates records on load', (t) => { + const rootDir = tempProject(t); + appendCostRecord(record(), rootDir); + appendCostRecord(record({ operationId: 'operation-002' }), rootDir); + + const loaded = loadCostRecords(rootDir); + assert.equal(loaded.length, 2); + assert.equal(loaded[0].operationId, 'operation-001'); + assert.equal(loaded[1].operationId, 'operation-002'); + + const ledgerPath = path.join(rootDir, '.development-kit', 'telemetry', 'cost-records.jsonl'); + assert.equal(fs.existsSync(ledgerPath), true); + assert.equal(fs.readdirSync(path.dirname(ledgerPath)).length, 1); +}); + +test('AC-033 comparison reports reductions only for available measurements', () => { + const baseline = record({ + operationId: 'baseline', + context: 'x'.repeat(400), + counters: { repositoryReads: 10, toolCalls: 5 }, + }); + const current = record({ + operationId: 'current', + context: 'x'.repeat(200), + counters: { repositoryReads: 4, toolCalls: 5 }, + }); + + const comparison = compareCostRecords(baseline, current); + assert.equal(comparison.metrics.contextBytes.reductionPercent, 50); + assert.equal(comparison.metrics.repositoryReads.reductionPercent, 60); + assert.equal(comparison.metrics.toolCalls.reductionPercent, 0); + assert.equal(comparison.metrics.providerInputTokens.available, false); + assert.equal(comparison.metrics.providerInputTokens.reductionPercent, null); +}); + +test('Cost Observatory is observational and cannot alter deterministic acceptance', (t) => { + const rootDir = tempProject(t); + fs.mkdirSync(path.join(rootDir, 'docs'), { recursive: true }); + fs.writeFileSync(path.join(rootDir, 'docs', 'spec.md'), '# Spec\nREQ-1 works.\n', 'utf8'); + + const contract = createPolicyBoundDevelopmentContract({ + rootDir, + task: { + id: 'TASK-COST-ACCEPTANCE', + projectId: 'cost-acceptance', + status: 'approved', + objective: 'Prove observability does not affect acceptance', + scope: { in: ['src/'], out: [] }, + requirements: ['REQ-1'], + acceptanceCriteria: [{ + id: 'AC-COST-ACCEPTANCE', + statement: 'Feature passes', + verificationType: ['test'], + requiredEvidence: true, + }], + architectureConstraints: [], + designConstraints: [], + securityConstraints: [], + risk: { level: 0, reasons: [] }, + requiredVerification: ['tests'], + requiredReviewers: ['code-reviewer'], + }, + authoritativeSources: [{ path: 'docs/spec.md', kind: 'specification', authority: 'required' }], + createdAt: '2026-10-07T17:00:00.000Z', + }); + + const verification = createVerificationRecord({ + contract, + runId: 'run-cost-acceptance', + role: 'spec-verifier', + sourceFingerprint: contract.sourceFingerprint, + createdAt: '2026-10-07T17:01:00.000Z', + criteria: [{ + id: 'AC-COST-ACCEPTANCE', + status: 'PASS', + evidence: [{ type: 'test', id: 'cost-observatory-tests' }], + }], + }); + + const review = createReviewResult({ + contract, + runId: 'run-cost-acceptance', + role: 'code-reviewer', + sourceFingerprint: contract.sourceFingerprint, + createdAt: '2026-10-07T17:02:00.000Z', + findings: [], + }); + + const before = decideAcceptance({ contract, verification, reviews: [review], rootDir }); + appendCostRecord(record({ operationId: 'acceptance-observation' }), rootDir); + const after = decideAcceptance({ contract, verification, reviews: [review], rootDir }); + + assert.equal(before.state, 'ACCEPTED'); + assert.equal(after.state, 'ACCEPTED'); + assert.deepEqual( + { ...after, createdAt: '' }, + { ...before, createdAt: '' }, + ); +}); + +test('cost-record JSON schema is present and describes the runtime version', () => { + const schemaPath = new URL('../schemas/cost-record.schema.json', import.meta.url); + const schema = JSON.parse(fs.readFileSync(schemaPath, 'utf8')); + assert.equal(schema.title, 'Development Kit Cost Record'); + assert.equal(schema.properties.schemaVersion.const, '1.0.0'); + assert.equal(schema.properties.contextEstimator.const, COST_ESTIMATOR); +}); diff --git a/scripts/execution-capsule.test.mjs b/scripts/execution-capsule.test.mjs new file mode 100644 index 00000000..4ee9fada --- /dev/null +++ b/scripts/execution-capsule.test.mjs @@ -0,0 +1,431 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + captureRepositoryContextInputs, + clearContextCache, + resolveRepositoryContextCache, +} from '../runtime/orchestration/context-cache.mjs'; +import { + checkExecutionCapsuleStaleness, + createExecutionCapsule, + prepareExecutionCapsule, + validateExecutionCapsule, +} from '../runtime/orchestration/execution-capsule.mjs'; +import { + buildContextPackage, + assertIndependentVerificationContext, +} from '../runtime/orchestration/context-package.mjs'; +import { createDevelopmentContract } from '../runtime/orchestration/development-contract.mjs'; +import { + compareCostRecords, + createCostRecord, +} from '../runtime/orchestration/cost-observatory.mjs'; +import { persistWorkspaceTargetRegistry } from '../runtime/orchestration/workspace-targets.mjs'; + +function tempProject(t, { bindBeta = false, manyFiles = false } = {}) { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dk-execution-capsule-')); + t.after(() => fs.rmSync(rootDir, { recursive: true, force: true })); + fs.mkdirSync(path.join(rootDir, 'docs'), { recursive: true }); + fs.mkdirSync(path.join(rootDir, 'units', 'alpha', 'tests'), { recursive: true }); + fs.mkdirSync(path.join(rootDir, 'units', 'beta', 'tests'), { recursive: true }); + + fs.writeFileSync( + path.join(rootDir, 'docs', 'spec.md'), + '# Specification\n\n## REQ-1\nApproved behavior only.\n\n## REQ-2\nUnrelated requirement.\n', + 'utf8', + ); + fs.writeFileSync(path.join(rootDir, 'units', 'alpha', 'index.mjs'), 'export const alpha = 1;\n', 'utf8'); + fs.writeFileSync(path.join(rootDir, 'units', 'alpha', 'tests', 'alpha.test.mjs'), 'export const alphaTest = true;\n', 'utf8'); + fs.writeFileSync(path.join(rootDir, 'units', 'beta', 'index.mjs'), 'export const beta = 1;\n', 'utf8'); + fs.writeFileSync(path.join(rootDir, 'units', 'beta', 'tests', 'beta.test.mjs'), 'export const betaTest = true;\n', 'utf8'); + + if (manyFiles) { + for (let i = 0; i < 80; i += 1) { + fs.writeFileSync(path.join(rootDir, 'units', 'alpha', `module-${String(i).padStart(3, '0')}.mjs`), `export const a${i} = ${i};\n`, 'utf8'); + fs.writeFileSync(path.join(rootDir, 'units', 'beta', `module-${String(i).padStart(3, '0')}.mjs`), `export const b${i} = ${i};\n`, 'utf8'); + } + } + + persistWorkspaceTargetRegistry({ + schemaVersion: 1, + targets: { + alpha: { + path: 'units/alpha', + kind: 'arbitrary-a', + dependencies: [], + commands: { test: 'node tests/alpha.test.mjs' }, + capabilities: ['quality'], + verification: ['test'], + adapter: null, + }, + beta: { + path: 'units/beta', + kind: 'arbitrary-b', + dependencies: [], + commands: { test: 'node tests/beta.test.mjs' }, + capabilities: ['quality'], + verification: ['test'], + adapter: null, + }, + }, + }, rootDir); + + const task = { + id: bindBeta ? 'TASK-CAPSULE-COMPARE' : 'TASK-CAPSULE', + projectId: 'proj-capsule', + status: 'approved', + objective: 'Build deterministic compact role context', + scope: { in: ['units/alpha'], out: ['Do not change unrelated targets'] }, + requirements: ['REQ-1'], + acceptanceCriteria: [{ + id: 'AC-CAPSULE', + statement: 'Capsule remains fresh and source-authoritative', + source: 'REQ-1', + verificationType: ['test'], + requiredEvidence: true, + }], + architectureConstraints: [], + designConstraints: [], + securityConstraints: [], + risk: { level: 2, reasons: [] }, + requiredVerification: ['tests'], + requiredReviewers: ['code-reviewer'], + targetBinding: { + primary: 'alpha', + affected: ['alpha'], + verification: bindBeta ? ['alpha', 'beta'] : ['alpha'], + }, + }; + + const contract = createDevelopmentContract({ + rootDir, + task, + authoritativeSources: [{ + path: 'docs/spec.md', + kind: 'specification', + authority: 'required', + sections: ['REQ-1'], + }], + createdAt: '2026-10-07T19:10:00.000Z', + }); + + return { rootDir, contract }; +} + +function capsuleInput(contract, rootDir, overrides = {}) { + return { + contract, + rootDir, + targetIds: ['alpha'], + relevantFiles: ['units/alpha/index.mjs'], + relevantTests: ['units/alpha/tests/alpha.test.mjs'], + changedFiles: ['units/alpha/index.mjs'], + runId: 'run-capsule', + lifecycleId: 'life-capsule', + verifiedRuntimeFacts: [{ + id: 'fact-node-runtime', + value: { available: true }, + observedAt: '2026-10-07T19:11:00.000Z', + provenance: { + kind: 'verification-command', + id: 'node-version-check', + fingerprint: 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + }, + }], + upstreamAcceptedTasks: [{ + taskId: 'TASK-UPSTREAM', + contractId: 'INC-TASK-UPSTREAM', + sourceFingerprint: 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + acceptanceFingerprint: 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + }], + requiredGateProfileIds: ['standard-current-gates'], + createdAt: '2026-10-07T19:12:00.000Z', + ...overrides, + }; +} + +test('AC-008 identical semantic inputs produce the same deterministic capsule across cache miss/hit', (t) => { + const { rootDir, contract } = tempProject(t); + const first = prepareExecutionCapsule(capsuleInput(contract, rootDir)); + const second = prepareExecutionCapsule(capsuleInput(contract, rootDir, { + createdAt: '2026-10-07T19:13:00.000Z', + })); + + assert.equal(first.cache.status, 'MISS'); + assert.equal(second.cache.status, 'HIT'); + assert.equal(first.capsule.capsuleId, second.capsule.capsuleId); + assert.equal(first.capsule.capsuleFingerprint, second.capsule.capsuleFingerprint); + assert.equal(validateExecutionCapsule(second.capsule), true); +}); + +test('AC-008 capsule feeds fresh implementation and independent verification contexts without replacing source rehydration', (t) => { + const { rootDir, contract } = tempProject(t); + const prepared = prepareExecutionCapsule(capsuleInput(contract, rootDir)); + + const implementation = buildContextPackage({ + contract, + role: 'implementation-agent', + rootDir, + executionCapsule: prepared.capsule, + createdAt: '2026-10-07T19:14:00.000Z', + }); + + assert.equal(implementation.executionCapsule.capsuleId, prepared.capsule.capsuleId); + assert.equal(implementation.isolationMetadata.repositoryReRead, false); + assert.equal(implementation.isolationMetadata.repositoryContextFromCapsule, true); + assert.equal(implementation.isolationMetadata.capsuleFreshnessVerified, true); + assert.equal(implementation.isolationMetadata.authoritativeSourcesReRead, true); + assert.equal(implementation.authoritativeSources[0].delivery.mode, 'scoped'); + assert.match(implementation.authoritativeSources[0].content, /Approved behavior only/); + assert.doesNotMatch(implementation.authoritativeSources[0].content, /Unrelated requirement/); + + const verifier = buildContextPackage({ + contract, + role: 'spec-reviewer', + rootDir, + executionCapsule: prepared.capsule, + createdAt: '2026-10-07T19:15:00.000Z', + }); + assert.equal(assertIndependentVerificationContext(verifier), true); + assert.equal(verifier.isolationMetadata.sourceRehydrated, true); +}); + +test('AC-009 relevant repository file changes invalidate a capsule and block role-context construction', (t) => { + const { rootDir, contract } = tempProject(t); + const prepared = prepareExecutionCapsule(capsuleInput(contract, rootDir)); + + fs.writeFileSync(path.join(rootDir, 'units', 'alpha', 'index.mjs'), 'export const alpha = 2;\n', 'utf8'); + + const stale = checkExecutionCapsuleStaleness({ + capsule: prepared.capsule, + contract, + rootDir, + }); + assert.equal(stale.stale, true); + assert.ok(stale.changes.some((change) => change.code === 'CAPSULE_CACHE_STALE')); + + assert.throws(() => buildContextPackage({ + contract, + role: 'implementation-agent', + rootDir, + executionCapsule: prepared.capsule, + }), /stale Execution Capsule/); +}); + +test('AC-009 authoritative source changes still invalidate the contract/capsule before context construction', (t) => { + const { rootDir, contract } = tempProject(t); + const prepared = prepareExecutionCapsule(capsuleInput(contract, rootDir)); + + fs.appendFileSync(path.join(rootDir, 'docs', 'spec.md'), '\nChanged authority.\n', 'utf8'); + const stale = checkExecutionCapsuleStaleness({ + capsule: prepared.capsule, + contract, + rootDir, + }); + + assert.equal(stale.stale, true); + assert.ok(stale.changes.some((change) => change.code === 'CAPSULE_CONTRACT_STALE')); + assert.throws(() => buildContextPackage({ + contract, + role: 'spec-reviewer', + rootDir, + executionCapsule: prepared.capsule, + }), /stale Development Contract|stale Execution Capsule/); +}); + +test('unrelated target changes do not invalidate a target-scoped capsule when authority remains unchanged', (t) => { + const { rootDir, contract } = tempProject(t); + const prepared = prepareExecutionCapsule(capsuleInput(contract, rootDir)); + + fs.writeFileSync(path.join(rootDir, 'units', 'beta', 'index.mjs'), 'export const beta = 99;\n', 'utf8'); + fs.writeFileSync(path.join(rootDir, 'units', 'beta', 'extra.mjs'), 'export {};\n', 'utf8'); + + const report = checkExecutionCapsuleStaleness({ + capsule: prepared.capsule, + contract, + rootDir, + }); + assert.equal(report.stale, false); + + const next = prepareExecutionCapsule(capsuleInput(contract, rootDir, { + changedFiles: [], + createdAt: '2026-10-07T19:16:00.000Z', + })); + assert.equal(next.cache.status, 'HIT'); +}); + +test('missing disposable cache invalidates the old capsule, while a full cache miss rebuilds correct context', (t) => { + const { rootDir, contract } = tempProject(t); + const prepared = prepareExecutionCapsule(capsuleInput(contract, rootDir)); + clearContextCache(rootDir); + + const stale = checkExecutionCapsuleStaleness({ + capsule: prepared.capsule, + contract, + rootDir, + }); + assert.equal(stale.stale, true); + assert.ok(stale.changes.some((change) => change.code === 'CAPSULE_CACHE_ENTRY_MISSING')); + + const rebuilt = prepareExecutionCapsule(capsuleInput(contract, rootDir, { + createdAt: '2026-10-07T19:17:00.000Z', + })); + assert.equal(rebuilt.cache.status, 'MISS'); + + const context = buildContextPackage({ + contract, + role: 'implementation-agent', + rootDir, + executionCapsule: rebuilt.capsule, + }); + assert.equal(context.isolationMetadata.capsuleFreshnessVerified, true); +}); + +test('present changed files automatically join the capsule invalidation set', (t) => { + const { rootDir, contract } = tempProject(t); + const prepared = prepareExecutionCapsule(capsuleInput(contract, rootDir, { + relevantFiles: [], + changedFiles: ['units/alpha/index.mjs'], + })); + + assert.deepEqual( + prepared.capsule.repositoryContext.relevantFiles.map((ref) => ref.path), + ['units/alpha/index.mjs'], + ); + + fs.writeFileSync(path.join(rootDir, 'units', 'alpha', 'index.mjs'), 'export const alpha = 55;\n', 'utf8'); + const report = checkExecutionCapsuleStaleness({ + capsule: prepared.capsule, + contract, + rootDir, + }); + assert.equal(report.stale, true); + assert.ok(report.changes.some((change) => change.code === 'CAPSULE_CACHE_STALE')); +}); + +test('createExecutionCapsule rejects forged or internally inconsistent cache entries', (t) => { + const { rootDir, contract } = tempProject(t); + const resolution = resolveRepositoryContextCache({ + contract, + rootDir, + targetIds: ['alpha'], + relevantFiles: ['units/alpha/index.mjs'], + }); + const forged = structuredClone(resolution); + forged.entry.entryFingerprint = 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd'; + + assert.throws(() => createExecutionCapsule({ + contract, + rootDir, + cacheResolution: forged, + changedFiles: [], + createdAt: '2026-10-07T19:18:00.000Z', + }), /cacheResolution entry is invalid/); +}); + +test('capsule stores authority references and provenance, not copied source history or authoritative summaries', (t) => { + const { rootDir, contract } = tempProject(t); + const prepared = prepareExecutionCapsule(capsuleInput(contract, rootDir)); + const capsule = prepared.capsule; + + assert.equal(capsule.authoritativeSources[0].path, 'docs/spec.md'); + assert.equal(Object.hasOwn(capsule.authoritativeSources[0], 'content'), false); + assert.equal(capsule.verifiedRuntimeFacts[0].authority, 'non-authoritative-observation'); + assert.equal(capsule.verifiedRuntimeFacts[0].provenance.kind, 'verification-command'); + assert.deepEqual(capsule.targetDelta.changedTargets, ['alpha']); + assert.deepEqual(capsule.targetDelta.affectedTargets, ['alpha']); + assert.deepEqual(capsule.targetDelta.dependencyTargets, ['alpha']); +}); + +test('AC-035 Cost Observatory reports measured target-orientation and context-volume improvement where observable', (t) => { + const { rootDir, contract } = tempProject(t, { bindBeta: true, manyFiles: true }); + + const baselineSnapshot = captureRepositoryContextInputs({ + contract, + rootDir, + targetIds: ['alpha', 'beta'], + relevantFiles: ['units/alpha/index.mjs', 'units/beta/index.mjs'], + relevantTests: ['units/alpha/tests/alpha.test.mjs', 'units/beta/tests/beta.test.mjs'], + }); + const baselineContext = buildContextPackage({ + contract, + role: 'implementation-agent', + rootDir, + repositoryState: { + authority: 'non-authoritative-repository-orientation', + targetFacts: baselineSnapshot.targetFacts, + }, + createdAt: '2026-10-07T19:20:00.000Z', + }); + + prepareExecutionCapsule(capsuleInput(contract, rootDir, { + changedFiles: [], + createdAt: '2026-10-07T19:21:00.000Z', + })); + const currentPrepared = prepareExecutionCapsule(capsuleInput(contract, rootDir, { + changedFiles: [], + createdAt: '2026-10-07T19:22:00.000Z', + })); + assert.equal(currentPrepared.cache.status, 'HIT'); + + const currentContext = buildContextPackage({ + contract, + role: 'implementation-agent', + rootDir, + executionCapsule: currentPrepared.capsule, + createdAt: '2026-10-07T19:23:00.000Z', + }); + + const baselineRecord = createCostRecord({ + operationId: 'capsule-baseline', + contractId: contract.contractId, + taskId: contract.taskId, + role: 'implementation-agent', + context: JSON.stringify(baselineContext), + counters: { + repositoryReads: baselineSnapshot.metrics.repositoryReads, + repositoryScans: baselineSnapshot.metrics.repositoryScans, + cacheHits: 0, + cacheMisses: 0, + }, + startedAt: '2026-10-07T19:24:00.000Z', + completedAt: '2026-10-07T19:24:01.000Z', + }); + const currentRecord = createCostRecord({ + operationId: 'capsule-current', + contractId: contract.contractId, + taskId: contract.taskId, + role: 'implementation-agent', + context: JSON.stringify(currentContext), + counters: { + repositoryReads: currentPrepared.cache.metrics.repositoryReads, + repositoryScans: currentPrepared.cache.metrics.repositoryScans, + cacheHits: currentPrepared.cache.metrics.cacheHits, + cacheMisses: currentPrepared.cache.metrics.cacheMisses, + }, + startedAt: '2026-10-07T19:25:00.000Z', + completedAt: '2026-10-07T19:25:01.000Z', + }); + + const comparison = compareCostRecords(baselineRecord, currentRecord); + process.stdout.write(`T03_COST_EVIDENCE ${JSON.stringify({ + baselineContextBytes: baselineRecord.contextBytes, + currentContextBytes: currentRecord.contextBytes, + contextBytesReductionPercent: comparison.metrics.contextBytes.reductionPercent, + baselineEstimatedContextTokens: baselineRecord.estimatedContextTokens, + currentEstimatedContextTokens: currentRecord.estimatedContextTokens, + estimatedContextTokensReductionPercent: comparison.metrics.estimatedContextTokens.reductionPercent, + baselineRepositoryScans: baselineRecord.repositoryScans, + currentRepositoryScans: currentRecord.repositoryScans, + repositoryScansReductionPercent: comparison.metrics.repositoryScans.reductionPercent, + cacheHits: currentRecord.cacheHits, + cacheMisses: currentRecord.cacheMisses, + })}\n`); + assert.equal(comparison.metrics.repositoryScans.reductionPercent, 50); + assert.ok(comparison.metrics.contextBytes.reductionPercent > 0); + assert.ok(comparison.metrics.estimatedContextTokens.reductionPercent > 0); +}); diff --git a/scripts/orchestration-contract.test.mjs b/scripts/orchestration-contract.test.mjs index ff5d0db0..a51683fa 100644 --- a/scripts/orchestration-contract.test.mjs +++ b/scripts/orchestration-contract.test.mjs @@ -6,6 +6,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { + DEVELOPMENT_CONTRACT_SCHEMA_VERSION, ContractPersistenceError, ContractValidationError, StaleContractError, @@ -83,7 +84,7 @@ test('ORCH-001 creates a validated contract with stable evidence boundary and ex assert.equal(contract.executionSafety.destructiveOperations, 'explicit-approval'); assert.equal(contract.executionSafety.remoteMutation, 'explicit-contract'); assert.equal(contract.acceptanceCriteria.length, 2); - assert.equal(contract.schemaVersion, '1.2.0'); + assert.equal(contract.schemaVersion, DEVELOPMENT_CONTRACT_SCHEMA_VERSION); assert.equal(contract.developmentMode.resolved.mode, 'balanced'); assert.equal(contract.developmentMode.source, 'backward-compatible-default'); assert.match(contract.developmentMode.fingerprint, /^sha256:[a-f0-9]{64}$/); @@ -240,6 +241,31 @@ test('ORCH-001 runtime validation rejects malformed or broadened persisted contr const unexpectedProperty = structuredClone(contract); unexpectedProperty.executionOverride = 'allow-everything'; assert.throws(() => validateDevelopmentContract(unexpectedProperty), ContractValidationError); + + // P2 regression: malformed requirementIds or malformed requirements structure must throw ContractValidationError, not TypeError + const invalidRequirementIds = structuredClone(contract); + invalidRequirementIds.acceptanceCriteria[0].requirementIds = 'not-an-array'; + assert.throws(() => validateDevelopmentContract(invalidRequirementIds), (err) => { + return err instanceof ContractValidationError && err.details.some((e) => e.includes('requirementIds must be an array')); + }); + + const nullRequirementIds = structuredClone(contract); + nullRequirementIds.acceptanceCriteria[0].requirementIds = null; + assert.throws(() => validateDevelopmentContract(nullRequirementIds), (err) => { + return err instanceof ContractValidationError && err.details.some((e) => e.includes('requirementIds must be an array')); + }); + + const malformedRequirements = structuredClone(contract); + malformedRequirements.requirements = 'not-an-array'; + assert.throws(() => validateDevelopmentContract(malformedRequirements), (err) => { + return err instanceof ContractValidationError && err.details.some((e) => e.includes('requirements must be an array')); + }); + + const primitiveItemsInRequirements = structuredClone(contract); + primitiveItemsInRequirements.requirements = [123, null, 'REQ-01']; + assert.throws(() => validateDevelopmentContract(primitiveItemsInRequirements), (err) => { + return err instanceof ContractValidationError && err.details.some((e) => e.includes('requirements entries must be strings or plain objects')); + }); }); test('ORCH-001 loader fails closed when contract.json is manually corrupted', (t) => { diff --git a/scripts/phase01-scale-context-baseline.mjs b/scripts/phase01-scale-context-baseline.mjs new file mode 100644 index 00000000..14f67c56 --- /dev/null +++ b/scripts/phase01-scale-context-baseline.mjs @@ -0,0 +1,283 @@ +#!/usr/bin/env node + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +import { buildTokenAudit } from './token-audit.mjs'; +import { createPolicyBoundDevelopmentContract } from '../runtime/orchestration/contract-policy.mjs'; +import { buildContextPackage } from '../runtime/orchestration/context-package.mjs'; +import { selectRequiredGates } from '../runtime/orchestration/gate-selector.mjs'; +import { createInitialState } from '../runtime/autopilot/transition-model.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const pkg = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf8')); + +function walk(root, { exclude = [] } = {}) { + const out = []; + if (!fs.existsSync(root)) return out; + for (const entry of fs.readdirSync(root, { withFileTypes: true })) { + const absolute = path.join(root, entry.name); + const rel = path.relative(ROOT, absolute).replaceAll('\\', '/'); + if (exclude.some((prefix) => rel === prefix || rel.startsWith(prefix + '/'))) continue; + if (entry.isDirectory()) out.push(...walk(absolute, { exclude })); + else if (entry.isFile()) { + const stat = fs.statSync(absolute); + out.push({ path: rel, bytes: stat.size }); + } + } + return out; +} + +function tempProject(label) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), `dkf-phase01-${label}-`)); + fs.mkdirSync(path.join(root, 'docs'), { recursive: true }); + return root; +} + +function baseTask(id, objective, requiredVerification = ['tests']) { + return { + id, + projectId: 'phase01-baseline', + status: 'approved', + objective, + scope: { in: ['fixture/'], out: ['No production changes'] }, + requirements: [`REQ-${id}`], + acceptanceCriteria: [{ + id: `AC-${id}`, + statement: objective, + source: `REQ-${id}`, + verificationType: ['test'], + requiredEvidence: true, + }], + architectureConstraints: [], + designConstraints: [], + securityConstraints: [], + risk: { level: 1, reasons: ['Phase 0/1 synthetic benchmark'] }, + requiredVerification, + requiredReviewers: ['code-reviewer'], + }; +} + +function writeFixtureFiles(root, layout) { + const paths = []; + for (const [dir, count] of Object.entries(layout)) { + for (let i = 1; i <= count; i += 1) { + const file = path.join(root, dir, `file-${String(i).padStart(3, '0')}.js`); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `export const value${i} = ${i};\n`, 'utf8'); + paths.push(path.relative(root, file).replaceAll('\\', '/')); + } + } + return paths; +} + +function contextFixture({ label, layout, relevantLines, unrelatedLines, repositoryState }) { + const root = tempProject(label); + try { + const paths = writeFixtureFiles(root, layout); + const req = `REQ-TASK-${label.toUpperCase().replace(/[^A-Z0-9]/g, '-')}`; + const relevant = Array.from({ length: relevantLines }, (_, i) => `Required behaviour ${i + 1}.`); + const unrelated = Array.from({ length: unrelatedLines }, (_, i) => `Unrelated project detail ${i + 1}.`); + const spec = [ + '# Specification', + '', + `## ${req}`, + ...relevant, + '', + '## REQ-UNRELATED', + ...unrelated, + '', + ].join('\n'); + fs.writeFileSync(path.join(root, 'docs', 'spec.md'), spec, 'utf8'); + + const task = baseTask(`TASK-${label.toUpperCase().replace(/[^A-Z0-9]/g, '-')}`, `Benchmark ${label}`); + task.requirements = [req]; + task.acceptanceCriteria[0].source = req; + + const contract = createPolicyBoundDevelopmentContract({ + rootDir: root, + task, + authoritativeSources: [{ + path: 'docs/spec.md', + kind: 'specification', + authority: 'required', + sections: [req], + }], + }); + + const context = buildContextPackage({ + contract, + role: 'implementation-agent', + rootDir: root, + repositoryState: { + files: paths, + ...repositoryState, + }, + }); + + const gates = selectRequiredGates(contract); + return { + workspaceFiles: paths.length, + rawSourceTokens: context.tokenProfile.rawSourceTokens, + deliveredSourceTokens: context.tokenProfile.deliveredSourceTokens, + sourceSavingsPercent: context.tokenProfile.sourceSavingsPercent, + estimatedPackageTokens: context.tokenProfile.estimatedPackageTokens, + overBudget: context.tokenProfile.overBudget, + deliveryMode: context.authoritativeSources[0].delivery.mode, + selectedVerification: gates.verification, + selectedReviewers: gates.reviewers, + }; + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +} + +function legacyStateFixture(revisions = 26) { + const root = tempProject('legacy-state'); + try { + // Historical baseline fixture: construct the pre-v0.12 legacy snapshot chain + // directly so this measurement remains stable after State Engine V2 changes + // the live saveStateRevision() implementation. + const state = createInitialState({ autonomy: 'guided-autopilot' }, root); + const stateDir = path.join(root, '.development-kit', 'autopilot', 'state'); + fs.mkdirSync(stateDir, { recursive: true }); + + for (let revision = 1; revision <= revisions; revision += 1) { + state.stateRevision = revision; + state.updatedAt = new Date(Date.parse(state.createdAt) + revision * 1000).toISOString(); + const name = 'revision-' + String(revision).padStart(6, '0') + '.json'; + fs.writeFileSync(path.join(stateDir, name), JSON.stringify(state, null, 2), 'utf8'); + } + + fs.writeFileSync(path.join(stateDir, 'current.json'), JSON.stringify({ + currentRevision: revisions, + currentRevisionFile: 'revision-' + String(revisions).padStart(6, '0') + '.json', + workflowId: state.workflowId, + updatedAt: state.updatedAt, + }, null, 2), 'utf8'); + + const files = fs.readdirSync(stateDir).filter((name) => fs.statSync(path.join(stateDir, name)).isFile()); + const revisionFiles = files.filter((name) => /^revision-\d{6}\.json$/.test(name)); + const bytes = files.reduce((sum, name) => sum + fs.statSync(path.join(stateDir, name)).size, 0); + return { + requestedRevisions: revisions, + revisionFiles: revisionFiles.length, + totalStateFiles: files.length, + totalStateBytes: bytes, + pointerFiles: files.filter((name) => name === 'current.json').length, + filesPerRevisionApprox: Number((files.length / revisions).toFixed(3)), + }; + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +} + +function gitSha() { + if (process.env.GITHUB_SHA) return process.env.GITHUB_SHA; + try { + return execFileSync('git', ['rev-parse', 'HEAD'], { cwd: ROOT, encoding: 'utf8' }).trim(); + } catch { + return null; + } +} + +const repoFiles = walk(ROOT, { exclude: ['.git', 'node_modules', 'website/node_modules', 'website/.next'] }); +const tokenAudit = buildTokenAudit(); + +const singlePackage = contextFixture({ + label: 'single-package', + layout: { src: 24, test: 8 }, + relevantLines: 10, + unrelatedLines: 40, + repositoryState: { topology: 'single-package' }, +}); + +const multiPackage = contextFixture({ + label: 'multi-package', + layout: { + 'apps/web/src': 80, + 'apps/mobile/src': 70, + 'packages/shared/src': 40, + 'packages/calculations/src': 50, + 'supabase/migrations': 25, + }, + relevantLines: 12, + unrelatedLines: 180, + repositoryState: { + topology: 'multi-package', + packageRoots: ['apps/web', 'apps/mobile', 'packages/shared', 'packages/calculations', 'supabase'], + note: 'Current context package has no first-class workspace target selector.', + }, +}); + +const calculationHeavy = contextFixture({ + label: 'calculation-heavy', + layout: { 'src/engine': 90, 'test/golden': 60 }, + relevantLines: 30, + unrelatedLines: 300, + repositoryState: { + topology: 'calculation-heavy', + domain: 'engineering', + note: 'Current gate selector has no domain/numerical verification profile.', + }, +}); + +const baseline = { + schemaVersion: 1, + phase: 'PHASE-0-AND-1', + generatedAt: new Date().toISOString(), + repository: 'eybersjp/development-kit', + frameworkVersion: pkg.version, + commit: gitSha(), + platform: process.platform, + node: process.version, + releaseTargetFinding: { + requestedSpecificationVersion: '0.11.0', + currentRepositoryVersion: pkg.version, + versionCollision: pkg.version !== '0.11.0', + recommendedNextFeatureVersion: '0.12.0', + }, + repositoryInventory: { + files: repoFiles.length, + bytes: repoFiles.reduce((sum, item) => sum + item.bytes, 0), + testFiles: repoFiles.filter((item) => /\.test\.mjs$/.test(item.path)).length, + instructionFiles: repoFiles.filter((item) => + item.path === 'AGENTS.md' + || /^agents\/.*\.md$/.test(item.path) + || /^commands\/.*\.md$/.test(item.path) + || /^skills\/.*\/SKILL\.md$/.test(item.path) + ).length, + }, + tokenAudit: { + estimator: tokenAudit.estimator, + note: tokenAudit.note, + priorityRuntimeSkills: tokenAudit.priorityRuntimeSkills, + implementationHotPath: tokenAudit.implementationHotPath, + categories: tokenAudit.categories, + }, + representativeFixtures: { + singlePackage, + multiPackage, + calculationHeavy, + legacyState26Revisions: legacyStateFixture(26), + }, + instrumentationCoverage: { + providerTokenUsage: { available: false, value: null, reason: 'Current DKF runtime does not expose provider billing-token telemetry.' }, + repositoryReadCount: { available: false, value: null, reason: 'Current DKF runtime does not centrally instrument repository reads.' }, + repositoryScanCount: { available: false, value: null, reason: 'Current DKF runtime does not centrally instrument repository orientation scans.' }, + agentInvocationCount: { available: false, value: null, reason: 'Current DKF runtime does not centrally instrument host agent invocations.' }, + toolCallCount: { available: false, value: null, reason: 'Current DKF runtime does not centrally instrument host tool calls.' }, + }, +}; + +const json = JSON.stringify(baseline, null, 2) + '\n'; +const outArg = process.argv.find((arg) => arg.startsWith('--out=')); +if (outArg) { + const outputPath = path.resolve(ROOT, outArg.slice('--out='.length)); + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync(outputPath, json, 'utf8'); +} +process.stdout.write(json); diff --git a/scripts/phase02-scale-context-plan-validate.mjs b/scripts/phase02-scale-context-plan-validate.mjs new file mode 100644 index 00000000..9df2da98 --- /dev/null +++ b/scripts/phase02-scale-context-plan-validate.mjs @@ -0,0 +1,38 @@ +#!/usr/bin/env node + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { validatePlanModel } from '../runtime/orchestration/plan-validator.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const PLAN_PATH = path.join( + ROOT, + 'docs', + '04-architecture', + 'dkf-scale-context-iteration-v0.12-plan-model.json', +); + +const model = JSON.parse(fs.readFileSync(PLAN_PATH, 'utf8')); + +const report = validatePlanModel({ + tasks: model.tasks, + declaredTaskCount: model.declaredTaskCount, + declaredDependencyEdges: model.declaredDependencyEdges, + requiredResources: model.requiredResources, + requiredAcceptanceCriteria: model.requiredAcceptanceCriteria, + availableConfigurationRequirements: [], +}); + +const output = { + planId: model.planId, + planPath: path.relative(ROOT, PLAN_PATH).replaceAll('\\', '/'), + ...report, +}; + +process.stdout.write(`${JSON.stringify(output, null, 2)}\n`); + +if (!report.valid) { + process.exitCode = 1; +} diff --git a/scripts/phase03-cost-observatory-contract.mjs b/scripts/phase03-cost-observatory-contract.mjs new file mode 100644 index 00000000..95ccb948 --- /dev/null +++ b/scripts/phase03-cost-observatory-contract.mjs @@ -0,0 +1,167 @@ +#!/usr/bin/env node + +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + checkContractStaleness, + ensureDevelopmentContract, + validateDevelopmentContract, +} from '../runtime/orchestration/development-contract.mjs'; +import { selectRequiredGates } from '../runtime/orchestration/gate-selector.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +const task = { + id: 'DKF120-T01', + projectId: 'proj_cad505c1-901d-46f0-a0d1-297598bdae18', + status: 'approved', + objective: 'Add generic, truthful orchestration-cost measurement before optimising later v0.12 components.', + scope: { + in: [ + 'runtime/orchestration/cost-observatory.mjs', + 'runtime/orchestration/index.mjs', + 'schemas/cost-record.schema.json', + 'scripts/cost-observatory.test.mjs', + 'scripts/phase03-cost-observatory-contract.mjs', + 'package.json', + '.github/workflows/ci.yml', + 'docs/', + 'memory.md' + ], + out: [ + 'Workspace Target Engine', + 'Execution Capsules or context caching', + 'State Engine V2', + 'Lifecycle Instances', + 'Discovery/re-entry', + 'Gate Profiles', + 'Verification Extension SDK', + 'Complexity Delta Guard', + 'Provider-specific mandatory SDKs', + 'Application-specific or industry-specific business logic' + ], + }, + requirements: [ + 'Record DKF-controlled measurable orchestration counts.', + 'Accept optional host/provider token telemetry without requiring it.', + 'Keep unavailable provider/host metrics null rather than inferred.', + 'Provide canonical newline-normalised logical text/context comparison.', + 'Preserve the existing historical static token audit unchanged.', + 'Provide before/after cost-record comparison for later v0.12 replay evidence.', + ], + acceptanceCriteria: [ + { + id: 'DKF-120-AC-001', + statement: 'Every instrumented orchestration operation can emit a cost record containing only measured or deterministic-estimate fields.', + source: 'DKF-120-AC-001', + verificationType: ['test'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-002', + statement: 'Provider token fields remain null/unavailable unless supplied by the host/provider.', + source: 'DKF-120-AC-002', + verificationType: ['test'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-003', + statement: 'New cross-platform logical text/context comparisons normalise newline representation while preserving historical audit compatibility.', + source: 'DKF-120-AC-003', + verificationType: ['test'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-033', + statement: 'Release evidence can include baseline-versus-v0.12 cost comparison from validated cost records.', + source: 'DKF-120-AC-033', + verificationType: ['test'], + requiredEvidence: true, + }, + ], + architectureConstraints: [ + 'DKF core remains general-purpose and host/provider independent.', + 'Cost telemetry is observational and must not change deterministic acceptance semantics.', + 'Existing token/context hardening is extended rather than replaced.', + 'Cost records require numeric usage observations only and do not require provider credentials or secrets.', + 'Unavailable host/provider telemetry must never be fabricated.', + ], + designConstraints: [], + securityConstraints: [], + risk: { + level: 2, + reasons: ['Instrumentation crosses orchestration boundaries but must remain non-authoritative.'], + }, + requiredVerification: ['tests'], + requiredReviewers: ['code-reviewer'], + correctionPolicy: { maxAttempts: 3 }, + executionSafety: { + resourceScope: 'project-only', + destructiveOperations: 'explicit-approval', + remoteMutation: 'explicit-contract', + }, + configurationDependencies: [], +}; + +const authoritativeSources = [ + { + path: 'docs/04-architecture/dkf-scale-context-iteration-v0.12-specification.md', + kind: 'specification', + authority: 'required', + sections: ['4.1 Cost Observatory', 'DKF-120-AC-001', 'DKF-120-AC-002', 'DKF-120-AC-003', 'DKF-120-AC-033'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-v0.12-technical-design.md', + kind: 'technical-design', + authority: 'required', + sections: ['6. Cost Observatory'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-v0.12-implementation-plan.md', + kind: 'implementation-plan', + authority: 'required', + sections: ['3. Task DKF120-T01 — Cost Observatory'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-phase01-baseline.md', + kind: 'baseline-evidence', + authority: 'supporting', + sections: ['4. Existing static token baseline', '7. Observability gaps'], + }, +]; + +const result = ensureDevelopmentContract({ + rootDir: ROOT, + projectId: task.projectId, + task, + authoritativeSources, + contractId: 'INC-DKF120-T01', +}); + +validateDevelopmentContract(result.contract); +const gates = selectRequiredGates(result.contract); +if (JSON.stringify(gates.reviewers) !== JSON.stringify(['code-reviewer'])) { + throw new Error(`Unexpected T01 reviewer gates: ${gates.reviewers.join(', ')}`); +} +if (gates.controlDomains.length > 0 || gates.humanApprovals.length > 0) { + throw new Error('T01 unexpectedly derived specialist control or human-approval gates'); +} +const staleness = checkContractStaleness(result.contract, ROOT); +if (staleness.stale) { + throw new Error('DKF120-T01 Development Contract is stale immediately after creation'); +} + +process.stdout.write(`${JSON.stringify({ + contractId: result.contract.contractId, + taskId: result.contract.taskId, + schemaVersion: result.contract.schemaVersion, + created: result.created, + sourceFingerprint: result.contract.sourceFingerprint, + acceptanceCriteria: result.contract.acceptanceCriteria.map((criterion) => criterion.id), + requiredVerification: result.contract.requiredVerification, + requiredReviewers: result.contract.requiredReviewers, + derivedGates: gates, + riskLevel: result.contract.risk.level, + stale: staleness.stale, +}, null, 2)}\n`); diff --git a/scripts/phase04-workspace-target-engine-contract.mjs b/scripts/phase04-workspace-target-engine-contract.mjs new file mode 100644 index 00000000..5f49fee7 --- /dev/null +++ b/scripts/phase04-workspace-target-engine-contract.mjs @@ -0,0 +1,187 @@ +#!/usr/bin/env node + +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + checkContractStaleness, + ensureDevelopmentContract, + validateDevelopmentContract, +} from '../runtime/orchestration/development-contract.mjs'; +import { selectRequiredGates } from '../runtime/orchestration/gate-selector.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +const task = { + id: 'DKF120-T02', + projectId: 'proj_cad505c1-901d-46f0-a0d1-297598bdae18', + status: 'approved', + objective: 'Introduce generic project-defined workspace target boundaries and capability/configuration-driven routing without privileging any application, industry, repository topology, or target name.', + scope: { + in: [ + 'runtime/orchestration/workspace-targets.mjs', + 'runtime/orchestration/development-contract.mjs', + 'runtime/orchestration/index.mjs', + 'schemas/workspace-target-registry.schema.json', + 'schemas/development-contract.schema.json', + 'scripts/workspace-targets.test.mjs', + 'scripts/orchestration-contract.test.mjs', + 'scripts/phase04-workspace-target-engine-contract.mjs', + '.development-kit/workspace.json', + 'package.json', + '.github/workflows/ci.yml', + 'docs/', + 'memory.md' + ], + out: [ + 'Execution Capsules or context caching', + 'State Engine V2', + 'Lifecycle Instances', + 'Discovery/re-entry', + 'Gate Profiles', + 'Verification Extension SDK', + 'Complexity Delta Guard', + 'Control Center v0.12 integration', + 'Application-specific or industry-specific target semantics', + 'Technology-name-driven routing' + ], + }, + requirements: [ + 'Support arbitrary project-defined target IDs.', + 'Support both single-target and multi-target repositories.', + 'Discover common repository boundaries without privileging target names or business domains in DKF core.', + 'Support target dependency relationships and deterministic dependency/reverse-dependant closures.', + 'Allow Development Contracts to bind primary, affected, and verification targets.', + 'Bind target-aware Development Contracts to the workspace registry fingerprint so registry drift makes the contract stale.', + 'Resolve target commands and target selection from declared capabilities/configuration/adapters rather than target names, kinds, or technologies.', + ], + acceptanceCriteria: [ + { + id: 'DKF-120-AC-004', + statement: 'A project can register arbitrary target IDs without changing DKF core.', + source: 'DKF-120-AC-004', + verificationType: ['test'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-005', + statement: 'Target discovery/configuration does not assume a specific application domain or fixed repository topology.', + source: 'DKF-120-AC-005', + verificationType: ['test'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-006', + statement: 'Tasks/contracts can bind primary, affected, and verification targets.', + source: 'DKF-120-AC-006', + verificationType: ['test'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-007', + statement: 'Target-specific commands/capabilities resolve without hard-coded target names.', + source: 'DKF-120-AC-007', + verificationType: ['test'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-039', + statement: 'Core routing is capability/configuration-driven rather than technology-name driven.', + source: 'DKF-120-AC-039', + verificationType: ['test'], + requiredEvidence: true, + }, + ], + architectureConstraints: [ + 'DKF core remains general-purpose and host/provider independent.', + 'Target IDs, kinds, capabilities, commands, and adapter metadata are project configuration rather than built-in business policy.', + 'A single-root repository remains a valid first-class workspace.', + 'T02 extends Development Contracts and existing orchestration rather than creating a competing contract or execution system.', + 'DKF120-T01 is accepted and is the required predecessor for this increment.', + ], + designConstraints: [], + securityConstraints: [], + risk: { + level: 2, + reasons: ['Workspace routing changes execution boundaries and Development Contract authority but does not itself perform remote or destructive actions.'], + }, + requiredVerification: ['tests'], + requiredReviewers: ['code-reviewer'], + correctionPolicy: { maxAttempts: 3 }, + executionSafety: { + resourceScope: 'project-only', + destructiveOperations: 'explicit-approval', + remoteMutation: 'explicit-contract', + }, + configurationDependencies: [], +}; + +const authoritativeSources = [ + { + path: 'docs/04-architecture/dkf-scale-context-iteration-v0.12-specification.md', + kind: 'specification', + authority: 'required', + sections: ['4.2 Workspace Target Engine', 'Workspace targets'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-v0.12-technical-design.md', + kind: 'technical-design', + authority: 'required', + sections: ['4. Workspace Target Engine'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-v0.12-implementation-plan.md', + kind: 'implementation-plan', + authority: 'required', + sections: ['4. Task DKF120-T02 — Workspace Target Engine'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-phase03-t01-validation.md', + kind: 'dependency-acceptance', + authority: 'supporting', + sections: ['8. Acceptance decision', '9. Next increment'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-phase01-baseline.md', + kind: 'baseline-evidence', + authority: 'supporting', + sections: ['5.2 Multi-package fixture'], + }, +]; + +const result = ensureDevelopmentContract({ + rootDir: ROOT, + projectId: task.projectId, + task, + authoritativeSources, + contractId: 'INC-DKF120-T02', +}); + +validateDevelopmentContract(result.contract); +const gates = selectRequiredGates(result.contract); +if (JSON.stringify(gates.reviewers) !== JSON.stringify(['code-reviewer'])) { + throw new Error(`Unexpected T02 reviewer gates: ${gates.reviewers.join(', ')}`); +} +if (gates.controlDomains.length > 0 || gates.humanApprovals.length > 0) { + throw new Error('T02 unexpectedly derived specialist control or human-approval gates'); +} +const staleness = checkContractStaleness(result.contract, ROOT); +if (staleness.stale) { + throw new Error('DKF120-T02 Development Contract is stale immediately after creation'); +} + +process.stdout.write(`${JSON.stringify({ + contractId: result.contract.contractId, + taskId: result.contract.taskId, + schemaVersion: result.contract.schemaVersion, + created: result.created, + sourceFingerprint: result.contract.sourceFingerprint, + acceptanceCriteria: result.contract.acceptanceCriteria.map((criterion) => criterion.id), + requiredVerification: result.contract.requiredVerification, + requiredReviewers: result.contract.requiredReviewers, + derivedGates: gates, + riskLevel: result.contract.risk.level, + targetBinding: result.contract.workspaceTargets ?? null, + bootstrapNote: 'T02 contract predates Workspace Target Engine availability; future target-aware contracts use the T02 binding contract.', + stale: staleness.stale, +}, null, 2)}\n`); diff --git a/scripts/phase05-execution-capsule-contract.mjs b/scripts/phase05-execution-capsule-contract.mjs new file mode 100644 index 00000000..361e04fd --- /dev/null +++ b/scripts/phase05-execution-capsule-contract.mjs @@ -0,0 +1,213 @@ +#!/usr/bin/env node + +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + checkContractStaleness, + ensureDevelopmentContract, + validateDevelopmentContract, +} from '../runtime/orchestration/development-contract.mjs'; +import { selectRequiredGates } from '../runtime/orchestration/gate-selector.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +const task = { + id: 'DKF120-T03', + projectId: 'proj_cad505c1-901d-46f0-a0d1-297598bdae18', + status: 'approved', + objective: 'Reduce repeated repository/runtime reorientation with deterministic Execution Capsules and a safely invalidated repository context cache while preserving source authority and fresh-role isolation.', + scope: { + in: [ + 'runtime/orchestration/execution-capsule.mjs', + 'runtime/orchestration/context-cache.mjs', + 'runtime/orchestration/context-package.mjs', + 'runtime/orchestration/index.mjs', + 'schemas/execution-capsule.schema.json', + 'schemas/context-cache-entry.schema.json', + 'scripts/execution-capsule.test.mjs', + 'scripts/context-cache.test.mjs', + 'scripts/phase05-execution-capsule-contract.mjs', + 'package.json', + '.github/workflows/ci.yml', + 'docs/', + 'memory.md' + ], + out: [ + 'State Engine V2', + 'Lifecycle Instances', + 'Discovery/re-entry', + 'Gate Profiles', + 'Verification Extension SDK', + 'Complexity Delta Guard', + 'Control Center v0.12 integration', + 'Replacing buildContextPackage()', + 'Weakening authoritative whole-file fingerprint checks', + 'Treating summaries or cached repository facts as authority' + ], + }, + requirements: [ + 'Build deterministic Execution Capsules that reference authority and repository evidence by fingerprint instead of copying project history.', + 'Reuse current authoritative-source section materialisation through buildContextPackage().', + 'Persist only acceleration-grade repository structural facts and invalidate them deterministically.', + 'Include target/dependency delta, relevant file/test fingerprints, upstream accepted-task references, verified runtime facts with provenance, and required gate/profile references.', + 'Reject stale Execution Capsules before role context construction.', + 'A relevant file change must invalidate the capsule.', + 'An unrelated target file change must not invalidate an unrelated target-scoped cache when the bound contract and selected target inputs remain unchanged.', + 'A cache miss must still produce correct context and may only reduce performance, never reliability.', + 'Cost Observatory evidence must report measurable cache/context/orientation effects where observable.', + ], + acceptanceCriteria: [ + { + id: 'DKF-120-AC-008', + statement: 'Fresh implementation/review contexts can receive deterministic Execution Capsules.', + source: 'DKF-120-AC-008', + verificationType: ['test'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-009', + statement: 'Capsule fingerprints invalidate stale capsules when relevant authority/repository state changes.', + source: 'DKF-120-AC-009', + verificationType: ['test'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-010', + statement: 'Repository/context caching has deterministic invalidation.', + source: 'DKF-120-AC-010', + verificationType: ['test'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-011', + statement: 'Existing section-aware source materialisation and source-fingerprint protections remain green.', + source: 'DKF-120-AC-011', + verificationType: ['test'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-035', + statement: 'Comparative evidence reports measured context/orientation improvement where observable.', + source: 'DKF-120-AC-035', + verificationType: ['test'], + requiredEvidence: true, + }, + ], + architectureConstraints: [ + 'DKF core remains general-purpose, project/domain neutral, host independent, and provider independent.', + 'Execution Capsules are deterministic reference manifests, not a new source of business or specification authority.', + 'Context cache entries are disposable acceleration artifacts and may not become canonical state.', + 'Fresh-role isolation and independent verification remain mandatory.', + 'T03 must build on the accepted Workspace Target Engine and Cost Observatory rather than duplicating either capability.', + 'T03 depends on accepted DKF120-T02.', + ], + designConstraints: [], + securityConstraints: [], + risk: { + level: 3, + reasons: ['A stale or incorrectly validated cache hit could feed obsolete repository context into implementation or independent verification.'], + }, + requiredVerification: ['tests'], + requiredReviewers: ['code-reviewer'], + correctionPolicy: { maxAttempts: 3 }, + executionSafety: { + resourceScope: 'project-only', + destructiveOperations: 'explicit-approval', + remoteMutation: 'explicit-contract', + }, + configurationDependencies: [], + targetBinding: { + primary: 'dkf-framework', + affected: ['dkf-framework'], + verification: ['dkf-framework'], + }, +}; + +const authoritativeSources = [ + { + path: 'docs/04-architecture/dkf-scale-context-iteration-v0.12-specification.md', + kind: 'specification', + authority: 'required', + sections: ['4.3 Execution Capsules and context cache', 'Context economy'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-v0.12-technical-design.md', + kind: 'technical-design', + authority: 'required', + sections: ['5. Execution Capsules and Context Engine', '6. Cost Observatory'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-v0.12-implementation-plan.md', + kind: 'implementation-plan', + authority: 'required', + sections: ['5. Task DKF120-T03 — Execution Capsules & Context Cache'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-phase04-t02-validation.md', + kind: 'dependency-acceptance', + authority: 'required', + sections: ['10. Acceptance decision', '11. Next increment'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-phase03-t01-validation.md', + kind: 'telemetry-foundation', + authority: 'supporting', + sections: ['2. Implemented capability', '3. Acceptance-criterion evidence'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-phase01-baseline.md', + kind: 'baseline-evidence', + authority: 'supporting', + sections: ['2.5 Context rehydration is already safe but remains expensive by design', '5. Representative context fixtures'], + }, +]; + +const result = ensureDevelopmentContract({ + rootDir: ROOT, + projectId: task.projectId, + task, + authoritativeSources, + contractId: 'INC-DKF120-T03', +}); + +validateDevelopmentContract(result.contract); + +if (!result.contract.workspaceTargets) { + throw new Error('T03 must use a target-aware Development Contract'); +} +if (result.contract.workspaceTargets.primary !== 'dkf-framework') { + throw new Error('T03 primary target binding is not dkf-framework'); +} + +const gates = selectRequiredGates(result.contract); +const expectedReviewers = ['architecture-reviewer', 'code-reviewer', 'security-reviewer']; +if (JSON.stringify(gates.reviewers) !== JSON.stringify(expectedReviewers)) { + throw new Error(`Unexpected T03 reviewer gates: ${gates.reviewers.join(', ')}`); +} +if (JSON.stringify(gates.controlDomains) !== JSON.stringify(['security'])) { + throw new Error(`Unexpected T03 control domains: ${gates.controlDomains.join(', ')}`); +} +if (gates.humanApprovals.length > 0) { + throw new Error('T03 unexpectedly derived a human approval gate'); +} + +const staleness = checkContractStaleness(result.contract, ROOT); +if (staleness.stale) { + throw new Error('DKF120-T03 Development Contract is stale immediately after creation'); +} + +process.stdout.write(`${JSON.stringify({ + contractId: result.contract.contractId, + taskId: result.contract.taskId, + schemaVersion: result.contract.schemaVersion, + created: result.created, + sourceFingerprint: result.contract.sourceFingerprint, + acceptanceCriteria: result.contract.acceptanceCriteria.map((criterion) => criterion.id), + requiredVerification: result.contract.requiredVerification, + requiredReviewers: result.contract.requiredReviewers, + derivedGates: gates, + riskLevel: result.contract.risk.level, + targetBinding: result.contract.workspaceTargets, + stale: staleness.stale, +}, null, 2)}\n`); diff --git a/scripts/phase06-state-engine-v2-contract.mjs b/scripts/phase06-state-engine-v2-contract.mjs new file mode 100644 index 00000000..7ccfcaac --- /dev/null +++ b/scripts/phase06-state-engine-v2-contract.mjs @@ -0,0 +1,225 @@ +#!/usr/bin/env node + +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + checkContractStaleness, + ensureDevelopmentContract, + validateDevelopmentContract, +} from '../runtime/orchestration/development-contract.mjs'; +import { selectRequiredGates } from '../runtime/orchestration/gate-selector.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +const T04_REQUIREMENTS = [ + 'Persist canonical runtime state history as an inspectable append-only event ledger with monotonic sequence and hash-chain integrity.', + 'Materialise current runtime snapshot state from canonical events and permit deterministic snapshot rebuild.', + 'Maintain a disposable rebuildable index whose deletion cannot destroy canonical state.', + 'Perform canonical state commits atomically so interrupted temporary writes cannot partially advance canonical history.', + 'Migrate supported legacy Autopilot and orchestration run-state revisions with semantic equivalence.', + 'Make supported migration idempotent and fail closed on corrupt or semantically inconsistent legacy input.', + 'Preserve recoverable legacy state and a deterministic backup until verified cutover; do not delete legacy state in T04.', + 'Preserve current acceptance state/meaning during migration.', + 'Keep public simple-project workflows backward compatible and permit legacy fallback where State Engine V2 is not active.', + 'Measure representative transition-generated state micro-file reduction against the Phase 1 baseline and report the result even if the >=80% objective is missed.', +]; + +const task = { + id: 'DKF120-T04', + projectId: 'proj_cad505c1-901d-46f0-a0d1-297598bdae18', + status: 'approved', + objective: 'Replace transition-proportional canonical snapshot chains with compact event-backed state while preserving auditability, semantic equivalence, recovery, and backward compatibility.', + scope: { + in: [ + 'runtime/orchestration/state-engine-v2.mjs', + 'runtime/orchestration/state-engine-migration.mjs', + 'runtime/orchestration/orchestration-run.mjs', + 'runtime/orchestration/index.mjs', + 'runtime/autopilot/state-store.mjs', + 'schemas/state-engine-event.schema.json', + 'schemas/state-engine-snapshot.schema.json', + 'schemas/state-engine-index.schema.json', + 'scripts/state-engine-v2.test.mjs', + 'scripts/state-engine-v2-migration.test.mjs', + 'scripts/phase06-state-engine-v2-contract.mjs', + 'package.json', + '.github/workflows/ci.yml', + '.gitignore', + 'docs/', + 'memory.md', + ], + out: [ + 'Lifecycle Instances', + 'Discovery and controlled re-entry', + 'Gate Profiles', + 'Verification Extension SDK', + 'Complexity Delta Guard', + 'Control Center v0.12 integration', + 'Deleting legacy state before verified migration/cutover', + 'Changing acceptance semantics', + 'Making a binary index the source of truth', + 'Application-specific or industry-specific state semantics', + ], + }, + requirements: T04_REQUIREMENTS, + acceptanceCriteria: [ + { + id: 'DKF-120-AC-012', + requirementIds: [T04_REQUIREMENTS[0], T04_REQUIREMENTS[3]], + statement: 'Canonical runtime history is append-only and reconstructable.', + source: 'DKF-120-AC-012', + verificationType: ['test'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-013', + requirementIds: [T04_REQUIREMENTS[1]], + statement: 'Current snapshot can be rebuilt from canonical history.', + source: 'DKF-120-AC-013', + verificationType: ['test'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-014', + requirementIds: [T04_REQUIREMENTS[2]], + statement: 'Disposable index deletion/rebuild produces semantically equivalent query state.', + source: 'DKF-120-AC-014', + verificationType: ['test'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-015', + requirementIds: [T04_REQUIREMENTS[4], T04_REQUIREMENTS[5], T04_REQUIREMENTS[6], T04_REQUIREMENTS[7], T04_REQUIREMENTS[8]], + statement: 'Supported legacy migration is idempotent, fail-closed, and recoverable before cutover acceptance.', + source: 'DKF-120-AC-015', + verificationType: ['test', 'migration'], + requiredEvidence: true, + }, + { + id: 'DKF-120-AC-034', + requirementIds: [T04_REQUIREMENTS[9]], + statement: 'Representative migrated runtime state achieves the state micro-file reduction objective or reports the miss explicitly.', + source: 'DKF-120-AC-034', + verificationType: ['test'], + requiredEvidence: true, + }, + ], + architectureConstraints: [ + 'events.jsonl is canonical State Engine V2 runtime history.', + 'snapshot.json is a materialised view derived from canonical history.', + 'index.db is disposable acceleration state and may not become authority.', + 'State Engine V2 remains general-purpose and domain neutral.', + 'Existing contracts, evidence, artifacts, and acceptance records remain in their current authoritative stores unless explicitly migrated by a later approved increment.', + 'T04 depends on accepted DKF120-T03.', + ], + designConstraints: [], + securityConstraints: [ + 'State paths must remain project-local and fail closed on path escape or corrupt canonical history.', + 'Hash-chain validation must detect event modification, deletion, insertion, or reordering.', + 'Legacy migration must validate all inputs before canonical cutover state is committed.', + ], + risk: { + level: 3, + reasons: [ + 'Persistent-state migration and recovery are control-plane critical.', + 'Incorrect event replay or migration could silently change orchestration/acceptance state.', + ], + }, + requiredVerification: ['tests', 'migration'], + requiredReviewers: ['code-reviewer'], + correctionPolicy: { maxAttempts: 3 }, + executionSafety: { + resourceScope: 'project-only', + destructiveOperations: 'explicit-approval', + remoteMutation: 'explicit-contract', + }, + configurationDependencies: [], + targetBinding: { + primary: 'dkf-framework', + affected: ['dkf-framework'], + verification: ['dkf-framework'], + }, +}; + +const authoritativeSources = [ + { + path: 'docs/04-architecture/dkf-scale-context-iteration-v0.12-specification.md', + kind: 'specification', + authority: 'required', + sections: ['4.4 State Engine V2', '7. Migration requirements', '8. Performance objectives', 'State Engine V2'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-v0.12-technical-design.md', + kind: 'technical-design', + authority: 'required', + sections: ['7. State Engine V2'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-v0.12-implementation-plan.md', + kind: 'implementation-plan', + authority: 'required', + sections: ['6. Task DKF120-T04 — State Engine V2'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-phase05-t03-validation.md', + kind: 'dependency-acceptance', + authority: 'required', + sections: ['11. Acceptance decision', '12. Next increment'], + }, + { + path: 'docs/04-architecture/dkf-scale-context-iteration-phase01-baseline.md', + kind: 'baseline-evidence', + authority: 'supporting', + sections: ['2.4 Current state persistence confirms the micro-file scaling concern', '6. Legacy state-growth fixture'], + }, +]; + +const result = ensureDevelopmentContract({ + rootDir: ROOT, + projectId: task.projectId, + task, + authoritativeSources, + contractId: 'INC-DKF120-T04', +}); + +validateDevelopmentContract(result.contract); + +if (!result.contract.workspaceTargets) { + throw new Error('T04 must use a target-aware Development Contract'); +} +if (result.contract.workspaceTargets.primary !== 'dkf-framework') { + throw new Error('T04 primary target binding is not dkf-framework'); +} + +const gates = selectRequiredGates(result.contract); +const expectedReviewers = ['architecture-reviewer', 'code-reviewer', 'security-reviewer']; +if (JSON.stringify(gates.reviewers) !== JSON.stringify(expectedReviewers)) { + throw new Error(`Unexpected T04 reviewer gates: ${gates.reviewers.join(', ')}`); +} +if (JSON.stringify(gates.controlDomains) !== JSON.stringify(['security'])) { + throw new Error(`Unexpected T04 control domains: ${gates.controlDomains.join(', ')}`); +} +if (gates.humanApprovals.length > 0) { + throw new Error('T04 unexpectedly derived a human approval gate'); +} + +const staleness = checkContractStaleness(result.contract, ROOT); +if (staleness.stale) { + throw new Error('DKF120-T04 Development Contract is stale immediately after creation'); +} + +process.stdout.write(`${JSON.stringify({ + contractId: result.contract.contractId, + taskId: result.contract.taskId, + schemaVersion: result.contract.schemaVersion, + created: result.created, + sourceFingerprint: result.contract.sourceFingerprint, + acceptanceCriteria: result.contract.acceptanceCriteria.map((criterion) => criterion.id), + requiredVerification: result.contract.requiredVerification, + requiredReviewers: result.contract.requiredReviewers, + derivedGates: gates, + riskLevel: result.contract.risk.level, + targetBinding: result.contract.workspaceTargets, + stale: staleness.stale, +}, null, 2)}\n`); diff --git a/scripts/state-engine-v2-benchmark.mjs b/scripts/state-engine-v2-benchmark.mjs new file mode 100644 index 00000000..3161c827 --- /dev/null +++ b/scripts/state-engine-v2-benchmark.mjs @@ -0,0 +1,60 @@ +// Deterministic, disposable T04 endurance fixture. Records observed performance; +// it does not turn a host-specific elapsed time into a false release acceptance. +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { + appendEntityState, + getStateEnginePaths, + loadCanonicalEvents, + loadEntityState, + verifyStateEngineIntegrity, +} from '../runtime/orchestration/state-engine-v2.mjs'; + +const countArg = process.argv.includes('--events') + ? Number(process.argv[process.argv.indexOf('--events') + 1]) + : 300; +assert.ok(Number.isInteger(countArg) && countArg >= 1 && countArg <= 5000, '--events must be between 1 and 5000'); +const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dk-t04-endurance-')); +try { + const start = performance.now(); + let modeledOldRewriteBytes = 0; + for (let revision = 1; revision <= countArg; revision += 1) { + appendEntityState({ + rootDir, + entityType: 'endurance-fixture', + entityId: 'same-entity', + state: { stateRevision: revision, value: revision, mark: 'T04' }, + actorClass: 'endurance-test', + timestamp: new Date(Date.UTC(2026, 9, 9, 0, 0, 0) + revision * 1000).toISOString(), + }); + modeledOldRewriteBytes += fs.statSync(getStateEnginePaths(rootDir).events).size; + } + const writeDurationMs = performance.now() - start; + const readStart = performance.now(); + const events = loadCanonicalEvents(rootDir); + const state = loadEntityState('endurance-fixture', 'same-entity', rootDir); + const integrity = verifyStateEngineIntegrity(rootDir); + const readVerifyDurationMs = performance.now() - readStart; + assert.equal(events.length, countArg); + assert.equal(state.stateRevision, countArg); + assert.equal(integrity.valid, true); + const bytes = fs.statSync(getStateEnginePaths(rootDir).events).size; + console.log('T04_ENDURANCE_EVIDENCE ' + JSON.stringify({ + events: countArg, + platform: process.platform, + node: process.version, + writeDurationMs: Math.round(writeDurationMs), + readVerifyDurationMs: Math.round(readVerifyDurationMs), + canonicalLedgerBytes: bytes, + physicallyAppendedEventBytes: bytes, + historicalFullRewriteByteModel: modeledOldRewriteBytes, + rewriteBytesAvoidedPercentModeled: Number(((1 - bytes / modeledOldRewriteBytes) * 100).toFixed(2)), + pass: integrity.valid, + limitations: 'Historical full-rewrite traffic is computed from file sizes, not a measured legacy run. CPU parsing/replay is still full-history.', + })); +} finally { + fs.rmSync(rootDir, { recursive: true, force: true }); +} diff --git a/scripts/state-engine-v2-hardening.test.mjs b/scripts/state-engine-v2-hardening.test.mjs new file mode 100644 index 00000000..251cf55e --- /dev/null +++ b/scripts/state-engine-v2-hardening.test.mjs @@ -0,0 +1,1032 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { createHash } from 'node:crypto'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; + +import { + appendEntityState, + appendMetadataEvent, + getStateEnginePaths, + loadCanonicalEvents, + loadStateSnapshot, + verifyStateEngineIntegrity, + recoverAbandonedStateLock, + ensureStateEngineLayout, +} from '../runtime/orchestration/state-engine-v2.mjs'; + +function tempProject(t) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'dk-t04-hardening-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + return dir; +} + +test('T04-H01 physical event ledger retains its open file identity across commits', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const ledger = getStateEnginePaths(rootDir).events; + const descriptor = fs.openSync(ledger, 'r'); + try { + const before = fs.fstatSync(descriptor).size; + appendMetadataEvent({ rootDir, eventType: 'FOLLOWUP_EVENT', payload: { value: 2 } }); + assert.ok(fs.fstatSync(descriptor).size > before, 'append must grow the same canonical file, not rewrite its inode'); + assert.equal(loadCanonicalEvents(rootDir).length, 2); + } finally { + fs.closeSync(descriptor); + } +}); + +test('T04-H02 a live owner cannot lose an aged lock', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const lockFile = getStateEnginePaths(rootDir).lock; + fs.writeFileSync(lockFile, JSON.stringify({ + owner: 'live-long-running-worker', + pid: process.pid, + hostname: os.hostname(), + acquiredAt: '2026-10-08T00:00:00.000Z', + }), 'utf8'); + const old = new Date(Date.now() - 30_000); + fs.utimesSync(lockFile, old, old); + assert.throws( + () => appendMetadataEvent({ rootDir, eventType: 'MUST_NOT_COMMIT', payload: { value: 2 } }), + /lock acquisition timed out/i, + ); + assert.equal(loadCanonicalEvents(rootDir).length, 1); + assert.equal(fs.existsSync(lockFile), true); +}); + +function runWorker(rootDir, workerId) { + return new Promise((resolve, reject) => { + const moduleUrl = new URL('../runtime/orchestration/state-engine-v2.mjs', import.meta.url).href; + const worker = [ + 'import { appendEntityState } from ' + JSON.stringify(moduleUrl) + ';', + 'const dir = process.argv[1], id = process.argv[2];', + 'for (let n = 1; n <= 12; n++) appendEntityState({ rootDir: dir, entityType: "worker-state", entityId: id, state: { stateRevision: n, value: n }, actorClass: "concurrency-test" });', + ].join('\n'); + const child = spawn(process.execPath, ['--input-type=module', '-e', worker, rootDir, workerId], { stdio: ['ignore', 'pipe', 'pipe'] }); + let stderr = ''; + child.stderr.on('data', data => { stderr += data; }); + child.on('error', reject); + child.on('close', code => code === 0 ? resolve() : reject(new Error('Worker ' + workerId + ' exited ' + code + ': ' + stderr.slice(-1600)))); + }); +} + +test('T04-H03 four processes append without losing events or corrupting sequence', async (t) => { + const rootDir = tempProject(t); + await Promise.all(Array.from({ length: 4 }, (_, i) => runWorker(rootDir, 'worker-' + i))); + const events = loadCanonicalEvents(rootDir); + assert.equal(events.length, 48); + assert.equal(events[0].sequence, 1); + assert.equal(events.at(-1).sequence, 48); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); +}); + + + +test('T04-H04 torn append with a valid pending journal deterministically repairs without losing committed history', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const before = fs.readFileSync(paths.events, 'utf8'); + appendMetadataEvent({ rootDir, eventType: 'FOLLOWUP_EVENT', payload: { value: 2 } }); + const expected = fs.readFileSync(paths.events, 'utf8'); + const suffix = expected.slice(before.length); + fs.writeFileSync(paths.events, before + suffix.slice(0, 23), 'utf8'); + fs.writeFileSync(paths.pending, JSON.stringify({ + schemaVersion: '1.0.0', + previousByteLength: Buffer.byteLength(before), + previousHash: 'sha256:' + createHash('sha256').update(before).digest('hex'), + suffix, + }), 'utf8'); + + const recovered = loadCanonicalEvents(rootDir); + assert.equal(recovered.length, 2); + assert.equal(fs.readFileSync(paths.events, 'utf8'), expected); + assert.equal(fs.existsSync(paths.pending), false); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); +}); + +test('T04-H07 conflicting pending suffix must fail closed, not truncate canonical evidence', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const before = fs.readFileSync(paths.events, 'utf8'); + fs.writeFileSync(paths.events, before + 'tampered', 'utf8'); + fs.writeFileSync(paths.pending, JSON.stringify({ + schemaVersion: '1.0.0', + previousByteLength: Buffer.byteLength(before), + previousHash: 'sha256:' + createHash('sha256').update(before).digest('hex'), + suffix: '{"valid":true}\n', + }), 'utf8'); + + assert.throws(() => loadCanonicalEvents(rootDir), /suffix conflicts/i); + assert.ok(fs.readFileSync(paths.events, 'utf8').endsWith('tampered')); + assert.equal(fs.existsSync(paths.pending), true); +}); + +test('T04-H08 stale lock from a provably dead local process can be recovered', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + fs.writeFileSync(paths.lock, JSON.stringify({ + owner: 'dead-worker', + pid: 2147483647, + hostname: os.hostname(), + acquiredAt: '2026-10-08T00:00:00.000Z', + }), 'utf8'); + const old = new Date(Date.now() - 30_000); + fs.utimesSync(paths.lock, old, old); + appendMetadataEvent({ rootDir, eventType: 'RECOVERED_LOCK', payload: { value: 2 } }); + assert.equal(loadCanonicalEvents(rootDir).length, 2); +}); + + +test('T04-H09 internal canonical events file may not be redirected through a symlink', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const outsideRoot = tempProject(t); + const outsideFile = path.join(outsideRoot, 'outside-ledger.jsonl'); + const originalContent = fs.readFileSync(paths.events, 'utf8'); + fs.writeFileSync(outsideFile, originalContent, 'utf8'); + fs.rmSync(paths.events); + try { + fs.symlinkSync(outsideFile, paths.events, 'file'); + } catch (error) { + if (['EPERM', 'EACCES', 'ENOTSUP'].includes(error.code)) { + t.skip('Host does not permit file symlink creation'); + return; + } + throw error; + } + assert.throws(() => loadCanonicalEvents(rootDir), /not a regular file/i); + assert.throws(() => appendMetadataEvent({ rootDir, eventType: 'DANGER', payload: {} }), /not a regular file/i); + assert.equal(fs.readFileSync(outsideFile, 'utf8'), originalContent); +}); + + +test('T04-H10 appending to a valid ledger without a terminal newline preserves record separation', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const original = fs.readFileSync(paths.events, 'utf8'); + assert.ok(original.endsWith('\n')); + fs.writeFileSync(paths.events, original.slice(0, -1), 'utf8'); + appendMetadataEvent({ rootDir, eventType: 'FOLLOWUP_EVENT', payload: { value: 2 } }); + const history = loadCanonicalEvents(rootDir); + assert.equal(history.length, 2); + assert.equal(history[1].previousEventHash, history[0].eventHash); + assert.ok(fs.readFileSync(paths.events, 'utf8').endsWith('\n')); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); +}); + +test('T04-H11 missing aged lock during stale-owner recheck retries acquisition safely', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + fs.writeFileSync(paths.lock, JSON.stringify({ + owner: 'provably-dead-owner', + pid: 2147483647, + hostname: os.hostname(), + acquiredAt: '2026-10-08T00:00:00.000Z', + })); + const aged = new Date(Date.now() - 30_000); + fs.utimesSync(paths.lock, aged, aged); + const originalStat = fs.statSync; + let lockStatCalls = 0; + fs.statSync = function injectConcurrentLockRemoval(target, ...args) { + if (target === paths.lock && ++lockStatCalls === 2) { + const err = new Error('simulated competing stale-lock reclamation'); + err.code = 'ENOENT'; + throw err; + } + return originalStat.call(this, target, ...args); + }; + try { + appendMetadataEvent({ rootDir, eventType: 'AFTER_STALE_LOCK_RACE', payload: { ok: true } }); + } finally { + fs.statSync = originalStat; + } + assert.ok(lockStatCalls >= 3); + assert.equal(loadCanonicalEvents(rootDir).length, 2); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); +}); + + +test('T04-H12 snapshot reader tolerates a legitimate writer committing between ledger and snapshot reads', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const originalRead = fs.readFileSync; + let insertedWriter = false; + fs.readFileSync = function simulateReadWriteInterleave(file, ...args) { + if (!insertedWriter && file === paths.snapshot) { + insertedWriter = true; + appendMetadataEvent({ rootDir, eventType: 'SIMULTANEOUS_EVENT', payload: { value: 2 } }); + } + return originalRead.call(this, file, ...args); + }; + let snapshot; + try { + snapshot = loadStateSnapshot(rootDir); + } finally { + fs.readFileSync = originalRead; + } + assert.equal(insertedWriter, true); + assert.equal(snapshot.lastEventSequence, 2); + assert.equal(loadCanonicalEvents(rootDir).length, 2); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); +}); + + +test('T04-H14 transient Windows EPERM on lock creation retries without bypassing ownership', (t) => { + if (process.platform !== 'win32') { + t.skip('Windows-specific filesystem contention semantics'); + return; + } + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const originalOpen = fs.openSync; + let attempts = 0; + fs.openSync = function injectTemporaryWindowsLockContention(file, flags, ...args) { + if (file === paths.lock && flags === 'wx') { + attempts += 1; + if (attempts === 1) { + const err = new Error('simulated Windows transient lock-file sharing violation'); + err.code = 'EPERM'; + throw err; + } + } + return originalOpen.call(this, file, flags, ...args); + }; + try { + appendMetadataEvent({ rootDir, eventType: 'LOCK_RETRY_SUCCEEDED', payload: { value: 2 } }); + } finally { + fs.openSync = originalOpen; + } + assert.ok(attempts >= 2); + assert.equal(loadCanonicalEvents(rootDir).length, 2); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); +}); + + +test('T04-H15 simultaneous stale-lock reclaimers cannot delete a new owner', async (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + fs.writeFileSync(paths.lock, JSON.stringify({ + owner: 'dead-before-contention', pid: 2147483647, hostname: os.hostname(), + acquiredAt: '2026-10-08T00:00:00.000Z', + })); + const aged = new Date(Date.now() - 30_000); + fs.utimesSync(paths.lock, aged, aged); + await Promise.all(Array.from({ length: 4 }, (_, i) => runWorker(rootDir, 'stale-worker-' + i))); + const records = loadCanonicalEvents(rootDir); + assert.equal(records.length, 49); + assert.equal(records.at(-1).sequence, 49); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); +}); + +test('T04-H16 canonical ledger hardlink cannot append to an external file', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const outsideRoot = tempProject(t); + const victim = path.join(outsideRoot, 'external-history.jsonl'); + const originalContent = fs.readFileSync(paths.events); + fs.writeFileSync(victim, originalContent); + fs.rmSync(paths.events); + try { + fs.linkSync(victim, paths.events); + } catch (e) { + if (['EPERM','EACCES','ENOTSUP','EXDEV'].includes(e.code)) { + t.skip('File hardlinks unavailable'); return; + } + throw e; + } + assert.throws(() => appendMetadataEvent({ rootDir, eventType: 'SHOULD_FAIL', payload: {} }), /hard.?link|multiple links/i); + assert.deepEqual(fs.readFileSync(victim), originalContent); +}); + +test('T04-H17 interrupted commit refreshes derived snapshot and index before clearing journal', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const before = fs.readFileSync(paths.events); + const oldSnapshot = fs.readFileSync(paths.snapshot); + const oldIndex = fs.readFileSync(paths.index); + appendMetadataEvent({ rootDir, eventType: 'FOLLOWUP_EVENT', payload: { value: 2 } }); + const after = fs.readFileSync(paths.events); + fs.writeFileSync(paths.snapshot, oldSnapshot); + fs.writeFileSync(paths.index, oldIndex); + fs.writeFileSync(paths.pending, JSON.stringify({ + schemaVersion: '1.0.0', + previousByteLength: before.length, + previousHash: 'sha256:' + createHash('sha256').update(before).digest('hex'), + suffix: after.subarray(before.length).toString('utf8'), + })); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); + assert.equal(JSON.parse(fs.readFileSync(paths.snapshot, 'utf8')).lastEventSequence, 2); + assert.equal(JSON.parse(fs.readFileSync(paths.index, 'utf8')).sourceSequence, 2); + assert.equal(fs.existsSync(paths.pending), false); +}); + +test('T04-H18 stale reader cannot overwrite a later writer snapshot with old derived state', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const oldSnapshot = fs.readFileSync(paths.snapshot); + appendMetadataEvent({ rootDir, eventType: 'SECOND_EVENT', payload: { value: 2 } }); + fs.writeFileSync(paths.snapshot, oldSnapshot); + const originalRead = fs.readFileSync; + let insertedWriter = false; + fs.readFileSync = function staleReaderInterleave(file, ...args) { + if (!insertedWriter && file === paths.snapshot) { + insertedWriter = true; + const staleContent = originalRead.call(this, file, ...args); + appendMetadataEvent({ rootDir, eventType: 'THIRD_EVENT', payload: { value: 3 } }); + return staleContent; + } + return originalRead.call(this, file, ...args); + }; + try { + loadStateSnapshot(rootDir); + } finally { + fs.readFileSync = originalRead; + } + assert.equal(insertedWriter, true); + assert.equal(JSON.parse(fs.readFileSync(paths.snapshot, 'utf8')).lastEventSequence, 3); + assert.equal(JSON.parse(fs.readFileSync(paths.index, 'utf8')).sourceSequence, 3); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); +}); + + +test('T04-H22 event append refuses a symlink swapped into place immediately before open', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const outside = tempProject(t); + const victim = path.join(outside, 'must-not-change.txt'); + fs.writeFileSync(victim, 'EXTERNAL-SENTINEL', 'utf8'); + const originalOpen = fs.openSync; + let injected = false; + fs.openSync = function swapCanonicalPathAtAppendOpen(file, flags, ...args) { + const isAppend = flags === 'a' || + (typeof flags === 'number' && (flags & fs.constants.O_APPEND) !== 0); + const isTargetLedger = file === paths.events || + (typeof file === 'string' && file.includes('/proc/self/fd/') && file.endsWith('/events.jsonl')); + if (!injected && isTargetLedger && isAppend) { + const displaced = path.join(paths.stateRoot, 'original-ledger-for-test.jsonl'); + fs.renameSync(paths.events, displaced); + try { + fs.symlinkSync(victim, paths.events, 'file'); + } catch (err) { + fs.renameSync(displaced, paths.events); + if (['EPERM','EACCES','ENOTSUP'].includes(err.code)) { + t.skip('Host denies file symlink fixture'); + skipped = true; + return originalOpen.call(this, file, flags, ...args); + } + throw err; + } + injected = true; + } + return originalOpen.call(this, file, flags, ...args); + }; + let skipped = false; + try { + try { + appendMetadataEvent({ rootDir, eventType: 'MUST_NOT_ESCAPE', payload: {} }); + if (!skipped) { + assert.fail('Missing expected exception'); + } + } catch (err) { + if (skipped) return; + assert.match(String(err), /symlink|symbolic|inode|link|ELOOP|outside|State Engine/i); + } + } finally { + fs.openSync = originalOpen; + } + if (!injected || skipped) return; + assert.equal(fs.readFileSync(victim, 'utf8'), 'EXTERNAL-SENTINEL'); +}); + + +test('T04-H26 abandoned recovery guard is recovered only by explicit offline confirmation', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const guard = paths.lock + '.reclaim'; + fs.mkdirSync(guard, { mode: 0o700 }); + fs.writeFileSync(paths.lock, JSON.stringify({ + owner: 'dead-lock-owner', pid: 2147483647, + hostname: os.hostname(), acquiredAt: '2026-10-08T00:00:00.000Z', + })); + const ago = new Date(Date.now() - 120_000); + fs.utimesSync(paths.lock, ago, ago); + fs.utimesSync(guard, ago, ago); + assert.throws(() => recoverAbandonedStateLock({ rootDir }), /offline confirmation/i); + assert.equal(fs.existsSync(guard), true); + const recovered = recoverAbandonedStateLock({ rootDir, confirmOffline: true }); + assert.equal(recovered.recovered, true); + assert.equal(fs.existsSync(guard), false); + appendMetadataEvent({ rootDir, eventType: 'AFTER_ORPHAN_RECOVERY', payload: { value: 2 } }); + assert.equal(loadCanonicalEvents(rootDir).length, 2); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); +}); + +test('T04-H27 offline orphan guard recovery refuses a live owner', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const guard = paths.lock + '.reclaim'; + fs.mkdirSync(guard, { mode: 0o700 }); + fs.writeFileSync(path.join(guard, 'owner.json'), JSON.stringify({ + owner: 'still-running', pid: process.pid, hostname: os.hostname(), + })); + const ago = new Date(Date.now() - 120_000); + fs.utimesSync(guard, ago, ago); + assert.throws(() => recoverAbandonedStateLock({ rootDir, confirmOffline: true }), /live State Engine recovery guard/i); + assert.equal(fs.existsSync(guard), true); +}); + + +test('T04-H29 interrupted ledger recovery must never truncate an external hardlink swapped during read', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const prefix = fs.readFileSync(paths.events); + appendMetadataEvent({ rootDir, eventType: 'FOLLOWUP_EVENT', payload: { value: 2 } }); + const complete = fs.readFileSync(paths.events); + const suffix = complete.subarray(prefix.length); + fs.writeFileSync(paths.events, Buffer.concat([prefix, suffix.subarray(0, 40)])); + fs.writeFileSync(paths.pending, JSON.stringify({ + schemaVersion: '1.0.0', + previousByteLength: prefix.length, + previousHash: 'sha256:' + createHash('sha256').update(prefix).digest('hex'), + suffix: suffix.toString('utf8'), + })); + const outside = tempProject(t); + const victim = path.join(outside, 'external-ledger.jsonl'); + fs.writeFileSync(victim, Buffer.concat([prefix, Buffer.alloc(500, 0x58)])); + const expectedExternal = fs.readFileSync(victim); + const originalRead = fs.readFileSync; + let swapped = false; + fs.readFileSync = function swapAfterLedgerRead(file, ...args) { + const bytes = originalRead.call(this, file, ...args); + if (!swapped && file === paths.events) { + swapped = true; + fs.renameSync(paths.events, path.join(paths.stateRoot, 'displaced-ledger')); + fs.linkSync(victim, paths.events); + } + return bytes; + }; + try { + // The corrected implementation may reject any inode substitution or + // complete on its original descriptor. Neither may corrupt the victim. + try { loadCanonicalEvents(rootDir); } catch (error) { + assert.match(String(error), /canonical|inode|link|history|State Engine/i); + } + } finally { + fs.readFileSync = originalRead; + } + assert.deepEqual(fs.readFileSync(victim), expectedExternal); +}); + + +test('T04-H36 pending journal cannot overwrite external file after state-directory swap', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'FIRST', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const outside = tempProject(t); + const protectedFile = path.join(outside, 'pending-commit.json'); + fs.writeFileSync(protectedFile, 'EXTERNAL-DATA-MUST-STAY', 'utf8'); + const moved = paths.stateRoot + '-held-for-test'; + const originalOpen = fs.openSync; + let swapped = false; + fs.openSync = function swapRootAtJournalTempOpen(target, flags, ...rest) { + if (!swapped && typeof target === 'string' && + target.includes('pending-commit.json.tmp-') && flags === 'wx') { + try { + fs.renameSync(paths.stateRoot, moved); + } catch (err) { + if (process.platform === 'win32' && err.code === 'EPERM') { + t.skip('Windows locks directory handles against renaming while pinned open'); + return originalOpen.call(this, target, flags, ...rest); + } + throw err; + } + try { + fs.symlinkSync(outside, paths.stateRoot, process.platform === 'win32' ? 'junction' : 'dir'); + } catch (error) { + fs.renameSync(moved, paths.stateRoot); + if (['EPERM', 'EACCES', 'ENOTSUP'].includes(error.code)) { + t.skip('Host does not permit test directory symlinks/junctions'); + return originalOpen.call(this, target, flags, ...rest); + } + throw error; + } + swapped = true; + } + return originalOpen.call(this, target, flags, ...rest); + }; + try { + assert.throws( + () => appendMetadataEvent({ rootDir, eventType: 'SECOND_MUST_NOT_PUBLISH', payload: {} }), + /directory|symbolic|changed|outside|project root|symlink|mismatch/i, + ); + } finally { + fs.openSync = originalOpen; + if (swapped) { + fs.unlinkSync(paths.stateRoot); + fs.renameSync(moved, paths.stateRoot); + } + } + assert.equal(fs.readFileSync(protectedFile, 'utf8'), 'EXTERNAL-DATA-MUST-STAY'); + assert.equal(loadCanonicalEvents(rootDir).length, 1); +}); + +test('T04-H37 pending journal unlink fails closed when state directory is replaced during commit', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'FIRST', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const outside = tempProject(t); + const protectedFile = path.join(outside, 'pending-commit.json'); + fs.writeFileSync(protectedFile, 'EXTERNAL-SENTINEL-DO-NOT-DELETE', 'utf8'); + const moved = paths.stateRoot + '-held-for-unlink-test'; + const originalOpen = fs.openSync; + let swapped = false; + + // Intercept before unlink or atomic write + let journalCreated = false; + fs.openSync = function swapRootAtJournalUnlinkOpen(target, flags, ...rest) { + if (typeof target === 'string' && target.includes('pending-commit.json.tmp-')) { + journalCreated = true; + } + if (journalCreated && !swapped && typeof target === 'string' && (target.includes('snapshot.json') || target === paths.snapshot) && typeof flags === 'string' && flags.includes('w')) { + try { + fs.renameSync(paths.stateRoot, moved); + try { + fs.symlinkSync(outside, paths.stateRoot, process.platform === 'win32' ? 'junction' : 'dir'); + } catch (error) { + fs.renameSync(moved, paths.stateRoot); + if (['EPERM', 'EACCES', 'ENOTSUP'].includes(error.code)) { + t.skip('Host does not permit test directory symlinks/junctions'); + return originalOpen.call(this, target, flags, ...rest); + } + throw error; + } + swapped = true; + } catch (err) { + if (process.platform === 'win32' && err.code === 'EPERM') { + // Open directory descriptors prevent renaming the directory on Windows + t.skip('Windows locks directory handles against renaming while pinned open'); + return originalOpen.call(this, target, flags, ...rest); + } + throw err; + } + } + return originalOpen.call(this, target, flags, ...rest); + }; + + try { + assert.throws( + () => appendMetadataEvent({ rootDir, eventType: 'SECOND_MUST_FAIL_CLEANUP', payload: {} }), + /directory|symbolic|changed|replaced|journal cleanup|mismatch/i, + ); + } finally { + fs.openSync = originalOpen; + if (swapped) { + fs.unlinkSync(paths.stateRoot); + fs.renameSync(moved, paths.stateRoot); + } + } + + assert.equal(fs.readFileSync(protectedFile, 'utf8'), 'EXTERNAL-SENTINEL-DO-NOT-DELETE', + 'External victim pending-commit.json must not be unlinked'); +}); + +test('T04-H38 pending journal unlink fails closed when state directory is swapped with an ordinary directory across transaction', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'FIRST', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const moved = paths.stateRoot + '-held-for-ordinary-dir-swap'; + const originalOpen = fs.openSync; + let swapped = false; + let journalCreated = false; + + fs.openSync = function swapRootWithOrdinaryDirectory(target, flags, ...rest) { + if (typeof target === 'string' && target.includes('pending-commit.json.tmp-')) { + journalCreated = true; + } + // Trigger swap after journal file is opened/written but before unlink + const isTargetLedger = target === paths.events || + (typeof target === 'string' && target.includes('/proc/self/fd/') && target.endsWith('/events.jsonl')); + if (journalCreated && !swapped && isTargetLedger && typeof flags === 'number') { + try { + fs.renameSync(paths.stateRoot, moved); + // Create a brand new ordinary directory at paths.stateRoot with a decoy pending-commit.json + fs.mkdirSync(paths.stateRoot, { recursive: true }); + fs.writeFileSync(paths.pending, 'DECOY-JOURNAL', 'utf8'); + swapped = true; + } catch (err) { + if (process.platform === 'win32' && err.code === 'EPERM') { + t.skip('Windows locks directory handles against renaming while pinned open'); + return originalOpen.call(this, target, flags, ...rest); + } + throw err; + } + } + return originalOpen.call(this, target, flags, ...rest); + }; + + try { + assert.throws( + () => appendMetadataEvent({ rootDir, eventType: 'SECOND_MUST_FAIL_CLEANUP', payload: {} }), + /directory|changed|replaced/i, + ); + } finally { + fs.openSync = originalOpen; + if (swapped) { + fs.rmSync(paths.stateRoot, { recursive: true, force: true }); + fs.renameSync(moved, paths.stateRoot); + } + } +}); + +test('T04-H40 durable transaction writes fail closed when state directory is replaced before append or derived writes', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'FIRST', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const moved = paths.stateRoot + '-held-for-swap'; + const originalOpen = fs.openSync; + let swapped = false; + let journalCreated = false; + + fs.openSync = function swapRootBeforeAppend(target, flags, ...rest) { + if (typeof target === 'string' && target.includes('pending-commit.json.tmp-')) { + journalCreated = true; + } + // Swap state directory right before appendDurably opens paths.events + if (journalCreated && !swapped && (target === paths.events || (typeof target === 'string' && target.endsWith('/events.jsonl')))) { + try { + fs.renameSync(paths.stateRoot, moved); + fs.mkdirSync(paths.stateRoot, { recursive: true }); + fs.writeFileSync(paths.events, fs.readFileSync(path.join(moved, 'events.jsonl'))); + swapped = true; + } catch (err) { + if (process.platform === 'win32' && err.code === 'EPERM') { + t.skip('Windows locks directory handles against renaming while pinned open'); + return originalOpen.call(this, target, flags, ...rest); + } + throw err; + } + } + return originalOpen.call(this, target, flags, ...rest); + }; + + try { + assert.throws( + () => appendMetadataEvent({ rootDir, eventType: 'SECOND_MUST_FAIL_APPEND', payload: {} }), + /directory|changed|replaced/i, + ); + } finally { + fs.openSync = originalOpen; + if (swapped) { + fs.rmSync(paths.stateRoot, { recursive: true, force: true }); + fs.renameSync(moved, paths.stateRoot); + } + } +}); + +test('T04-H41 recovery resolves pending journal, events, and snapshot relative to pinned directory descriptor', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const before = fs.readFileSync(paths.events, 'utf8'); + appendMetadataEvent({ rootDir, eventType: 'FOLLOWUP_EVENT', payload: { value: 2 } }); + const expected = fs.readFileSync(paths.events, 'utf8'); + const suffix = expected.slice(before.length); + // Simulate torn append with pending journal + fs.writeFileSync(paths.events, before + suffix.slice(0, 10), 'utf8'); + fs.writeFileSync(paths.pending, JSON.stringify({ + schemaVersion: '1.0.0', + previousByteLength: Buffer.byteLength(before), + previousHash: 'sha256:' + createHash('sha256').update(before).digest('hex'), + suffix, + }), 'utf8'); + + const openedTargets = []; + const originalOpen = fs.openSync; + fs.openSync = function trackOpen(target, flags, ...rest) { + if (typeof target === 'string') { + openedTargets.push(target); + } + return originalOpen.call(this, target, flags, ...rest); + }; + + try { + const recovered = loadStateSnapshot(rootDir, { rebuildIfNeeded: true }); + assert.ok(recovered); + assert.equal(recovered.lastEventSequence, 2); + if (process.platform === 'linux') { + const procFdEvents = openedTargets.find(p => p.includes('/proc/self/fd/') && p.endsWith('events.jsonl')); + assert.ok(procFdEvents, 'Linux recovery must open events ledger through /proc/self/fd/ descriptor path'); + } + } finally { + fs.openSync = originalOpen; + } +}); + +test('T04-H42 appendDurably fails closed if canonical ledger entry is displaced during write', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'INITIAL_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const displaced = path.join(paths.stateRoot, 'displaced-events.jsonl'); + + const originalWriteSync = fs.writeSync; + let swapped = false; + fs.writeSync = function hookedWrite(fd, buffer, offset, length, position) { + if (!swapped) { + try { + fs.renameSync(paths.events, displaced); + fs.writeFileSync(paths.events, 'MUTATED_NEW_FILE\n', 'utf8'); + swapped = true; + } catch (err) { + if (process.platform === 'win32' && err.code === 'EPERM') { + t.skip('Windows locks open files against renaming'); + return originalWriteSync.call(this, fd, buffer, offset, length, position); + } + throw err; + } + } + return originalWriteSync.call(this, fd, buffer, offset, length, position); + }; + + try { + assert.throws( + () => appendMetadataEvent({ rootDir, eventType: 'MUST_FAIL_EVENT', payload: { value: 2 } }), + /displaced|changed|mismatch/i, + ); + } finally { + fs.writeSync = originalWriteSync; + if (swapped) { + fs.rmSync(paths.events, { force: true }); + fs.renameSync(displaced, paths.events); + } + } +}); + + + + +function swapLedgerBeforeJournalUnlink(t, paths, replacementContent) { + const originalUnlink = fs.unlinkSync; + const displaced = path.join(paths.stateRoot, 'displaced-events.jsonl'); + const state = { swapped: false, skipped: false, displaced, restore() { + fs.unlinkSync = originalUnlink; + } }; + fs.unlinkSync = function swapBetweenFinalLstatAndUnlink(target, ...rest) { + if (!state.swapped && typeof target === 'string' && target.endsWith('pending-commit.json')) { + try { + fs.renameSync(paths.events, displaced); + fs.writeFileSync(paths.events, replacementContent, 'utf8'); + state.swapped = true; + } catch (err) { + if (process.platform === 'win32' && ['EPERM', 'EBUSY', 'EACCES'].includes(err.code)) { + state.skipped = true; + t.skip('Windows denies renaming the open ledger; Ubuntu CI provides the inode-swap evidence'); + } else { + throw err; + } + } + } + return originalUnlink.call(this, target, ...rest); + }; + return state; +} + +test('T04-H43 ledger swap between final lstat and journal unlink is detected fail-closed on next load (append)', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const preLedger = fs.readFileSync(paths.events, 'utf8'); + const state = swapLedgerBeforeJournalUnlink(t, paths, preLedger); + try { + appendMetadataEvent({ rootDir, eventType: 'AMBIGUOUS_EVENT', payload: { value: 2 } }); + } finally { + state.restore(); + } + if (state.skipped) return; + assert.equal(state.swapped, true); + // Post-condition witnesses: canonical authority is the replaced ledger left directly in place by the hook, + // the pending journal was removed, and derived state is ahead of the replaced ledger. + assert.equal(fs.existsSync(paths.pending), false); + assert.equal(fs.existsSync(state.displaced), true); + assert.equal(fs.readFileSync(state.displaced, 'utf8').split('\n').filter(Boolean).length, 2, + 'displaced ledger holds both committed events'); + assert.equal(fs.readFileSync(paths.events, 'utf8'), preLedger, + 'events.jsonl contains the authentic swapped preLedger without synthetic post-operation rewrite'); + assert.equal(loadCanonicalEvents(rootDir).length, 1); + assert.throws(() => loadStateSnapshot(rootDir, { rebuildIfNeeded: false }), /truncated|differs|materialized/i); + assert.throws(() => verifyStateEngineIntegrity(rootDir), /truncated|differs|materialized|equivalent/i); +}); + +test('T04-H44 ledger swap between final lstat and journal unlink is detected fail-closed on next load (recovery)', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + const before = fs.readFileSync(paths.events, 'utf8'); + appendMetadataEvent({ rootDir, eventType: 'FOLLOWUP_EVENT', payload: { value: 2 } }); + const suffix = fs.readFileSync(paths.events, 'utf8').slice(before.length); + fs.writeFileSync(paths.events, before + suffix.slice(0, 10), 'utf8'); + fs.writeFileSync(paths.pending, JSON.stringify({ + schemaVersion: '1.0.0', + previousByteLength: Buffer.byteLength(before), + previousHash: 'sha256:' + createHash('sha256').update(before).digest('hex'), + suffix, + }), 'utf8'); + const state = swapLedgerBeforeJournalUnlink(t, paths, before); + try { + assert.throws(() => loadStateSnapshot(rootDir, { rebuildIfNeeded: true }), /truncated|differs|materialized/i); + } finally { + state.restore(); + } + if (state.skipped) return; + assert.equal(state.swapped, true); + assert.equal(fs.existsSync(paths.pending), false); + assert.equal(fs.existsSync(state.displaced), true); + assert.equal(fs.readFileSync(paths.events, 'utf8'), before, + 'events.jsonl contains the authentic swapped before ledger without synthetic rewrite'); + assert.throws(() => loadStateSnapshot(rootDir, { rebuildIfNeeded: false }), /truncated|differs|materialized/i); + assert.throws(() => verifyStateEngineIntegrity(rootDir), /truncated|differs|materialized|equivalent/i); +}); + +test('T04-H45 missing events.jsonl with pending journal present fails closed across all accessors', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'BASELINE_EVENT', payload: { value: 1 } }); + const paths = getStateEnginePaths(rootDir); + + // Setup pending commit marker + fs.writeFileSync(paths.pending, JSON.stringify({ + schemaVersion: '1.0.0', + previousByteLength: 0, + previousHash: 'sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + suffix: '{"schemaVersion":"1.0.0"}\n', + }), 'utf8'); + + // Remove the canonical events ledger + fs.unlinkSync(paths.events); + + // Verify loadCanonicalEvents fails closed + assert.throws( + () => loadCanonicalEvents(rootDir), + /events\.jsonl is missing while pending-commit\.json is present/i, + ); + + // Verify loadStateSnapshot fails closed + assert.throws( + () => loadStateSnapshot(rootDir, { rebuildIfNeeded: false }), + /events\.jsonl is missing while pending-commit\.json is present/i, + ); + + // Verify ensureStateEngineLayout fails closed rather than wiping/creating an empty ledger + assert.throws( + () => ensureStateEngineLayout(rootDir), + /events\.jsonl is missing while pending-commit\.json is present/i, + ); + + // Verify verifyStateEngineIntegrity fails closed + assert.throws( + () => verifyStateEngineIntegrity(rootDir), + /events\.jsonl is missing while pending-commit\.json is present/i, + ); +}); + +test('T04-H46 recovery fsyncs a fully present but unflushed suffix before acknowledging journal', (t) => { + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'H46_INITIAL', payload: { seq: 1 } }); + const paths = getStateEnginePaths(rootDir); + const before = fs.readFileSync(paths.events, 'utf8'); + appendMetadataEvent({ rootDir, eventType: 'H46_FOLLOWUP', payload: { seq: 2 } }); + const full = fs.readFileSync(paths.events, 'utf8'); + const suffix = full.slice(before.length); + // Writer died after writing the complete suffix, before fsync and journal removal. + fs.writeFileSync(paths.pending, JSON.stringify({ + schemaVersion: '1.0.0', + previousByteLength: Buffer.byteLength(before), + previousHash: 'sha256:' + createHash('sha256').update(before).digest('hex'), + suffix, + }), 'utf8'); + const ledger = fs.statSync(paths.events); + const originalFsync = fs.fsyncSync; + const originalUnlink = fs.unlinkSync; + let ledgerFlushed = false; + let observedJournalRemoval = false; + fs.fsyncSync = function trackLedgerFsync(fd) { + const stat = fs.fstatSync(fd); + if (stat.isFile() && stat.dev === ledger.dev && stat.ino === ledger.ino) ledgerFlushed = true; + return originalFsync.call(this, fd); + }; + fs.unlinkSync = function guardJournalUnlink(target, ...args) { + if (typeof target === 'string' && target.endsWith('pending-commit.json')) { + observedJournalRemoval = true; + assert.ok(ledgerFlushed, + 'Recovery must fsync fully present canonical suffix before removing pending journal'); + } + return originalUnlink.call(this, target, ...args); + }; + try { + assert.equal(loadStateSnapshot(rootDir, { rebuildIfNeeded: true }).lastEventSequence, 2); + assert.ok(observedJournalRemoval); + assert.equal(fs.existsSync(paths.pending), false); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); + } finally { + fs.fsyncSync = originalFsync; + fs.unlinkSync = originalUnlink; + } +}); + + +test('T04-H49 Windows refuses canonical ledger mutation when pending-journal directory flush fails', (t) => { + if (process.platform !== 'win32') { + t.skip('Windows write-capable directory flush barrier'); + return; + } + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'H49_BASELINE', payload: { step: 1 } }); + const paths = getStateEnginePaths(rootDir); + const before = fs.readFileSync(paths.events); + const dir = fs.statSync(paths.stateRoot); + const originalFsync = fs.fsyncSync; + let injected = false; + fs.fsyncSync = function failStateDirectory(fd) { + const witness = fs.fstatSync(fd); + if (witness.isDirectory() && witness.dev === dir.dev && witness.ino === dir.ino) { + injected = true; + const error = new Error('injected Windows namespace durability failure'); + error.code = 'EIO'; + throw error; + } + return originalFsync.call(this, fd); + }; + try { + assert.throws( + () => appendMetadataEvent({ rootDir, eventType: 'H49_FOLLOWUP', payload: { step: 2 } }), + /directory durability barrier failed/i, + ); + } finally { + fs.fsyncSync = originalFsync; + } + assert.equal(injected, true); + assert.deepEqual(fs.readFileSync(paths.events), before, + 'The ledger must not be changed if publishing the pending journal is not durable'); + assert.equal(fs.existsSync(paths.pending), true, + 'Keep recovery evidence when pending journal namespace cannot be flushed'); +}); + +test('T04-H50 Windows flushes pending-journal directory before any physical canonical append', (t) => { + if (process.platform !== 'win32') { + t.skip('Windows pending-journal namespace ordering'); + return; + } + const rootDir = tempProject(t); + appendMetadataEvent({ rootDir, eventType: 'H50_BASELINE', payload: { step: 1 } }); + const paths = getStateEnginePaths(rootDir); + const dir = fs.statSync(paths.stateRoot); + const ledger = fs.statSync(paths.events); + const originalFsync = fs.fsyncSync; + const originalWrite = fs.writeSync; + let namespaceFlushed = false; + let sawLedgerAppend = false; + fs.fsyncSync = function watchStateDirectoryFlush(fd) { + const stat = fs.fstatSync(fd); + const returned = originalFsync.call(this, fd); + if (stat.isDirectory() && stat.dev === dir.dev && stat.ino === dir.ino) { + namespaceFlushed = true; + } + return returned; + }; + fs.writeSync = function guardLedgerAppend(fd, ...args) { + const stat = fs.fstatSync(fd); + if (stat.isFile() && stat.dev === ledger.dev && stat.ino === ledger.ino) { + sawLedgerAppend = true; + assert.equal(namespaceFlushed, true, + 'The pending journal must be flushed in its parent before appending ledger bytes'); + } + return originalWrite.call(this, fd, ...args); + }; + try { + appendMetadataEvent({ rootDir, eventType: 'H50_FOLLOWUP', payload: { step: 2 } }); + } finally { + fs.fsyncSync = originalFsync; + fs.writeSync = originalWrite; + } + assert.equal(sawLedgerAppend, true); + assert.equal(fs.existsSync(paths.pending), false); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); +}); diff --git a/scripts/state-engine-v2-migration.test.mjs b/scripts/state-engine-v2-migration.test.mjs new file mode 100644 index 00000000..fc624367 --- /dev/null +++ b/scripts/state-engine-v2-migration.test.mjs @@ -0,0 +1,1061 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync as requireSpawnSync } from 'node:child_process'; + +import { + appendEntityState, + canonicalStateJson, + countStateEngineFiles, + getStateEnginePaths, + loadCanonicalEvents, + loadEntityState, + verifyStateEngineIntegrity, +} from '../runtime/orchestration/state-engine-v2.mjs'; +import { + getLegacyBackupPath, + inspectLegacyState, + migrateLegacyStateToV2, + restoreLegacyBackup, + recoverLegacyRestore, +} from '../runtime/orchestration/state-engine-migration.mjs'; +import { + getCurrentState, + saveStateRevision, +} from '../runtime/autopilot/state-store.mjs'; +import { + loadCurrentRunState, +} from '../runtime/orchestration/orchestration-run.mjs'; + +function tempProject(t, prefix = 'dk-state-migration-') { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + t.after(() => fs.rmSync(rootDir, { recursive: true, force: true })); + return rootDir; +} + +function workflowState(revision, workflowId = 'wf-legacy-001') { + return { + schemaVersion: '1.0.0', + workflowId, + projectId: 'proj-legacy', + workspaceId: 'ws-legacy', + workflowMode: 'autopilot', + autonomyLevel: 'guided-autopilot', + workflowStatus: revision === 26 ? 'paused' : 'executing', + currentStage: revision >= 20 ? 'IMPLEMENT' : revision >= 10 ? 'DESIGN' : 'UNDERSTAND', + completedStages: revision >= 10 ? ['UNDERSTAND', 'DEFINE'] : [], + skippedStages: [], + blockedStages: [], + activeAction: null, + pendingApproval: null, + pendingConfirmation: null, + activeDecisionMenu: null, + stateRevision: revision, + createdAt: '2026-09-01T00:00:00.000Z', + updatedAt: new Date(Date.parse('2026-09-01T00:00:00.000Z') + revision * 1000).toISOString(), + frameworkVersion: '0.11.2', + }; +} + +function writeLegacyAutopilot(rootDir, count = 26) { + const stateDir = path.join(rootDir, '.development-kit', 'autopilot', 'state'); + fs.mkdirSync(stateDir, { recursive: true }); + const workflowId = 'wf-legacy-001'; + for (let revision = 1; revision <= count; revision += 1) { + const name = 'revision-' + String(revision).padStart(6, '0') + '.json'; + fs.writeFileSync( + path.join(stateDir, name), + JSON.stringify(workflowState(revision, workflowId), null, 2) + '\n', + 'utf8', + ); + } + fs.writeFileSync( + path.join(stateDir, 'current.json'), + JSON.stringify({ + currentRevision: count, + currentRevisionFile: 'revision-' + String(count).padStart(6, '0') + '.json', + workflowId, + updatedAt: '2026-09-01T01:00:00.000Z', + }, null, 2) + '\n', + 'utf8', + ); + return { stateDir, workflowId, current: workflowState(count, workflowId) }; +} + +function runState(revision, state = 'READY', acceptanceState = 'PENDING') { + return { + schemaVersion: '1.0.0', + contractId: 'INC-LEGACY-RUN', + taskId: 'TASK-LEGACY-RUN', + runId: 'run-legacy-001', + sourceFingerprint: 'sha256:' + 'b'.repeat(64), + createdAt: '2026-09-02T00:00:00.000Z', + updatedAt: new Date(Date.parse('2026-09-02T00:00:00.000Z') + revision * 1000).toISOString(), + stateRevision: revision, + state, + executionStrategy: 'sequential-fresh-context', + hostCapabilities: { + fileRead: true, + fileWrite: true, + shell: true, + git: true, + freshContext: true, + subagents: false, + parallelAgents: false, + browser: false, + visualInspection: false, + externalModelRouting: false, + }, + manualEvidenceRequired: false, + requiredGates: { + verification: ['specification', 'tests'], + reviewers: ['code-reviewer'], + controlDomains: [], + humanApprovals: [], + configurationReadiness: { required: false }, + }, + completedGates: state === 'ACCEPTED' ? ['specification', 'tests', 'code-reviewer'] : [], + correctionAttempt: 0, + failureSignatures: [], + verificationVerdict: state === 'ACCEPTED' ? 'PASS' : null, + acceptanceState, + }; +} + +function writeLegacyRun(rootDir) { + const run1 = runState(1, 'READY', 'PENDING'); + const run2 = runState(2, 'VERIFYING', 'PENDING'); + const run3 = runState(3, 'ACCEPTED', 'ACCEPTED'); + const runDir = path.join( + rootDir, + '.development-kit', + 'runs', + run1.contractId, + run1.runId, + ); + const revisions = path.join(runDir, 'state-revisions'); + fs.mkdirSync(revisions, { recursive: true }); + fs.writeFileSync(path.join(runDir, 'manifest.json'), JSON.stringify(run1, null, 2) + '\n', 'utf8'); + for (const run of [run1, run2, run3]) { + const name = String(run.stateRevision).padStart(8, '0') + '.json'; + fs.writeFileSync(path.join(revisions, name), JSON.stringify(run, null, 2) + '\n', 'utf8'); + } + fs.writeFileSync( + path.join(runDir, 'current-state.json'), + JSON.stringify({ + schemaVersion: '1.0.0', + contractId: run3.contractId, + runId: run3.runId, + stateRevision: run3.stateRevision, + revisionPath: 'state-revisions/00000003.json', + }, null, 2) + '\n', + 'utf8', + ); + fs.writeFileSync(path.join(runDir, 'final-state.json'), JSON.stringify(run3, null, 2) + '\n', 'utf8'); + return { runDir, current: run3 }; +} + +test('AC-015 unmigrated legacy Autopilot projects remain on legacy state until migration', (t) => { + const rootDir = tempProject(t); + const legacy = writeLegacyAutopilot(rootDir, 2); + assert.deepEqual(getCurrentState(rootDir), legacy.current); + + const third = workflowState(3, legacy.workflowId); + saveStateRevision(third, rootDir); + assert.equal( + fs.existsSync(path.join(legacy.stateDir, 'revision-000003.json')), + true, + ); + assert.deepEqual(getCurrentState(rootDir), third); + assert.equal(fs.existsSync(path.join(rootDir, '.development-kit', 'state')), false); +}); + +test('AC-015 partial V2 import cannot cut over an existing legacy project before MIGRATION_COMPLETED', (t) => { + const rootDir = tempProject(t); + const legacy = writeLegacyAutopilot(rootDir, 2); + + const partial = workflowState(2, legacy.workflowId); + partial.currentStage = 'REVIEW'; + appendEntityState({ + rootDir, + entityType: 'autopilot-workflow', + entityId: legacy.workflowId, + state: partial, + actorClass: 'legacy-migration', + timestamp: partial.updatedAt, + }); + + // A partially imported V2 entity exists, but there is no verified + // MIGRATION_COMPLETED event. The legacy store must therefore remain canonical. + const beforeCompletion = getCurrentState(rootDir); + assert.equal(beforeCompletion.currentStage, legacy.current.currentStage); + assert.notEqual(beforeCompletion.currentStage, partial.currentStage); + + const nextLegacy = workflowState(3, legacy.workflowId); + saveStateRevision(nextLegacy, rootDir); + assert.equal( + fs.existsSync(path.join(legacy.stateDir, 'revision-000003.json')), + true, + ); + assert.equal(getCurrentState(rootDir).stateRevision, 3); +}); + +test('AC-015 legacy migration is idempotent, semantically equivalent, and retains recoverable legacy state', (t) => { + const rootDir = tempProject(t); + const legacy = writeLegacyAutopilot(rootDir, 26); + const inspection = inspectLegacyState(rootDir); + assert.equal(inspection.legacyFileCount, 27); + assert.equal(inspection.entityCount, 1); + + const first = migrateLegacyStateToV2({ + rootDir, + completedAt: '2026-10-07T18:30:00.000Z', + }); + assert.equal(first.migrated, true); + assert.equal(first.idempotent, false); + assert.equal(first.semanticEquivalence, true); + assert.equal(first.legacyRetained, true); + assert.equal(canonicalStateJson(getCurrentState(rootDir)), canonicalStateJson(legacy.current)); + assert.equal(canonicalStateJson( + loadEntityState('autopilot-workflow', legacy.workflowId, rootDir), + ), canonicalStateJson(legacy.current)); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); + + for (let revision = 1; revision <= 26; revision += 1) { + assert.equal( + fs.existsSync(path.join( + legacy.stateDir, + 'revision-' + String(revision).padStart(6, '0') + '.json', + )), + true, + ); + } + assert.equal(fs.existsSync(path.join(legacy.stateDir, 'current.json')), true); + assert.equal(fs.existsSync(getLegacyBackupPath(rootDir)), true); + + const eventsBefore = loadCanonicalEvents(rootDir).length; + const second = migrateLegacyStateToV2({ + rootDir, + completedAt: '2026-10-07T18:31:00.000Z', + }); + assert.equal(second.migrated, false); + assert.equal(second.idempotent, true); + assert.equal(loadCanonicalEvents(rootDir).length, eventsBefore); +}); + +test('AC-015 completed migration remains idempotent after legitimate V2 progress', (t) => { + const rootDir = tempProject(t); + const legacy = writeLegacyAutopilot(rootDir, 3); + const first = migrateLegacyStateToV2({ + rootDir, + completedAt: '2026-10-07T18:31:30.000Z', + }); + assert.equal(first.migrated, true); + + const progressed = workflowState(4, legacy.workflowId); + progressed.currentStage = 'VERIFY'; + appendEntityState({ + rootDir, + entityType: 'autopilot-workflow', + entityId: legacy.workflowId, + state: progressed, + actorClass: 'autopilot', + timestamp: progressed.updatedAt, + }); + + const eventCountBefore = loadCanonicalEvents(rootDir).length; + const rerun = migrateLegacyStateToV2({ + rootDir, + completedAt: '2026-10-07T18:31:40.000Z', + }); + + assert.equal(rerun.migrated, false); + assert.equal(rerun.idempotent, true); + assert.equal(loadCanonicalEvents(rootDir).length, eventCountBefore); + assert.equal(getCurrentState(rootDir).stateRevision, 4); + assert.equal(getCurrentState(rootDir).currentStage, 'VERIFY'); +}); + +test('AC-015 bundled legacy backup can restore the complete legacy state chain', (t) => { + const rootDir = tempProject(t); + writeLegacyAutopilot(rootDir, 5); + migrateLegacyStateToV2({ + rootDir, + completedAt: '2026-10-07T18:32:00.000Z', + }); + + const restoreRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'dk-state-restore-')); + t.after(() => fs.rmSync(restoreRoot, { recursive: true, force: true })); + const restored = restoreLegacyBackup({ confirmOffline: true, + backupPath: getLegacyBackupPath(rootDir), + targetRoot: restoreRoot, + }); + assert.equal(restored.restoredFiles.length, 6); + + for (let revision = 1; revision <= 5; revision += 1) { + assert.equal(fs.existsSync(path.join( + restoreRoot, + '.development-kit', + 'autopilot', + 'state', + 'revision-' + String(revision).padStart(6, '0') + '.json', + )), true); + } + assert.equal(fs.existsSync(path.join( + restoreRoot, + '.development-kit', + 'autopilot', + 'state', + 'current.json', + )), true); +}); + +test('AC-015 missing Autopilot revision fails closed before migration cutover', (t) => { + const rootDir = tempProject(t); + const legacy = writeLegacyAutopilot(rootDir, 4); + fs.rmSync(path.join(legacy.stateDir, 'revision-000002.json')); + + assert.throws( + () => migrateLegacyStateToV2({ rootDir }), + /revision chain is missing revision-000002\.json/, + ); + assert.equal(fs.existsSync(path.join(rootDir, '.development-kit', 'state')), false); +}); + +test('AC-015 missing orchestration revision and manifest mismatch fail closed', (t) => { + const rootDir = tempProject(t); + const legacy = writeLegacyRun(rootDir); + fs.rmSync(path.join(legacy.runDir, 'state-revisions', '00000002.json')); + + assert.throws( + () => migrateLegacyStateToV2({ rootDir }), + /revision chain is missing 00000002\.json/, + ); + assert.equal(fs.existsSync(path.join(rootDir, '.development-kit', 'state')), false); + + // Restore revision 2 and then corrupt revision 1 relative to manifest. + const run2 = runState(2, 'VERIFYING', 'PENDING'); + fs.writeFileSync( + path.join(legacy.runDir, 'state-revisions', '00000002.json'), + JSON.stringify(run2, null, 2) + '\n', + 'utf8', + ); + const run1 = runState(1, 'IMPLEMENTING', 'PENDING'); + fs.writeFileSync( + path.join(legacy.runDir, 'state-revisions', '00000001.json'), + JSON.stringify(run1, null, 2) + '\n', + 'utf8', + ); + + assert.throws( + () => migrateLegacyStateToV2({ rootDir }), + /revision 1 differs from immutable manifest/, + ); + assert.equal(fs.existsSync(path.join(rootDir, '.development-kit', 'state')), false); +}); + +test('AC-015 corrupted legacy input fails closed before State Engine cutover', (t) => { + const rootDir = tempProject(t); + const legacy = writeLegacyAutopilot(rootDir, 3); + fs.writeFileSync(path.join(legacy.stateDir, 'revision-000002.json'), '{"broken":', 'utf8'); + + assert.throws( + () => migrateLegacyStateToV2({ rootDir }), + /invalid JSON/, + ); + assert.equal(fs.existsSync(path.join(rootDir, '.development-kit', 'state')), false); + assert.equal(fs.existsSync(path.join(legacy.stateDir, 'current.json')), true); +}); + +test('AC-015 partial orchestration import cannot override legacy current state before verified migration completion', (t) => { + const rootDir = tempProject(t); + const legacy = writeLegacyRun(rootDir); + + const partial = structuredClone(legacy.current); + partial.state = 'BLOCKED'; + partial.acceptanceState = 'BLOCKED'; + appendEntityState({ + rootDir, + entityType: 'orchestration-run', + entityId: partial.contractId + '/' + partial.runId, + state: partial, + refs: { + contractId: partial.contractId, + taskId: partial.taskId, + runId: partial.runId, + }, + actorClass: 'legacy-migration', + timestamp: partial.updatedAt, + }); + + const current = loadCurrentRunState( + legacy.current.contractId, + legacy.current.runId, + rootDir, + ); + assert.equal(current.state, 'ACCEPTED'); + assert.equal(current.acceptanceState, 'ACCEPTED'); +}); + +test('AC-015 orchestration migration preserves ACCEPTED meaning and switches current-state reads to V2', (t) => { + const rootDir = tempProject(t); + const legacy = writeLegacyRun(rootDir); + const before = loadCurrentRunState( + legacy.current.contractId, + legacy.current.runId, + rootDir, + ); + assert.equal(before.state, 'ACCEPTED'); + assert.equal(before.acceptanceState, 'ACCEPTED'); + + const migrated = migrateLegacyStateToV2({ + rootDir, + completedAt: '2026-10-07T18:33:00.000Z', + }); + assert.equal(migrated.semanticEquivalence, true); + + const v2 = loadEntityState( + 'orchestration-run', + legacy.current.contractId + '/' + legacy.current.runId, + rootDir, + ); + assert.equal(canonicalStateJson(v2), canonicalStateJson(legacy.current)); + + const after = loadCurrentRunState( + legacy.current.contractId, + legacy.current.runId, + rootDir, + ); + assert.equal(after.state, 'ACCEPTED'); + assert.equal(after.acceptanceState, 'ACCEPTED'); + assert.equal(after.verificationVerdict, 'PASS'); + + assert.equal(fs.existsSync(path.join(legacy.runDir, 'state-revisions', '00000003.json')), true); + assert.equal(fs.existsSync(path.join(legacy.runDir, 'current-state.json')), true); + assert.equal(fs.existsSync(path.join(legacy.runDir, 'final-state.json')), true); +}); + +test('AC-034 representative 26-revision migration exceeds the >=80% active canonical micro-file reduction objective', (t) => { + const rootDir = tempProject(t); + writeLegacyAutopilot(rootDir, 26); + const legacyCount = inspectLegacyState(rootDir).legacyFileCount; + assert.equal(legacyCount, 27); + + migrateLegacyStateToV2({ + rootDir, + completedAt: '2026-10-07T18:34:00.000Z', + }); + + const canonicalFiles = countStateEngineFiles(rootDir); + const reductionPercent = Number((((legacyCount - canonicalFiles) / legacyCount) * 100).toFixed(2)); + process.stdout.write('T04_STATE_FILE_EVIDENCE ' + JSON.stringify({ + legacyTransitionStateFiles: legacyCount, + stateEngineCanonicalFilesIncludingRecoveryBundle: canonicalFiles, + reductionPercent, + legacyFilesRetainedForRecovery: true, + }) + '\n'); + + assert.equal(canonicalFiles, 5); + assert.equal(reductionPercent, 81.48); + assert.ok(reductionPercent >= 80); + + const paths = getStateEnginePaths(rootDir); + for (const required of ['events', 'snapshot', 'index', 'schema']) { + assert.equal(fs.existsSync(paths[required]), true); + } + assert.equal(fs.existsSync(getLegacyBackupPath(rootDir)), true); +}); + + +test('T04-H05 restore refuses a preexisting symlink/junction parent and leaves outside directory untouched', (t) => { + const rootDir = tempProject(t, 'dk-restore-source-'); + writeLegacyAutopilot(rootDir, 2); + const migrated = migrateLegacyStateToV2({ rootDir }); + const restoreRoot = tempProject(t, 'dk-restore-target-'); + const outside = tempProject(t, 'dk-restore-outside-'); + const dkRoot = path.join(restoreRoot, '.development-kit'); + fs.mkdirSync(dkRoot, { recursive: true }); + try { + fs.symlinkSync(outside, path.join(dkRoot, 'autopilot'), process.platform === 'win32' ? 'junction' : 'dir'); + } catch (error) { + if (['EPERM', 'EACCES', 'ENOTSUP'].includes(error.code)) { + t.skip('Host does not permit directory symlink/junction creation'); + return; + } + throw error; + } + assert.throws( + () => restoreLegacyBackup({ confirmOffline: true, backupPath: migrated.backupPath, targetRoot: restoreRoot }), + /symbolic link|junction|escapes/i, + ); + assert.deepEqual(fs.readdirSync(outside), []); +}); + +test('T04-H06 restore refuses a final-file symlink even when overwrite is explicitly enabled', (t) => { + const rootDir = tempProject(t, 'dk-restore-source-'); + writeLegacyAutopilot(rootDir, 2); + const migrated = migrateLegacyStateToV2({ rootDir }); + const restoreRoot = tempProject(t, 'dk-restore-target-'); + const outside = tempProject(t, 'dk-restore-outside-'); + const filename = path.join(outside, 'should-not-change.json'); + fs.writeFileSync(filename, 'SAFE', 'utf8'); + const parent = path.join(restoreRoot, '.development-kit', 'autopilot', 'state'); + fs.mkdirSync(parent, { recursive: true }); + try { + fs.symlinkSync(filename, path.join(parent, 'revision-000001.json'), 'file'); + } catch (error) { + if (['EPERM', 'EACCES', 'ENOTSUP'].includes(error.code)) { + t.skip('Host does not permit file symlink creation'); + return; + } + throw error; + } + assert.throws( + () => restoreLegacyBackup({ confirmOffline: true, backupPath: migrated.backupPath, targetRoot: restoreRoot, overwrite: true }), + /regular file|symbolic link|junction/i, + ); + assert.equal(fs.readFileSync(filename, 'utf8'), 'SAFE'); +}); + + +test('T04-H13 backup restore rejects a hard-linked overwrite destination', (t) => { + const rootDir = tempProject(t, 'dk-hardlink-source-'); + writeLegacyAutopilot(rootDir, 2); + const migrated = migrateLegacyStateToV2({ rootDir }); + const restoreRoot = tempProject(t, 'dk-hardlink-target-'); + const outside = tempProject(t, 'dk-hardlink-outside-'); + const external = path.join(outside, 'protected.json'); + fs.writeFileSync(external, 'PROTECTED', 'utf8'); + const destinationDir = path.join(restoreRoot, '.development-kit', 'autopilot', 'state'); + fs.mkdirSync(destinationDir, { recursive: true }); + const destination = path.join(destinationDir, 'revision-000001.json'); + try { + fs.linkSync(external, destination); + } catch (error) { + if (['EPERM', 'EACCES', 'ENOTSUP', 'EXDEV'].includes(error.code)) { + t.skip('Filesystem disallows hard-link fixture'); + return; + } + throw error; + } + assert.throws( + () => restoreLegacyBackup({ confirmOffline: true, backupPath: migrated.backupPath, targetRoot: restoreRoot, overwrite: true }), + /hard.?link|multiple links/i, + ); + assert.equal(fs.readFileSync(external, 'utf8'), 'PROTECTED'); +}); + + +test('T04-H19 restoring a backup cannot write through a destination hardlink swapped at open time', (t) => { + const source = tempProject(t, 'dk-restore-race-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const targetRoot = tempProject(t, 'dk-restore-race-target-'); + const protectedRoot = tempProject(t, 'dk-restore-race-outside-'); + const victim = path.join(protectedRoot, 'external.json'); + fs.writeFileSync(victim, 'EXTERNAL-CONTENT', 'utf8'); + const parent = path.join(targetRoot, '.development-kit', 'autopilot', 'state'); + fs.mkdirSync(parent, {recursive:true}); + const destination = path.join(parent, 'revision-000001.json'); + fs.writeFileSync(destination, 'stale value', 'utf8'); + const originalOpen = fs.openSync; + let swapped = false; + fs.openSync = function swappedAtDestinationOpen(file, flags, ...rest) { + if (!swapped && file === destination) { + swapped = true; + fs.rmSync(destination); + fs.linkSync(victim, destination); + } + return originalOpen.call(this, file, flags, ...rest); + }; + try { + const restored = restoreLegacyBackup({ confirmOffline: true, backupPath:migrated.backupPath, targetRoot, overwrite:true }); + assert.ok(restored.restoredFiles.includes('.development-kit/autopilot/state/revision-000001.json')); + } finally { + fs.openSync = originalOpen; + } + assert.equal(fs.readFileSync(victim, 'utf8'), 'EXTERNAL-CONTENT'); + assert.equal(JSON.parse(fs.readFileSync(destination, 'utf8')).stateRevision, 1); +}); + + +test('T04-H20 restore never creates target-tree directories before atomic staged promotion', (t) => { + const source = tempProject(t, 'dk-restore-stage-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const targetRoot = tempProject(t, 'dk-restore-stage-target-'); + const originalMkdir = fs.mkdirSync; + let attemptedUnsafeWrite = false; + fs.mkdirSync = function blockPrepromotionTargetWrites(directory, ...args) { + const relative = path.relative(targetRoot, path.resolve(directory)); + if (relative && relative !== '..' && !relative.startsWith('..' + path.sep) + && !path.isAbsolute(relative)) { + attemptedUnsafeWrite = true; + throw new Error('restore preflight attempted a direct target-directory mutation'); + } + return originalMkdir.call(this, directory, ...args); + }; + try { + const restored = restoreLegacyBackup({ confirmOffline: true, backupPath: migrated.backupPath, targetRoot }); + assert.equal(restored.restoredFiles.length, 3); + } finally { + fs.mkdirSync = originalMkdir; + } + assert.equal(attemptedUnsafeWrite, false); + const pathToRestored = path.join(targetRoot, '.development-kit', 'autopilot', 'state', 'revision-000001.json'); + assert.equal(JSON.parse(fs.readFileSync(pathToRestored, 'utf8')).stateRevision, 1); +}); + + +test('T04-H21 stage-promoted restore supports an absent root within an existing parent directory', (t) => { + const source = tempProject(t, 'dk-restore-newroot-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const parent = tempProject(t, 'dk-restore-newroot-parent-'); + const root = path.join(parent, 'new-restore-root'); + assert.equal(fs.existsSync(root), false); + const restored = restoreLegacyBackup({ confirmOffline: true, backupPath: migrated.backupPath, targetRoot: root }); + assert.equal(restored.restoredFiles.length, 3); + assert.equal( + JSON.parse(fs.readFileSync(path.join(root, '.development-kit','autopilot','state','revision-000001.json'), 'utf8')).stateRevision, + 1, + ); +}); + + +test('T04-H23 staged restore preserves the existing destination root mode and ownership', (t) => { + if (process.platform === 'win32') { + t.skip('POSIX mode/uid semantics are not portable to Windows'); return; + } + const source = tempProject(t, 'dk-mode-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const targetRoot = tempProject(t, 'dk-mode-target-'); + fs.chmodSync(targetRoot, 0o755); + const before = fs.statSync(targetRoot); + restoreLegacyBackup({ confirmOffline: true, backupPath: migrated.backupPath, targetRoot }); + const after = fs.statSync(targetRoot); + assert.equal(after.mode & 0o7777, before.mode & 0o7777); + assert.equal(after.uid, before.uid); + assert.equal(after.gid, before.gid); + assert.equal( + JSON.parse(fs.readFileSync(path.join(targetRoot,'.development-kit','autopilot','state','revision-000001.json'),'utf8')).stateRevision, + 1, + ); +}); + + +test('T04-H24 legacy restore requires explicit offline capability', (t) => { + const source = tempProject(t, 'dk-offline-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const root = tempProject(t, 'dk-offline-target-'); + assert.throws(() => restoreLegacyBackup({ backupPath: migrated.backupPath, targetRoot: root }), + /offline confirmation/i); + assert.deepEqual(fs.readdirSync(root), []); +}); + +test('T04-H25 interrupted two-step root promotion can restore original using journal', (t) => { + const source = tempProject(t, 'dk-crash-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const root = tempProject(t, 'dk-crash-target-'); + const sentinel = path.join(root, 'persist-original.txt'); + fs.writeFileSync(sentinel, 'ORIGINAL'); + const originalRename = fs.renameSync; + let movedRoot = false; + let failureInjected = false; + fs.renameSync = function injectPowerFailure(oldPath, nextPath, ...args) { + if (movedRoot && !failureInjected && typeof oldPath === 'string' && + oldPath.includes('.dk-legacy-restore-stage-') && nextPath === root) { + failureInjected = true; + throw new Error('simulated crash between displacement and promotion'); + } + if (oldPath === root && String(nextPath).endsWith('-original')) movedRoot = true; + return originalRename.call(this, oldPath, nextPath, ...args); + }; + try { + assert.throws(() => restoreLegacyBackup({ + confirmOffline: true, backupPath: migrated.backupPath, targetRoot: root, + }), /simulated crash/); + } finally { fs.renameSync = originalRename; } + // If the in-process finally managed the rollback, the leftover journal is + // still authoritative. Recovery must be idempotent and must never erase it. + assert.equal(fs.readFileSync(sentinel, 'utf8'), 'ORIGINAL'); + assert.throws(() => restoreLegacyBackup({ + confirmOffline:true, backupPath:migrated.backupPath, targetRoot:root, + }), /Incomplete legacy restore/i); + assert.throws(() => recoverLegacyRestore({ targetRoot: root }), /offline confirmation/i); + const recovered = recoverLegacyRestore({ targetRoot: root, confirmOffline: true }); + assert.equal(recovered.recovered, true); + assert.equal(fs.readFileSync(sentinel,'utf8'), 'ORIGINAL'); + assert.equal(recoverLegacyRestore({ targetRoot: root, confirmOffline: true }).recovered, false); + const restored = restoreLegacyBackup({ + confirmOffline:true, backupPath:migrated.backupPath, targetRoot:root, + }); + assert.equal(restored.restoredFiles.length, 3); + assert.equal(fs.readFileSync(path.join(root,'persist-original.txt'),'utf8'),'ORIGINAL'); +}); + + +test('T04-H28 restore journals preserve original after a simulated process death between root renames', (t) => { + const source = tempProject(t, 'dk-crash-worker-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const root = tempProject(t, 'dk-crash-worker-target-'); + const sentinel = path.join(root, 'original.txt'); + fs.writeFileSync(sentinel, 'KEEP-ME', 'utf8'); + + const script = [ + 'import fs from "node:fs";', + 'import {restoreLegacyBackup} from '+JSON.stringify(new URL('../runtime/orchestration/state-engine-migration.mjs', import.meta.url).href)+';', + 'const backup=process.argv[1],root=process.argv[2],rename=fs.renameSync;', + 'fs.renameSync=function(a,b,...rest){', + ' if(a===root && String(b).endsWith("-original")) {const r=rename.call(this,a,b,...rest);process.exit(91);}', + ' return rename.call(this,a,b,...rest);', + '};', + 'restoreLegacyBackup({backupPath:backup,targetRoot:root,confirmOffline:true});', + ].join('\n'); + // spawnSync is intentionally used for a true process exit instead of a + // caught synchronous exception that runs finally/rollback. + const result = requireSpawnSync(process.execPath, ['--input-type=module','-e',script,migrated.backupPath,root]); + assert.equal(result.status, 91, result.stderr); + assert.equal(fs.existsSync(root), false); + assert.throws(() => restoreLegacyBackup({ + backupPath:migrated.backupPath,targetRoot:root,confirmOffline:true, + }), /Incomplete legacy restore/i); + const recovered = recoverLegacyRestore({ targetRoot: root, confirmOffline: true }); + assert.equal(recovered.outcome, 'rolled-back-original'); + assert.equal(fs.readFileSync(sentinel,'utf8'),'KEEP-ME'); + assert.equal(fs.existsSync(root),true); +}); + + +test('T04-H47 recovery fsyncs the restored parent entry before deleting its rollback journal', (t) => { + // Runs on Windows too: the barrier is FlushFileBuffers on a directory handle. + const source = tempProject(t, 'dk-rollback-fsync-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const root = tempProject(t, 'dk-rollback-fsync-target-'); + const sentinel = path.join(root, 'original.txt'); + fs.writeFileSync(sentinel, 'ROLLBACK-ORIGINAL', 'utf8'); + + // A real process exit avoids running the normal in-process finally rollback. + const script = [ + 'import fs from "node:fs";', + 'import {restoreLegacyBackup} from ' + + JSON.stringify(new URL('../runtime/orchestration/state-engine-migration.mjs', import.meta.url).href) + ';', + 'const backup=process.argv[1],root=process.argv[2],rename=fs.renameSync;', + 'fs.renameSync=function(a,b,...rest){', + ' if(a===root && String(b).endsWith("-original")) {rename.call(this,a,b,...rest);process.exit(91);}', + ' return rename.call(this,a,b,...rest);', + '};', + 'restoreLegacyBackup({backupPath:backup,targetRoot:root,confirmOffline:true});', + ].join('\n'); + const result = requireSpawnSync(process.execPath, + ['--input-type=module', '-e', script, migrated.backupPath, root]); + assert.equal(result.status, 91, result.stderr); + assert.equal(fs.existsSync(root), false); + + const parent = path.dirname(root); + const parentStat = fs.statSync(parent); + const originalRename = fs.renameSync; + const originalFsync = fs.fsyncSync; + const originalUnlink = fs.unlinkSync; + let rollbackRenamed = false; + let parentSyncedAfterRollback = false; + let journalRemoved = false; + fs.renameSync = function watchRollback(from, to, ...args) { + const returned = originalRename.call(this, from, to, ...args); + if (typeof from === 'string' && from.endsWith('-original') && to === root) { + rollbackRenamed = true; + } + return returned; + }; + fs.fsyncSync = function watchParentDurability(fd) { + const stat = fs.fstatSync(fd); + if (rollbackRenamed && stat.isDirectory() && + stat.dev === parentStat.dev && stat.ino === parentStat.ino) { + parentSyncedAfterRollback = true; + } + return originalFsync.call(this, fd); + }; + fs.unlinkSync = function guardJournalRemoval(target, ...args) { + if (typeof target === 'string' && + path.basename(target).startsWith('.dk-restore-journal-')) { + journalRemoved = true; + assert.equal(rollbackRenamed, true, 'Recovery must restore the displaced root'); + assert.equal(parentSyncedAfterRollback, true, + 'Rollback directory rename must be fsynced before deleting the journal'); + } + return originalUnlink.call(this, target, ...args); + }; + try { + const recovery = recoverLegacyRestore({ targetRoot: root, confirmOffline: true }); + assert.equal(recovery.outcome, 'rolled-back-original'); + assert.ok(journalRemoved); + assert.equal(fs.readFileSync(sentinel, 'utf8'), 'ROLLBACK-ORIGINAL'); + } finally { + fs.renameSync = originalRename; + fs.fsyncSync = originalFsync; + fs.unlinkSync = originalUnlink; + } +}); + +test('T04-H48 Windows crash rollback and recovery verify parent flush barrier before journal unlink', (t) => { + if (process.platform !== 'win32') { + t.skip('Windows parent flush barrier verification'); + return; + } + + const source = tempProject(t, 'dk-win-rollback-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const root = tempProject(t, 'dk-win-rollback-target-'); + const sentinel = path.join(root, 'original.txt'); + fs.writeFileSync(sentinel, 'WINDOWS-ORIGINAL', 'utf8'); + + const script = [ + 'import fs from "node:fs";', + 'import {restoreLegacyBackup} from ' + + JSON.stringify(new URL('../runtime/orchestration/state-engine-migration.mjs', import.meta.url).href) + ';', + 'const backup=process.argv[1],root=process.argv[2],rename=fs.renameSync;', + 'fs.renameSync=function(a,b,...rest){', + ' if(a===root && String(b).endsWith("-original")) {rename.call(this,a,b,...rest);process.exit(91);}', + ' return rename.call(this,a,b,...rest);', + '};', + 'restoreLegacyBackup({backupPath:backup,targetRoot:root,confirmOffline:true});', + ].join('\n'); + const result = requireSpawnSync(process.execPath, + ['--input-type=module', '-e', script, migrated.backupPath, root]); + assert.equal(result.status, 91, result.stderr); + assert.equal(fs.existsSync(root), false); + + const parent = path.dirname(root); + const parentIdentity = fs.statSync(parent); + const originalRename = fs.renameSync; + const originalFsync = fs.fsyncSync; + const originalUnlink = fs.unlinkSync; + let rollbackRenamed = false; + let parentFlushedAfterRollback = false; + let journalRemoved = false; + + fs.renameSync = function watchRollback(from, to, ...args) { + const returned = originalRename.call(this, from, to, ...args); + if (typeof from === 'string' && from.endsWith('-original') && to === root) { + rollbackRenamed = true; + } + return returned; + }; + fs.fsyncSync = function watchParentFlush(fd) { + const stat = fs.fstatSync(fd); + const returned = originalFsync.call(this, fd); + if (rollbackRenamed && stat.isDirectory() && + stat.dev === parentIdentity.dev && stat.ino === parentIdentity.ino) { + parentFlushedAfterRollback = true; + } + return returned; + }; + fs.unlinkSync = function guardJournalRemoval(target, ...args) { + if (typeof target === 'string' && + path.basename(target).startsWith('.dk-restore-journal-')) { + journalRemoved = true; + assert.equal(rollbackRenamed, true, 'Recovery must restore the displaced root'); + assert.equal(parentFlushedAfterRollback, true, + 'Windows rollback directory rename must be flushed to disk before deleting the journal'); + } + return originalUnlink.call(this, target, ...args); + }; + try { + const recovered = recoverLegacyRestore({ targetRoot: root, confirmOffline: true }); + assert.equal(recovered.outcome, 'rolled-back-original'); + assert.ok(journalRemoved); + assert.equal(fs.readFileSync(sentinel, 'utf8'), 'WINDOWS-ORIGINAL'); + } finally { + fs.renameSync = originalRename; + fs.fsyncSync = originalFsync; + fs.unlinkSync = originalUnlink; + } +}); + +test('T04-H30 offline staged restore preserves descendant file identity and POSIX permission metadata', (t) => { + if (process.platform === 'win32') { + t.skip('POSIX ownership and mode semantics require a POSIX host'); return; + } + const source = tempProject(t, 'dk-descendant-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const root = tempProject(t, 'dk-descendant-target-'); + const nested = path.join(root, 'unrelated', 'private.txt'); + fs.mkdirSync(path.dirname(nested), { recursive: true }); + fs.writeFileSync(nested, 'PROTECTED-CONTENT'); + fs.chmodSync(nested, 0o600); + if (process.getuid?.() === 0) { + fs.chownSync(nested, 65534, 65534); + } + const before = fs.statSync(nested); + restoreLegacyBackup({ backupPath:migrated.backupPath, targetRoot:root, confirmOffline:true }); + const after = fs.statSync(nested); + assert.equal(after.uid, before.uid); + assert.equal(after.gid, before.gid); + assert.equal(after.mode & 0o7777, before.mode & 0o7777); + assert.equal(fs.readFileSync(nested, 'utf8'), 'PROTECTED-CONTENT'); +}); + +test('T04-H31 staged restore does not advance historical root access and modification timestamps', (t) => { + if (process.platform === 'win32') { + t.skip('POSIX access-time semantics require a POSIX host'); return; + } + const source = tempProject(t, 'dk-atime-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const root = tempProject(t, 'dk-atime-target-'); + const old = new Date('2001-01-01T00:00:00.000Z'); + fs.utimesSync(root, old, old); + const before = fs.statSync(root); + restoreLegacyBackup({ backupPath:migrated.backupPath, targetRoot:root, confirmOffline:true }); + const after = fs.statSync(root); + assert.ok(Math.abs(after.atimeMs - before.atimeMs) < 1000, 'root access time must be preserved'); + assert.ok(Math.abs(after.mtimeMs - before.mtimeMs) < 1000, 'root modification time must be preserved'); +}); + + +test('T04-H32 offline staged restore preserves the text of unrelated relative symlinks', (t) => { + const source = tempProject(t, 'dk-link-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const targetRoot = tempProject(t, 'dk-link-target-'); + fs.writeFileSync(path.join(targetRoot, 'target.txt'), 'ORIGINAL', 'utf8'); + const link = path.join(targetRoot, 'relative-link.txt'); + try { + fs.symlinkSync('target.txt', link, 'file'); + } catch (error) { + if (['EPERM','EACCES','ENOTSUP'].includes(error.code)) { t.skip('Relative symlink unavailable'); return; } + throw error; + } + const priorLink = fs.readlinkSync(link); + restoreLegacyBackup({ backupPath: migrated.backupPath, targetRoot, confirmOffline: true }); + assert.equal(fs.readlinkSync(link), priorLink); + assert.equal(fs.readFileSync(link, 'utf8'), 'ORIGINAL'); +}); + +test('T04-H33 backup with duplicate normalized destinations is refused before staging', async (t) => { + const source = tempProject(t, 'dk-dup-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const targetRoot = tempProject(t, 'dk-dup-target-'); + const pathA = path.join(targetRoot, '.development-kit', 'autopilot', 'state', 'revision-000001.json'); + const backup = JSON.parse(fs.readFileSync(migrated.backupPath, 'utf8')); + const first = backup.files[0]; + const duplicate = { ...first, path: './' + first.path }; + backup.files.push(duplicate); + backup.fileCount = backup.files.length; + const manifest = backup.files.map(e => ({ path: e.path, fingerprint: e.fingerprint, bytes: e.bytes })); + const crypto = await import('node:crypto'); + function canonical(value) { + if (Array.isArray(value)) return value.map(canonical); + if (value && typeof value === 'object') return Object.fromEntries( + Object.entries(value).sort(([a],[b]) => a.localeCompare(b)).map(([key,val]) => [key, canonical(val)]) + ); + return value; + } + backup.sourceFingerprint = 'sha256:' + crypto.createHash('sha256').update(JSON.stringify(canonical(manifest))).digest('hex'); + const hostileBackup = path.join(source, 'alias-backup.json'); + fs.writeFileSync(hostileBackup, JSON.stringify(backup)); + assert.throws( + () => restoreLegacyBackup({ backupPath: hostileBackup, targetRoot, confirmOffline: true }), + /duplicate normalized/i, + ); + assert.equal(fs.existsSync(pathA), false); +}); + + +test('T04-H34 backup case aliases fail closed even on a case-sensitive test filesystem', async (t) => { + const source = tempProject(t, 'dk-restore-case-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const targetRoot = tempProject(t, 'dk-restore-case-target-'); + const backup = JSON.parse(fs.readFileSync(migrated.backupPath, 'utf8')); + const first = backup.files[0]; + const alias = { ...first, path: first.path.toUpperCase() }; + assert.notEqual(alias.path, first.path); + backup.files.push(alias); + backup.fileCount = backup.files.length; + const manifest = backup.files.map(e => ({ path: e.path, fingerprint: e.fingerprint, bytes: e.bytes })); + const { createHash } = await import('node:crypto'); + function stable(value) { + if (Array.isArray(value)) return value.map(stable); + if (value && typeof value === 'object') return Object.fromEntries( + Object.keys(value).sort().map(key => [key, stable(value[key])])); + return value; + } + backup.sourceFingerprint = 'sha256:' + createHash('sha256').update(JSON.stringify(stable(manifest))).digest('hex'); + const aliasBackup = path.join(source, 'case-alias-backup.json'); + fs.writeFileSync(aliasBackup, JSON.stringify(backup)); + assert.throws( + () => restoreLegacyBackup({backupPath:aliasBackup,targetRoot,confirmOffline:true}), + /duplicate normalized/i, + ); + assert.deepEqual(fs.readdirSync(targetRoot), []); +}); + + +test('T04-H35 backup with distinct unpaired surrogates cannot overwrite one filesystem name', async (t) => { + const source = tempProject(t, 'dk-unicode-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const targetRoot = tempProject(t, 'dk-unicode-target-'); + const backup = JSON.parse(fs.readFileSync(migrated.backupPath, 'utf8')); + const sourceEntry = backup.files[0]; + backup.files.push({ ...sourceEntry, path: 'suspicious-\uD800.json' }); + backup.files.push({ ...sourceEntry, path: 'suspicious-\uD801.json' }); + backup.fileCount = backup.files.length; + const entries = backup.files.map(e => ({path:e.path, fingerprint:e.fingerprint, bytes:e.bytes})); + const { createHash } = await import('node:crypto'); + function stable(value) { + if (Array.isArray(value)) return value.map(stable); + if (value && typeof value === 'object') { + return Object.fromEntries(Object.keys(value).sort().map(k => [k, stable(value[k])])); + } + return value; + } + backup.sourceFingerprint = 'sha256:' + createHash('sha256').update(JSON.stringify(stable(entries))).digest('hex'); + const hostileBackup = path.join(source, 'ill-formed-unicode.json'); + fs.writeFileSync(hostileBackup, JSON.stringify(backup)); + assert.throws( + () => restoreLegacyBackup({ backupPath: hostileBackup, targetRoot, confirmOffline: true }), + /ill-formed Unicode/i, + ); + assert.deepEqual(fs.readdirSync(targetRoot), []); +}); + +test('T04-H39 backup with trailing dot or space path alias is refused before restore', async (t) => { + const source = tempProject(t, 'dk-trailing-dot-source-'); + writeLegacyAutopilot(source, 2); + const migrated = migrateLegacyStateToV2({ rootDir: source }); + const targetRoot = tempProject(t, 'dk-trailing-dot-target-'); + const backup = JSON.parse(fs.readFileSync(migrated.backupPath, 'utf8')); + const sourceEntry = backup.files[0]; + // Attempt to add a trailing-dot alias path that would map to the same file on Windows + backup.files.push({ ...sourceEntry, path: sourceEntry.path + '.' }); + backup.fileCount = backup.files.length; + const entries = backup.files.map(e => ({ path: e.path, fingerprint: e.fingerprint, bytes: e.bytes })); + const { createHash } = await import('node:crypto'); + function stable(value) { + if (Array.isArray(value)) return value.map(stable); + if (value && typeof value === 'object') { + return Object.fromEntries(Object.keys(value).sort().map(k => [k, stable(value[k])])); + } + return value; + } + backup.sourceFingerprint = 'sha256:' + createHash('sha256').update(JSON.stringify(stable(entries))).digest('hex'); + const hostileBackup = path.join(source, 'trailing-dot-backup.json'); + fs.writeFileSync(hostileBackup, JSON.stringify(backup)); + assert.throws( + () => restoreLegacyBackup({ backupPath: hostileBackup, targetRoot, confirmOffline: true }), + /trailing dot|duplicate normalized/i, + ); + assert.deepEqual(fs.readdirSync(targetRoot), []); +}); diff --git a/scripts/state-engine-v2.test.mjs b/scripts/state-engine-v2.test.mjs new file mode 100644 index 00000000..9bda942d --- /dev/null +++ b/scripts/state-engine-v2.test.mjs @@ -0,0 +1,372 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + appendEntityState, + canonicalStateJson, + cleanupOrphanedStateTemps, + countStateEngineFiles, + getStateEnginePaths, + loadCanonicalEvents, + loadStateIndex, + loadStateSnapshot, + queryStateIndex, + rebuildSnapshotFromEvents, + rebuildStateIndex, + rebuildStateSnapshot, + verifyStateEngineIntegrity, +} from '../runtime/orchestration/state-engine-v2.mjs'; +import { + getCurrentState, + saveStateRevision, +} from '../runtime/autopilot/state-store.mjs'; +import { + loadCurrentRunState, + persistRunManifest, + persistRunStateRevision, +} from '../runtime/orchestration/orchestration-run.mjs'; + +function tempProject(t, prefix = 'dk-state-v2-') { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + t.after(() => fs.rmSync(rootDir, { recursive: true, force: true })); + return rootDir; +} + +function workflowState(revision = 1, overrides = {}) { + return { + schemaVersion: '1.0.0', + workflowId: 'wf-state-v2', + projectId: 'proj-state-v2', + workspaceId: 'ws-state-v2', + workflowMode: 'autopilot', + autonomyLevel: 'guided-autopilot', + workflowStatus: 'executing', + currentStage: revision > 1 ? 'DEFINE' : 'UNDERSTAND', + completedStages: revision > 1 ? ['UNDERSTAND'] : [], + skippedStages: [], + blockedStages: [], + activeAction: null, + pendingApproval: null, + pendingConfirmation: null, + activeDecisionMenu: null, + stateRevision: revision, + createdAt: '2026-10-07T18:00:00.000Z', + updatedAt: new Date(Date.parse('2026-10-07T18:00:00.000Z') + revision * 1000).toISOString(), + frameworkVersion: '0.11.2', + ...overrides, + }; +} + +function runState(revision = 1, overrides = {}) { + return { + schemaVersion: '1.0.0', + contractId: 'INC-STATE-V2', + taskId: 'TASK-STATE-V2', + runId: 'run-state-v2', + sourceFingerprint: 'sha256:' + 'a'.repeat(64), + createdAt: '2026-10-07T18:10:00.000Z', + updatedAt: new Date(Date.parse('2026-10-07T18:10:00.000Z') + revision * 1000).toISOString(), + stateRevision: revision, + state: revision === 1 ? 'READY' : 'IMPLEMENTING', + executionStrategy: 'sequential-fresh-context', + hostCapabilities: { + fileRead: true, + fileWrite: true, + shell: true, + git: true, + freshContext: true, + subagents: false, + parallelAgents: false, + browser: false, + visualInspection: false, + externalModelRouting: false, + }, + manualEvidenceRequired: false, + requiredGates: { + verification: ['specification', 'tests'], + reviewers: ['code-reviewer'], + controlDomains: [], + humanApprovals: [], + configurationReadiness: { required: false }, + }, + completedGates: [], + correctionAttempt: 0, + failureSignatures: [], + verificationVerdict: null, + acceptanceState: 'PENDING', + ...overrides, + }; +} + +test('AC-012 canonical event history is monotonic, hash-chained, and reconstructable', (t) => { + const rootDir = tempProject(t); + const first = workflowState(1); + const second = workflowState(2); + + appendEntityState({ + rootDir, + entityType: 'autopilot-workflow', + entityId: first.workflowId, + state: first, + actorClass: 'test', + timestamp: first.updatedAt, + }); + appendEntityState({ + rootDir, + entityType: 'autopilot-workflow', + entityId: second.workflowId, + state: second, + actorClass: 'test', + timestamp: second.updatedAt, + }); + + const events = loadCanonicalEvents(rootDir); + assert.equal(events.length, 2); + assert.equal(events[0].sequence, 1); + assert.equal(events[1].sequence, 2); + assert.equal(events[0].previousEventHash, null); + assert.equal(events[1].previousEventHash, events[0].eventHash); + + const rebuilt = rebuildSnapshotFromEvents(events); + assert.deepEqual( + rebuilt.entities['autopilot-workflow'][first.workflowId].state, + second, + ); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); +}); + +test('AC-012 canonical history detects mutation and reordering', (t) => { + const rootDir = tempProject(t); + appendEntityState({ + rootDir, + entityType: 'generic-state', + entityId: 'one', + state: { stateRevision: 1, value: 1 }, + actorClass: 'test', + timestamp: '2026-10-07T18:20:00.000Z', + }); + appendEntityState({ + rootDir, + entityType: 'generic-state', + entityId: 'one', + state: { stateRevision: 2, value: 2 }, + actorClass: 'test', + timestamp: '2026-10-07T18:20:01.000Z', + }); + + const paths = getStateEnginePaths(rootDir); + const lines = fs.readFileSync(paths.events, 'utf8').trim().split(/\r?\n/); + const mutated = JSON.parse(lines[0]); + mutated.payload.state.value = 99; + fs.writeFileSync(paths.events, JSON.stringify(mutated) + '\n' + lines[1] + '\n', 'utf8'); + assert.throws(() => loadCanonicalEvents(rootDir), /hash mismatch/i); + + fs.writeFileSync(paths.events, lines[1] + '\n' + lines[0] + '\n', 'utf8'); + assert.throws(() => loadCanonicalEvents(rootDir), /First state event sequence|sequence/i); +}); + +test('AC-012 canonical tail truncation fails closed when the materialized snapshot witnesses later history', (t) => { + const rootDir = tempProject(t); + appendEntityState({ + rootDir, + entityType: 'generic-state', + entityId: 'tail-witness', + state: { stateRevision: 1, value: 1 }, + actorClass: 'test', + timestamp: '2026-10-07T18:21:00.000Z', + }); + appendEntityState({ + rootDir, + entityType: 'generic-state', + entityId: 'tail-witness', + state: { stateRevision: 2, value: 2 }, + actorClass: 'test', + timestamp: '2026-10-07T18:21:01.000Z', + }); + + const paths = getStateEnginePaths(rootDir); + const lines = fs.readFileSync(paths.events, 'utf8').trim().split(/\r?\n/); + assert.equal(lines.length, 2); + + // Simulate loss/removal of the canonical tail while retaining the previously + // materialized snapshot as an integrity witness. + fs.writeFileSync(paths.events, lines[0] + '\n', 'utf8'); + + assert.throws( + () => loadStateSnapshot(rootDir, { rebuildIfNeeded: true }), + /history appears truncated/i, + ); +}); + +test('State Engine rejects a state-root symlink or junction that resolves outside the project', (t) => { + const rootDir = tempProject(t, 'dk-state-realpath-'); + const outside = fs.mkdtempSync(path.join(os.tmpdir(), 'dk-state-outside-')); + t.after(() => fs.rmSync(outside, { recursive: true, force: true })); + + const dkDir = path.join(rootDir, '.development-kit'); + fs.mkdirSync(dkDir, { recursive: true }); + const statePath = path.join(dkDir, 'state'); + + try { + fs.symlinkSync(outside, statePath, process.platform === 'win32' ? 'junction' : 'dir'); + } catch (error) { + if (['EPERM', 'EACCES', 'ENOTSUP'].includes(error.code)) { + t.skip('Host does not permit directory symlink/junction creation'); + return; + } + throw error; + } + + assert.throws( + () => appendEntityState({ + rootDir, + entityType: 'generic-state', + entityId: 'escape-attempt', + state: { stateRevision: 1, value: true }, + actorClass: 'test', + timestamp: '2026-10-07T18:22:00.000Z', + }), + /resolves outside project root/, + ); + + assert.equal(fs.readdirSync(outside).length, 0); +}); + +test('tampered State Engine schema contract fails closed on read', (t) => { + const rootDir = tempProject(t); + const state = workflowState(1); + saveStateRevision(state, rootDir); + const paths = getStateEnginePaths(rootDir); + + const schema = JSON.parse(fs.readFileSync(paths.schema, 'utf8')); + schema.authority.canonicalHistory = 'something-else.jsonl'; + fs.writeFileSync(paths.schema, JSON.stringify(schema, null, 2) + '\n', 'utf8'); + + assert.throws( + () => getCurrentState(rootDir), + /schema\.json does not match the runtime schema contract/, + ); +}); + +test('AC-013 materialized snapshot deletion/corruption rebuilds from canonical history', (t) => { + const rootDir = tempProject(t); + const state = workflowState(1); + appendEntityState({ + rootDir, + entityType: 'autopilot-workflow', + entityId: state.workflowId, + state, + actorClass: 'test', + timestamp: state.updatedAt, + }); + + const paths = getStateEnginePaths(rootDir); + const expected = loadStateSnapshot(rootDir); + fs.rmSync(paths.snapshot); + const rebuiltAfterDelete = loadStateSnapshot(rootDir, { rebuildIfNeeded: true }); + assert.equal(canonicalStateJson(rebuiltAfterDelete), canonicalStateJson(expected)); + + fs.writeFileSync(paths.snapshot, '{"corrupt":true}\n', 'utf8'); + const rebuiltAfterCorruption = rebuildStateSnapshot(rootDir); + assert.equal(canonicalStateJson(rebuiltAfterCorruption), canonicalStateJson(expected)); +}); + +test('AC-014 disposable index deletion/rebuild preserves semantically equivalent query state', (t) => { + const rootDir = tempProject(t); + const run = runState(1); + appendEntityState({ + rootDir, + entityType: 'orchestration-run', + entityId: run.contractId + '/' + run.runId, + state: run, + refs: { contractId: run.contractId, taskId: run.taskId, runId: run.runId }, + actorClass: 'test', + timestamp: run.updatedAt, + }); + + const before = loadStateIndex(rootDir); + const queryBefore = queryStateIndex({ rootDir, contractId: run.contractId }); + const paths = getStateEnginePaths(rootDir); + fs.rmSync(paths.index); + const rebuilt = rebuildStateIndex(rootDir); + const queryAfter = queryStateIndex({ rootDir, contractId: run.contractId }); + + assert.equal(canonicalStateJson(rebuilt), canonicalStateJson(before)); + assert.deepEqual(queryAfter, queryBefore); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); +}); + +test('interrupted temporary state files cannot advance canonical history', (t) => { + const rootDir = tempProject(t); + const state = workflowState(1); + appendEntityState({ + rootDir, + entityType: 'autopilot-workflow', + entityId: state.workflowId, + state, + actorClass: 'test', + timestamp: state.updatedAt, + }); + const paths = getStateEnginePaths(rootDir); + const before = fs.readFileSync(paths.events, 'utf8'); + + fs.writeFileSync( + path.join(paths.stateRoot, 'events.jsonl.tmp-interrupted'), + before + '{"partial":', + 'utf8', + ); + + assert.equal(loadCanonicalEvents(rootDir).length, 1); + assert.equal(verifyStateEngineIntegrity(rootDir).valid, true); + const removed = cleanupOrphanedStateTemps(rootDir); + assert.equal(removed.length, 1); + assert.equal(fs.readFileSync(paths.events, 'utf8'), before); +}); + +test('fresh Autopilot projects use State Engine V2 without legacy revision micro-files', (t) => { + const rootDir = tempProject(t); + const first = workflowState(1); + saveStateRevision(first, rootDir); + const second = workflowState(2); + saveStateRevision(second, rootDir); + + assert.deepEqual(getCurrentState(rootDir), second); + assert.equal(fs.existsSync(path.join(rootDir, '.development-kit', 'autopilot', 'state')), false); + assert.equal(loadCanonicalEvents(rootDir).length, 2); + assert.equal(countStateEngineFiles(rootDir), 4); +}); + +test('fresh orchestration transition state uses V2 while manifest remains an immutable artifact', (t) => { + const rootDir = tempProject(t); + const run = runState(1); + persistRunManifest(run, rootDir); + const first = persistRunStateRevision(run, rootDir); + assert.equal(first.stateEngine, 'v2'); + + const second = runState(2); + const next = persistRunStateRevision(second, rootDir); + assert.equal(next.stateEngine, 'v2'); + assert.deepEqual(loadCurrentRunState(run.contractId, run.runId, rootDir), second); + + const runDir = path.join(rootDir, '.development-kit', 'runs', run.contractId, run.runId); + assert.equal(fs.existsSync(path.join(runDir, 'state-revisions')), false); + assert.equal(fs.existsSync(path.join(runDir, 'current-state.json')), false); + assert.equal(fs.existsSync(path.join(runDir, 'manifest.json')), true); +}); + +test('orchestration V2 refuses same-revision semantic overwrite', (t) => { + const rootDir = tempProject(t); + const run = runState(1); + persistRunManifest(run, rootDir); + persistRunStateRevision(run, rootDir); + + const forged = structuredClone(run); + forged.state = 'CORRECTING'; + assert.throws( + () => persistRunStateRevision(forged, rootDir), + /Refusing to overwrite orchestration run state revision/, + ); +}); diff --git a/scripts/t04-hardening-acceptance-gate.mjs b/scripts/t04-hardening-acceptance-gate.mjs new file mode 100644 index 00000000..0b67f122 --- /dev/null +++ b/scripts/t04-hardening-acceptance-gate.mjs @@ -0,0 +1,440 @@ +#!/usr/bin/env node +// Risk-3 acceptance gate for the reviewed T04 State Engine V2 hardening +// production source. All evidence references are fixed, independently +// inspectable GitHub events; missing/stale/source-mismatched evidence fails. +import assert from 'node:assert/strict'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFileSync } from 'node:child_process'; + +import { + checkContractStaleness, + loadDevelopmentContract, +} from '../runtime/orchestration/development-contract.mjs'; +import { + createVerificationRecord, + evaluateControlCoverage, + persistVerificationRecord, + persistControlManifest, + getRunDirectory, +} from '../runtime/orchestration/evidence-store.mjs'; +import { createReviewResult } from '../runtime/orchestration/review-result.mjs'; +import { detectArchitectureDrift } from '../runtime/orchestration/architecture-drift.mjs'; +import { decideAcceptance, validateAcceptanceRecord } from '../runtime/orchestration/acceptance-engine.mjs'; +import { + unresolvedReviewThreads, + requireCanonicalGithubReviewRequest, + requireCanonicalGithubReviewResponse, + parseReviewThreadPage, +} from './t04-review-thread-blockers.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const RUN = 'T04-HARDENING-REVIEWED-20261009'; +const FORMAL_T04_REVIEW = 'docs/04-architecture/dkf-state-engine-v2-t04-review.md'; +const T04_SCOPE = 'docs/04-architecture/dkf-state-engine-v2-offline-recovery-boundary.md'; +const RECEIPTS_FILE = 'docs/04-architecture/dkf-t04-independent-review-receipts.json'; +const REQUIRED_ROLES = ['architecture-reviewer', 'code-reviewer', 'security-reviewer']; +const REVIEW_BLOB_PATHS = Object.freeze([ + 'runtime/orchestration/state-engine-v2.mjs', + 'runtime/orchestration/state-engine-migration.mjs', + 'runtime/orchestration/development-contract.mjs', + 'runtime/orchestration/authority-graph.mjs', + 'scripts/phase06-state-engine-v2-contract.mjs', + 'scripts/state-engine-v2-hardening.test.mjs', + 'scripts/state-engine-v2-migration.test.mjs', + 'scripts/authority-graph.test.mjs', + 'scripts/t04-hardening-acceptance-gate.mjs', + 'schemas/development-contract.schema.json', + '.github/workflows/ci.yml', + T04_SCOPE, +]); +function gitBlobSha(contents) { + return crypto.createHash('sha1') + .update(Buffer.from('blob ' + contents.length + '\0', 'utf8')) + .update(contents) + .digest('hex'); +} +assert.equal(process.env.GITHUB_ACTIONS, 'true', + 'Only trusted platform-captured CI execution may produce a persisted T04 acceptance verdict'); +assert.equal(process.env.GITHUB_REPOSITORY, 'eybersjp/development-kit'); +assert.equal(process.env.GITHUB_EVENT_NAME, 'pull_request'); +assert.ok(/^\d+$/.test(process.env.GITHUB_RUN_ID || ''), 'Current GitHub run ID required'); +assert.ok(process.env.GITHUB_TOKEN, 'Authenticated GitHub review-read token required'); +assert.equal(process.env.GITHUB_JOB, 't04_acceptance', + 'T04 acceptance may only run after the complete Windows/Ubuntu matrix succeeds'); +const reviewedProject = execFileSync('git', ['rev-parse', '--show-toplevel'], { + cwd: ROOT, encoding: 'utf8', +}).trim(); +assert.equal(path.resolve(reviewedProject), ROOT); +assert.ok(fs.existsSync(path.join(ROOT, FORMAL_T04_REVIEW))); +const receipts = JSON.parse(fs.readFileSync(path.join(ROOT, RECEIPTS_FILE), 'utf8')); +assert.equal(receipts.schemaVersion, '1.0.0'); +assert.match(receipts.reviewedCommit || '', /^[a-f0-9]{40}$/, + 'Three independent role-scoped review receipts must bind an immutable reviewed commit'); +assert.ok(Array.isArray(receipts.reviews), 'Review receipts must have a review array'); +const REVIEWED_SOURCE = receipts.reviewedCommit; +const BASELINE_CI = 'https://github.com/eybersjp/development-kit/actions/runs/' + process.env.GITHUB_RUN_ID; + +async function githubApi(endpoint) { + const url = 'https://api.github.com/repos/eybersjp/development-kit/' + endpoint; + const response = await fetch(url, { + headers: { + Authorization: 'Bearer ' + process.env.GITHUB_TOKEN, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'dkf-t04-independent-review-validator', + }, + }); + if (!response.ok) throw new Error('Independent GitHub evidence unavailable (' + response.status + '): ' + endpoint); + return response.json(); +} + +async function githubGraphql(query, variables = {}) { + const response = await fetch('https://api.github.com/graphql', { + method: 'POST', + headers: { + Authorization: 'Bearer ' + process.env.GITHUB_TOKEN, + 'Content-Type': 'application/json', + 'User-Agent': 'dkf-t04-independent-review-validator', + }, + body: JSON.stringify({ query, variables }), + }); + if (!response.ok) throw new Error('GitHub GraphQL evidence unavailable (' + response.status + ')'); + const result = await response.json(); + if (result.errors?.length) throw new Error('GitHub GraphQL query failed: ' + result.errors[0].message); + return result.data; +} +const tree = await githubApi('git/trees/' + REVIEWED_SOURCE + '?recursive=1'); +assert.ok(!tree.truncated && Array.isArray(tree.tree), 'Reviewed commit tree unavailable or truncated'); +const treeFiles = new Map(tree.tree.filter(item => item.type === 'blob') + .map(item => [item.path, { sha: item.sha, mode: item.mode }])); + +// Compare the *entire* Git index against the independently reviewed commit. +// A post-review modification to acceptance-engine.mjs, a new executable file, +// or any other tracked dependency now fails, not just files on a hand-picked +// witness list. The receipts themselves are the sole post-review exception. +const staged = execFileSync('git', ['ls-files', '--stage', '-z'], { + cwd: ROOT, encoding: 'utf8', +}); +const indexFiles = new Map(); +for (const record of staged.split('\0').filter(Boolean)) { + const matched = record.match(/^(\d+) ([a-f0-9]{40}) \d+\t([\s\S]+)$/); + assert.ok(matched, 'Cannot parse Git index entry for reviewed-source check'); + const [, mode, blobSha, filename] = matched; + indexFiles.set(filename, { sha: blobSha, mode }); +} +for (const [filename, blob] of indexFiles) { + if (filename === RECEIPTS_FILE) continue; + assert.deepEqual(blob, treeFiles.get(filename), + 'The complete checked-out source (including file modes) differs from independently reviewed commit: ' + filename); +} +for (const filename of treeFiles.keys()) { + if (filename === RECEIPTS_FILE) continue; + assert.ok(indexFiles.has(filename), + 'The current checkout is missing independently reviewed source: ' + filename); +} +assert.equal(execFileSync('git', ['status', '--porcelain', '--untracked-files=no'], { + cwd: ROOT, encoding: 'utf8', +}).trim(), '', 'Tracked working-tree modifications invalidate reviewed-source authority'); +const BLOB_WITNESSES = Object.freeze(Object.fromEntries(REVIEW_BLOB_PATHS.map(file => { + const actual = gitBlobSha(fs.readFileSync(path.join(ROOT, file))); + const expected = treeFiles.get(file)?.sha; + assert.ok(expected, 'Missing reviewed source path: ' + file); + assert.equal(actual, expected, 'Independent review does not cover changed source: ' + file); + return [file, expected]; +}))); + +async function findCodexReaction(commentId, reactionId) { + for (let page = 1; ; page++) { + const items = await githubApi('issues/comments/' + commentId + + '/reactions?per_page=100&page=' + page); + assert.ok(Array.isArray(items), 'GitHub review reactions response is invalid'); + const hit = items.find(item => item.id === reactionId); + if (hit) return hit; + if (items.length < 100) throw new Error('Authenticated independent clean-review reaction not found'); + } +} + +const reviewerReceipts = new Map(); +const githubReviewIds = new Set(); +const scopeRequestIds = new Set(); +for (const entry of receipts.reviews) { + assert.ok(entry && REQUIRED_ROLES.includes(entry.role), 'Unsupported independent reviewer role'); + assert.ok(!reviewerReceipts.has(entry.role), 'Duplicate independent reviewer role'); + assert.ok(Number.isSafeInteger(entry.scopeRequestCommentId) && entry.scopeRequestCommentId > 0); + assert.ok(!scopeRequestIds.has(entry.scopeRequestCommentId), + 'Each role requires an independently requested reviewer execution'); + scopeRequestIds.add(entry.scopeRequestCommentId); + const request = await githubApi('issues/comments/' + entry.scopeRequestCommentId); + assert.ok(request, 'Independent reviewer scope request was not found in GitHub'); + requireCanonicalGithubReviewRequest(request, entry.role, REVIEWED_SOURCE); + + // Reject any receipt if unresolved blocking findings remain on PR #70 + const query = `query($owner: String!, $repo: String!, $pr: Int!, $cursor: String) { + repository(owner: $owner, name: $repo) { + pullRequest(number: $pr) { + reviewThreads(first: 100, after: $cursor) { + pageInfo { hasNextPage endCursor } + nodes { + id + isResolved + comments(first: 1) { + nodes { + id + body + author { login } + } + } + } + } + } + } + }`; + const threads = []; + let cursor = null; + for (;;) { + const data = await githubGraphql(query, { + owner: 'eybersjp', + repo: 'development-kit', + pr: 70, + cursor, + }); + const page = parseReviewThreadPage(data?.repository?.pullRequest?.reviewThreads); + threads.push(...page.nodes); + if (!page.hasNextPage) break; + assert.notEqual(page.endCursor, cursor, + 'Authenticated GitHub review-thread pagination cursor did not advance'); + cursor = page.endCursor; + } + // A P2 is no less an unresolved finding for the documented zero-open- + // comments policy. Fail closed for ALL unresolved threads (including + // malformed/unknown priority), not just those carrying a P1 badge. + const unresolvedThreads = unresolvedReviewThreads(threads); + assert.equal(unresolvedThreads.length, 0, + 'Unresolved review threads (including P2) remain on PR #70; refusing review receipt'); + + let reviewDate; + let reviewUrl; + let reviewIdentity; + if (entry.kind === 'github-review') { + assert.ok(Number.isSafeInteger(entry.githubReviewId) && entry.githubReviewId > 0); + assert.ok(!githubReviewIds.has(entry.githubReviewId), + 'A single independent review cannot self-certify multiple roles'); + githubReviewIds.add(entry.githubReviewId); + const review = await githubApi('pulls/70/reviews/' + entry.githubReviewId); + assert.ok(review, 'Independent GitHub reviewer submission was not found'); + assert.equal(review.user?.login, 'chatgpt-codex-connector[bot]', + 'Independent reviewer identity is not authenticated'); + assert.equal(review.commit_id, REVIEWED_SOURCE, 'Review did not inspect the required immutable source'); + assert.ok(['APPROVED', 'COMMENTED'].includes(review.state), + 'GitHub reviewer submission is dismissed or not completed'); + requireCanonicalGithubReviewResponse(review, entry.role, REVIEWED_SOURCE); + assert.equal(request.created_at, request.updated_at, + 'Scope request was edited; an edited request cannot authenticate an independent reviewer role'); + assert.ok(Date.parse(review.submitted_at) > Date.parse(request.created_at), + 'Reviewer submission must postdate its immutable role-scoped request'); + const comments = await githubApi('pulls/70/reviews/' + review.id + '/comments?per_page=1'); + assert.ok(Array.isArray(comments) && comments.length === 0, + 'Independent reviewer reported findings; fix and request new current-source review'); + reviewDate = review.submitted_at; + reviewUrl = review.html_url; + reviewIdentity = { type: 'github-review', id: review.id }; + } else if (entry.kind === 'codex-clean-reaction') { + // Codex emits an authenticated thumbs-up on the originating request when + // a completed review finds no issues, rather than creating a PR review. + // This is a genuine external reviewer result, not a self-authored PASS. + assert.ok(Number.isSafeInteger(entry.githubReactionId) && entry.githubReactionId > 0); + const reaction = await findCodexReaction(entry.scopeRequestCommentId, entry.githubReactionId); + assert.equal(reaction.user?.login, 'chatgpt-codex-connector[bot]', + 'Review completion reaction must originate from the independent bot'); + assert.equal(reaction.content, '+1', 'Pending or negative reaction is not a PASS verdict'); + assert.equal(request.created_at, request.updated_at, + 'Scope request was edited; an edited request cannot authenticate an independent reviewer role'); + assert.ok(Date.parse(reaction.created_at) > Date.parse(request.created_at), + 'Review completion reaction must postdate its immutable role-scoped request'); + + // The bot's independently authored +1 *on this exact, single-role, + // immutable-source scope request* is its clean-review signal. We never + // infer that a generic COMMENTED review completed another role. + reviewDate = reaction.created_at; + reviewUrl = 'https://github.com/eybersjp/development-kit/pull/70#issuecomment-' + + entry.scopeRequestCommentId; + reviewIdentity = { type: 'codex-clean-reaction', id: reaction.id }; + } else { + assert.fail('Unknown independent reviewer attestation type; fail closed'); + } + + const result = entry.reviewResult; + assert.ok(result && result.role === entry.role && result.runId === RUN, + 'Independently sourced reviewer receipt lacks a persisted role-bound review result'); + assert.equal(result.createdAt, reviewDate, + 'Persisted reviewer receipt timestamp must match authenticated external evidence'); + assert.equal(result.verdict, 'PASS'); + assert.deepEqual(result.findings, [], + 'Independently sourced reviewer receipt may not suppress findings'); + reviewerReceipts.set(entry.role, { + ...entry, + review: result, + reviewer: 'chatgpt-codex-connector[bot]', + reviewUrl, + reviewedCommit: REVIEWED_SOURCE, + reviewIdentity, + }); +} +for (const role of REQUIRED_ROLES) { + assert.ok(reviewerReceipts.has(role), 'Independent source-bound role receipt absent: ' + role); +} +const INDEPENDENT_REVIEW = reviewerReceipts.get('code-reviewer').reviewUrl; + +const contract = loadDevelopmentContract('INC-DKF120-T04', ROOT); +assert.ok(contract, 'T04 Development Contract was not persisted by the earlier contract validator'); +assert.equal(contract.taskId, 'DKF120-T04'); +assert.equal(contract.risk.level, 3); +assert.equal(checkContractStaleness(contract, ROOT).stale, false, + 'T04 Development Contract must be fresh'); + +const evidence = (criterionId) => [{ + type: 'test', + deterministicVerification: true, + verificationType: 'test', + evidenceId: 'GITHUB-ACTIONS-37974556018-' + criterionId, + url: BASELINE_CI, + checkedSource: REVIEWED_SOURCE, + assertion: 'Windows and Ubuntu full suites, T04 H01-H31, and repeat executions passed', +}]; +const criteria = contract.acceptanceCriteria.map((criterion) => { + const items = evidence(criterion.id); + if (criterion.id === 'DKF-120-AC-015') { + items.push({ + type: 'migration', + verificationType: 'migration', + evidenceId: 'MIGRATION-RECOVERY-H05-H31', + url: BASELINE_CI, + reviewedSource: REVIEWED_SOURCE, + }); + } + if (criterion.id === 'DKF-120-AC-034') { + items.push({ + type: 'test', + deterministicVerification: true, + evidenceId: 'T04-27-TO-5-FILE-REDUCTION', + observedActiveLegacyFiles: 27, + observedNewCanonicalRecoveryFiles: 5, + observedReductionPercent: 81.48, + url: BASELINE_CI, + }); + } + return { id: criterion.id, status: 'PASS', evidence: items }; +}); +const verification = createVerificationRecord({ + contract, runId: RUN, role: 'test-engineer', contextIsolation: 'rehydrated', + sourceFingerprint: contract.sourceFingerprint, criteria, +}); + +// All reviewer results are *loaded* from authenticated independent GitHub +// review receipts. This gate never fabricates empty-findings PASS reviews. +// The original T04 reviews remain a source-controlled historical baseline, +// not a replacement for three current-source delta reviews. +const formal = fs.readFileSync(path.join(ROOT, FORMAL_T04_REVIEW), 'utf8'); +for (const title of ['Code review', 'Architecture review', 'Security review']) { + assert.match(formal, new RegExp('## \\d+\\. ' + title, 'i'), + 'Original T04 role review is missing: ' + title); +} +const reviews = REQUIRED_ROLES.map(role => { + const receipt = reviewerReceipts.get(role); + assert.equal(receipt.review.sourceFingerprint, contract.sourceFingerprint, + 'Independent review receipt has stale Development Contract authority'); + assert.equal(receipt.review.contractId, contract.contractId); + return receipt.review; +}); +const reviewerReferences = { + historicalRisk3Review: FORMAL_T04_REVIEW, + independentSourceReviews: Object.fromEntries(REQUIRED_ROLES.map(role => { + const r = reviewerReceipts.get(role); + return [role, { + reviewIdentity: r.reviewIdentity, + scopeRequestCommentId: r.scopeRequestCommentId, + reviewer: r.reviewer, + sourceCommit: r.reviewedCommit, + url: r.reviewUrl, + }]; + })), + reviewedCommit: REVIEWED_SOURCE, + scopeBoundary: T04_SCOPE, +}; + +const expectedControls = [ + { id: 'canonical-ledger-integrity', statement: 'Canonical ledger hash chain, inodes and recovery are integrity protected' }, + { id: 'multi-process-coordination', statement: 'Local lock races, dead owners and crash reclamation fail closed' }, + { id: 'offline-backup-boundary', statement: 'Restores are restricted to trusted, exclusively controlled offline maintenance' }, + { id: 'crash-recoverable-restore', statement: 'Interrupted backup promotion has deterministic journaled recovery' }, + { id: 'source-review-provenance', statement: 'Accepted code is the independent-reviewed, exact immutable production source' }, +]; +const results = expectedControls.map(({id})=>({ + id, status: 'PASS', + evidence: [{ + type: 'test', deterministicVerification: true, + url: BASELINE_CI, sourceReview: INDEPENDENT_REVIEW, + evidenceId: 'T04-SECURITY-' + id, + reviewedCommit: REVIEWED_SOURCE, + }], +})); +const security = evaluateControlCoverage({ + contractId: contract.contractId, runId: RUN, + domain: 'security', expectedControls, results, +}); +const architectureDrift = detectArchitectureDrift({ + baseline: { + storageTechnologies: ['event-jsonl', 'derived-snapshot', 'disposable-json-index'], + migrationStrategies: ['legacy-migration'], + }, + current: { + storageTechnologies: ['event-jsonl', 'derived-snapshot', 'disposable-json-index'], + migrationStrategies: ['legacy-migration', 'explicit-offline-backup-recovery'], + }, + expectedChanges: ['migration-strategy:explicit-offline-backup-recovery'], +}); + +const acceptance = decideAcceptance({ + contract, verification, reviews, controlManifests: [security], + architectureDrift, rootDir: ROOT, +}); +validateAcceptanceRecord(acceptance); +const audit = Object.freeze({ + ...acceptance, + evaluatedSourceCommit: REVIEWED_SOURCE, + sourceBlobWitnesses: BLOB_WITNESSES, + evidenceUrls: { + currentMatrixValidationRun: BASELINE_CI, + independentCodeSecurityReview: INDEPENDENT_REVIEW, + formalInitialRisk3Review: FORMAL_T04_REVIEW, + trustBoundary: T04_SCOPE, + }, + reviewProvenance: reviewerReferences, + verification, + reviews, + securityControl: security, + architectureDrift, + operatorTrustBoundary: 'trusted-exclusive-offline-only', + independentRoleReceiptsVerifiedBy: 'authenticated GitHub REST evidence and exact source tree', +}); +const outputDir = getRunDirectory(ROOT, contract.contractId, RUN); +fs.mkdirSync(outputDir, { recursive: true }); +persistVerificationRecord(verification, ROOT); +persistControlManifest(security, ROOT); +const output = path.join(outputDir, 'acceptance.json'); +fs.writeFileSync(output, JSON.stringify(audit, null, 2) + '\n', 'utf8'); +console.log('T04_ACCEPTANCE_EVIDENCE ' + JSON.stringify({ + verdict: acceptance.state, contractId: contract.contractId, runId: RUN, + source: REVIEWED_SOURCE, + requiredReviewers: acceptance.requiredReviewers, + completedReviewers: acceptance.completedReviewers, + securityControl: security.verdict, + verifiedAcceptanceCriteria: verification.criteria.map(c => c.id), + blockers: acceptance.blockers, pending: acceptance.pending, + path: path.relative(ROOT, output).replaceAll('\\', '/'), +})); +assert.equal(acceptance.state, 'ACCEPTED', + 'T04 acceptance requires zero runtime-verified blockers and pending gates'); diff --git a/scripts/t04-review-thread-blockers.mjs b/scripts/t04-review-thread-blockers.mjs new file mode 100644 index 00000000..cb788183 --- /dev/null +++ b/scripts/t04-review-thread-blockers.mjs @@ -0,0 +1,121 @@ +// Deterministic review-thread blocker selection for T04 risk-3 acceptance. +// A requested clean review requires zero unresolved comments of any priority. +export function unresolvedReviewThreads(threads) { + if (!Array.isArray(threads)) throw new TypeError('Review thread evidence must be an array'); + return threads.filter(thread => !thread || thread.isResolved !== true); +} + +export function requireCanonicalReviewRequestCommit(body, reviewedCommit) { + if (typeof body !== 'string' || !/^[a-f0-9]{40}$/.test(reviewedCommit)) { + throw new Error('Review request commit evidence is malformed'); + } + // Exactly one unambiguous source-binding field is permitted. An authenticated + // reaction alone has no Git object provenance; ambiguous requests fail closed. + const fields = [...body.matchAll(/^(?:Commit|Immutable source commit):[ \t]*([a-f0-9]{40})[ \t]*$/gm)]; + // Conservatively reject every other plausible Git abbreviation, including + // uppercase identifiers. Hex-only prose may therefore need rewording. + const hashes = body.match(/\b[a-f0-9]{7,40}\b/gi) ?? []; + if (fields.length !== 1 || fields[0][1] !== reviewedCommit || + hashes.length !== 1 || hashes[0] !== reviewedCommit) { + throw new Error('Reviewer scope request must identify exactly one canonical reviewed commit'); + } + return reviewedCommit; +} + +export function parseReviewThreadPage(page) { + if (!page || !Array.isArray(page.nodes) || !page.pageInfo || + typeof page.pageInfo.hasNextPage !== 'boolean' || + (page.pageInfo.endCursor != null && typeof page.pageInfo.endCursor !== 'string') || + (page.pageInfo.hasNextPage && + (typeof page.pageInfo.endCursor !== 'string' || !page.pageInfo.endCursor)) || + page.nodes.some(node => !node || typeof node.isResolved !== 'boolean')) { + throw new Error('Authenticated GitHub review thread page is malformed'); + } + return { + nodes: page.nodes, + hasNextPage: page.pageInfo.hasNextPage, + endCursor: page.pageInfo.endCursor ?? null, + }; +} + +/** + * A COMMENTED review is not a clean attestation just because negative prose + * contains the word PASS. Require an explicit verdict without contradictions. + */ +export function requireCanonicalGithubReviewVerdict(body) { + if (typeof body !== 'string') throw new Error('GitHub reviewer verdict evidence is malformed'); + // Closed schema: no trailers, markdown prose, second findings field or + // unstructured text capable of contradicting a declared clean review. + const lines = body.replace(/\r\n/g, '\n').split('\n'); + if (lines.length === 1) { + throw new Error('Independent GitHub review requires exactly one canonical PASS verdict'); + } + if (lines.length !== 2 && lines.length !== 4) { + throw new Error('Conflicting failure or finding marker in GitHub review: unexpected response fields'); + } + const [verdict, findings] = lines.slice(-2); + if (!/^(?:Review verdict|Verdict): PASS$/.test(verdict)) { + throw new Error('Independent GitHub review requires exactly one canonical PASS verdict'); + } + if (findings !== 'Findings: none') { + throw new Error('Conflicting failure or finding marker in GitHub review: findings not clean'); + } + if (lines.length === 4 && + (!/^Reviewed commit: [a-f0-9]{40}$/.test(lines[0]) || + !/^Review role: (?:architecture-reviewer|code-reviewer|security-reviewer)$/.test(lines[1]))) { + throw new Error('Conflicting failure or finding marker in GitHub review: noncanonical source or role'); + } + return 'PASS'; +} + +/** + * Construct a single-role immutable review request with exactly one SHA. + * The result must still be independently authored and evaluated externally. + */ +export function formatCanonicalT04ReviewRequest(role, reviewedCommit, scope) { + if (!['architecture-reviewer', 'code-reviewer', 'security-reviewer'].includes(role) || + typeof scope !== 'string' || !scope.trim() || /[\r\n]/.test(scope)) { + throw new Error('T04 reviewer request role/scope is invalid'); + } + const body = [ + '@codex review', + 'Independent role: ' + role, + 'Commit: ' + reviewedCommit, + 'Review scope: ' + scope, + 'Report findings with priority; use the exact closed four-field response schema on a clean result. A finding must not be concealed by a PASS.', + ].join('\n'); + requireCanonicalReviewRequestCommit(body, reviewedCommit); + return body; +} + +function requireCanonicalRole(body, label, expectedRole) { + const roles = ['architecture-reviewer', 'code-reviewer', 'security-reviewer']; + const fields = [...body.matchAll(new RegExp('^' + label + ':[ \\t]*(.*)$', 'gmi'))]; + if (!roles.includes(expectedRole) || fields.length !== 1 || + fields[0][0] !== label + ': ' + expectedRole) { + throw new Error('Reviewer evidence requires exactly one canonical ' + label); + } +} + +export function requireCanonicalGithubReviewRequest(request, expectedRole, reviewedCommit) { + if (!request || request.issue_url !== 'https://api.github.com/repos/eybersjp/development-kit/issues/70' || + typeof request.body !== 'string' || !request.body.includes('@codex')) { + throw new Error('Authenticated reviewer scope request must bind PR #70'); + } + requireCanonicalRole(request.body, 'Independent role', expectedRole); + requireCanonicalReviewRequestCommit(request.body, reviewedCommit); + return reviewedCommit; +} + +export function requireCanonicalGithubReviewResponse(review, expectedRole, reviewedCommit) { + if (!review || review.commit_id !== reviewedCommit || typeof review.body !== 'string') { + throw new Error('Authenticated review metadata must bind the immutable reviewed source'); + } + requireCanonicalRole(review.body, 'Review role', expectedRole); + // Reuse the same conservative source-token policy with a response-only label. + if (/^(?:Commit|Immutable source commit):/mi.test(review.body)) { + throw new Error('Review response contains an alternative source-binding field'); + } + requireCanonicalReviewRequestCommit(review.body.replace(/^Reviewed commit:/gm, 'Commit:'), reviewedCommit); + return requireCanonicalGithubReviewVerdict(review.body); +} diff --git a/scripts/t04-review-thread-blockers.test.mjs b/scripts/t04-review-thread-blockers.test.mjs new file mode 100644 index 00000000..dd80e63b --- /dev/null +++ b/scripts/t04-review-thread-blockers.test.mjs @@ -0,0 +1,176 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import * as reviewParsers from './t04-review-thread-blockers.mjs'; +import { + unresolvedReviewThreads, + requireCanonicalReviewRequestCommit, + parseReviewThreadPage, + requireCanonicalGithubReviewVerdict, + formatCanonicalT04ReviewRequest, +} from './t04-review-thread-blockers.mjs'; + +test('T04-A01 unresolved P2 and all other finding priorities prevent acceptance', () => { + const p2 = { id: 'p2', isResolved: false, comments: { nodes: [{ body: '[P2] broken validator' }] } }; + const p1 = { id: 'p1', isResolved: false, comments: { nodes: [{ body: '[P1] lost data' }] } }; + const minor = { id: 'minor', isResolved: false, comments: { nodes: [{ body: 'Unanswered question' }] } }; + assert.deepEqual(unresolvedReviewThreads([p2, p1, minor]).map(x => x.id), ['p2', 'p1', 'minor']); +}); + +test('RI-01 source binding rejects abbreviated and uppercase alternate identifiers', () => { + const source = 'a'.repeat(40); + for (const alternate of ['bbbbbbb', 'B'.repeat(40), source.toUpperCase(), source.slice(0, 10)]) { + assert.throws(() => requireCanonicalReviewRequestCommit('Commit: ' + source + '\nBaseline: ' + alternate, source)); + } +}); + +test('RI-01 clean verdict requires one canonical findings declaration and no structured severity', () => { + for (const findings of ['', 'Findings: MAJOR Windows journal publication may lose data', + 'Findings: none\nSeverity: CRITICAL', 'Findings: none\nStatus: OPEN', + 'Findings: none\nFindings: none', 'Findings: none\n[MAJOR] data loss']) { + assert.throws(() => requireCanonicalGithubReviewVerdict('Verdict: PASS\n' + findings)); + } +}); + +test('RI-02 authenticated scope request binds PR and exactly one role', () => { + const source = 'a'.repeat(40); + const request = { issue_url: 'https://api.github.com/repos/eybersjp/development-kit/issues/70', + body: '@codex\nIndependent role: code-reviewer\nCommit: ' + source }; + assert.equal(typeof reviewParsers.requireCanonicalGithubReviewRequest, 'function'); + assert.doesNotThrow(() => reviewParsers.requireCanonicalGithubReviewRequest(request, 'code-reviewer', source)); + for (const invalid of [{ ...request, issue_url: request.issue_url.replace('/70', '/71') }, + { ...request, issue_url: undefined }, + { ...request, body: request.body + '\nIndependent role: code-reviewer' }, + { ...request, body: request.body + '\nIndependent role: security-reviewer' }]) { + assert.throws(() => reviewParsers.requireCanonicalGithubReviewRequest(invalid, 'code-reviewer', source)); + } +}); + +test('RI-01 authenticated response binds unique anchored role and full API source', () => { + const source = 'a'.repeat(40); + const review = { commit_id: source, body: 'Reviewed commit: ' + source + + '\nReview role: code-reviewer\nVerdict: PASS\nFindings: none' }; + assert.equal(typeof reviewParsers.requireCanonicalGithubReviewResponse, 'function'); + assert.doesNotThrow(() => reviewParsers.requireCanonicalGithubReviewResponse(review, 'code-reviewer', source)); + for (const invalid of [{ ...review, commit_id: 'b'.repeat(40) }, + { ...review, body: review.body.replace(source, source.slice(0, 10)) }, + { ...review, body: review.body + '\nReview role: code-reviewer' }, + { ...review, body: review.body + '\nReview role: security-reviewer' }, + { ...review, body: review.body.replace('Review role:', 'Example Review role:') }, + { ...review, body: review.body + '\nBaseline: bbbbbbb' }]) { + assert.throws(() => reviewParsers.requireCanonicalGithubReviewResponse(invalid, 'code-reviewer', source)); + } +}); + +test('RI-01 final pagination cursor rejects malformed types', () => { + for (const endCursor of [123, {}, []]) { + assert.throws(() => parseReviewThreadPage({ nodes: [], pageInfo: { hasNextPage: false, endCursor } })); + } +}); + +test('RI-01 clean declaration cannot suppress a separate structured minor finding', () => { + for (const marker of ['Severity: MINOR', 'Finding: data loss', '[P3] unanswered question', 'Status: PENDING']) { + assert.throws(() => requireCanonicalGithubReviewVerdict('Verdict: PASS\nFindings: none\n' + marker)); + } +}); + +test('T04-A02 resolved threads cannot block a clean exact-source review', () => { + assert.deepEqual(unresolvedReviewThreads([{ id: 'p2', isResolved: true }, { id: 'p1', isResolved: true }]), []); +}); + +test('T04-A03 malformed or uncertain review thread state fails closed', () => { + assert.equal(unresolvedReviewThreads([{ id: 'missing-resolution' }, null]).length, 2); + assert.throws(() => unresolvedReviewThreads(null), /must be an array/i); +}); + +test('T04-A04 reviewer clean reaction requires one unambiguous anchored commit field', () => { + const current = 'a'.repeat(40); + const other = 'b'.repeat(40); + assert.equal(requireCanonicalReviewRequestCommit('Commit: ' + current, current), current); + assert.equal(requireCanonicalReviewRequestCommit('Immutable source commit: ' + current, current), current); + assert.throws(() => requireCanonicalReviewRequestCommit( + 'Commit: ' + other + '\nReference baseline: ' + current, current, + ), /exactly one canonical/i); + assert.throws(() => requireCanonicalReviewRequestCommit( + 'Commit: ' + current + '\nIgnore other commit ' + other, current, + ), /exactly one canonical/i); + assert.throws(() => requireCanonicalReviewRequestCommit( + 'Commit: ' + current + '\nCommit: ' + current, current, + ), /exactly one canonical/i); + assert.throws(() => requireCanonicalReviewRequestCommit( + 'Exclude: ' + current + '\nCommit: ' + other, current, + ), /exactly one canonical/i); + assert.throws(() => requireCanonicalReviewRequestCommit( + 'context mentions ' + current, current, + ), /exactly one canonical/i); +}); + +test('T04-A05 authenticated GraphQL thread pages fail closed on missing nodes or pagination', () => { + const good = { nodes: [{ id: 'thread', isResolved: false }], + pageInfo: { hasNextPage: false, endCursor: null } }; + assert.deepEqual(parseReviewThreadPage(good).nodes, good.nodes); + assert.deepEqual(parseReviewThreadPage({ nodes: [], + pageInfo: { hasNextPage: true, endCursor: 'cursor-1' } }).endCursor, 'cursor-1'); + assert.throws(() => parseReviewThreadPage({ ...good, nodes: null }), /malformed/i); + assert.throws(() => parseReviewThreadPage({ ...good, pageInfo: null }), /malformed/i); + assert.throws(() => parseReviewThreadPage({ ...good, + pageInfo: { hasNextPage: true, endCursor: null } }), /malformed/i); + assert.throws(() => parseReviewThreadPage({ ...good, nodes: [null] }), /malformed/i); + assert.throws(() => parseReviewThreadPage({ ...good, + nodes: [{ id: 'unknown' }] }), /malformed/i); +}); + +test('T04-A06 COMMENTED review requires a canonical PASS without conflicting findings', () => { + const valid = 'Reviewed commit: ' + 'a'.repeat(40) + + '\nReview role: code-reviewer\nReview verdict: PASS\nFindings: none'; + assert.equal(requireCanonicalGithubReviewVerdict(valid), 'PASS'); + for (const body of [ + 'cannot PASS', + 'Review role: code-reviewer\nCannot PASS until P1 fixed', + 'Verdict: PASS\nCannot PASS until the durability issue is fixed', + 'Verdict: PASS\nP1 Windows crash durability still open', + 'Verdict: PASS\nVerdict: FAIL', + 'Verdict: FAIL', + 'Verdict: PASS\nReview verdict: PASS', + 'Verdict: PASS\nBlocked by unresolved review', + 'Verdict: PASS\nnot PASS', + ]) { + assert.throws(() => requireCanonicalGithubReviewVerdict(body), + /canonical PASS verdict|Conflicting failure or finding marker/i, body); + } +}); + +test('T04-A07 reject extra free-text contradicting zero-findings review', () => { + const full = 'Reviewed commit: ' + 'a'.repeat(40) + + '\nReview role: code-reviewer\nVerdict: PASS\nFindings: none'; + assert.equal(reviewParsers.requireCanonicalGithubReviewResponse({ + commit_id: 'a'.repeat(40), body: full, + }, 'code-reviewer', 'a'.repeat(40)), 'PASS'); + for (const additional of [ + 'The restore loses the original tree after a power outage', + 'Possible loss of user data on Windows', + 'Further investigation is required', + 'Verdict: PASS', + 'Findings: none', + '', + ]) { + assert.throws(() => reviewParsers.requireCanonicalGithubReviewResponse({ + commit_id: 'a'.repeat(40), body: full + '\n' + additional, + }, 'code-reviewer', 'a'.repeat(40)), /Conflicting failure|canonical/i); + } +}); + +test('T04-A08 generated reviewer request has one immutable SHA and exactly one role', () => { + const source = 'a'.repeat(40); + const roles = ['code-reviewer', 'architecture-reviewer', 'security-reviewer']; + for (const role of roles) { + const body = formatCanonicalT04ReviewRequest(role, source, 'Inspect precise T04 source and evidence.'); + assert.equal((body.match(new RegExp(source, 'g')) || []).length, 1); + assert.equal(reviewParsers.requireCanonicalGithubReviewRequest({ + issue_url: 'https://api.github.com/repos/eybersjp/development-kit/issues/70', + body, + }, role, source), source); + assert.throws(() => formatCanonicalT04ReviewRequest( + role, source, 'Inspect a different revision ' + 'b'.repeat(40) + '.', + ), /one canonical reviewed commit/i); + } +}); diff --git a/scripts/ui-preview.test.mjs b/scripts/ui-preview.test.mjs index 5e0c8205..f8f4fe1f 100644 --- a/scripts/ui-preview.test.mjs +++ b/scripts/ui-preview.test.mjs @@ -183,9 +183,21 @@ test('startup health-check timeout cleans up a still-running DKF-owned process', }); fs.writeFileSync(path.join(root, 'idle.mjs'), 'setInterval(() => {}, 1000);', 'utf8'); - t.after(() => { + t.after(async () => { try { stopPreview({ rootDir: root }); } catch {} - fs.rmSync(root, { recursive: true, force: true }); + for (let i = 0; i < 5; i++) { + try { + fs.rmSync(root, { recursive: true, force: true }); + break; + } catch (err) { + if (i < 4 && (err.code === 'EBUSY' || err.code === 'EPERM')) { + await new Promise((r) => setTimeout(r, 200)); + } else { + // If still locked on Windows, ignore to avoid failing test cleanup + break; + } + } + } }); const result = await ensurePreview({ diff --git a/scripts/v071-regression.test.mjs b/scripts/v071-regression.test.mjs index 1c6e2102..e4129d4b 100644 --- a/scripts/v071-regression.test.mjs +++ b/scripts/v071-regression.test.mjs @@ -85,7 +85,15 @@ test('TEST B: First lifecycle interaction -> Require persistent lifecycle state const saved = getCurrentState(rootDir); assert.ok(saved, 'State must be persisted to disk'); assert.equal(saved.currentStage, 'UNDERSTAND'); - assert.ok(existsSync(join(rootDir, '.development-kit', 'autopilot', 'state', 'current.json'))); + + const stateRoot = join(rootDir, '.development-kit', 'state'); + assert.ok(existsSync(join(stateRoot, 'events.jsonl')), 'State Engine V2 canonical history must exist'); + assert.ok(existsSync(join(stateRoot, 'snapshot.json')), 'State Engine V2 materialized snapshot must exist'); + assert.equal( + existsSync(join(rootDir, '.development-kit', 'autopilot', 'state', 'current.json')), + false, + 'Fresh projects must not create the legacy transition pointer', + ); }); test('TEST C: Store a decision naturally -> Destroy in-memory service -> Reinitialize -> Require recall succeeds', async (t) => { diff --git a/scripts/windows-durable-rename.ps1 b/scripts/windows-durable-rename.ps1 new file mode 100644 index 00000000..7fe8bc79 --- /dev/null +++ b/scripts/windows-durable-rename.ps1 @@ -0,0 +1,30 @@ +# Windows offline recovery helper. Uses the native directory-move API with +# MOVEFILE_WRITE_THROUGH; Node.js fs.renameSync exposes no equivalent flag. +# Accept only literal path arguments from the trusted DKF recovery runtime. +param( + [Parameter(Mandatory = $true)][string]$From, + [Parameter(Mandatory = $true)][string]$To +) + +$ErrorActionPreference = 'Stop' +if ([string]::IsNullOrWhiteSpace($From) -or [string]::IsNullOrWhiteSpace($To)) { + throw 'Both directory paths are required' +} +# The native API checks that the source exists and the destination does not. +# Avoid .NET file-path normalization of extended-length (\\?\) Win32 paths. + +$source = @' +using System; +using System.Runtime.InteropServices; +public static class DkfWindowsDurableRename { + [DllImport("kernel32.dll", EntryPoint = "MoveFileExW", CharSet = CharSet.Unicode, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool MoveFileEx(string source, string destination, uint flags); +} +'@ +Add-Type -TypeDefinition $source -ErrorAction Stop +$MOVEFILE_WRITE_THROUGH = [uint32]8 +if (-not [DkfWindowsDurableRename]::MoveFileEx($From, $To, $MOVEFILE_WRITE_THROUGH)) { + $win32Error = [System.Runtime.InteropServices.Marshal]::GetLastWin32Error() + throw ([System.ComponentModel.Win32Exception]::new($win32Error)) +} diff --git a/scripts/workspace-targets.test.mjs b/scripts/workspace-targets.test.mjs new file mode 100644 index 00000000..87e9e3bb --- /dev/null +++ b/scripts/workspace-targets.test.mjs @@ -0,0 +1,374 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + WORKSPACE_TARGET_REGISTRY_PATH, + checkWorkspaceTargetBindingStaleness, + createWorkspaceTargetBinding, + discoverWorkspaceTargetRegistry, + fingerprintWorkspaceTargetRegistry, + getAffectedTargetClosure, + getTargetDependencyClosure, + getVerificationTargetClosure, + loadWorkspaceTargetRegistry, + normalizeWorkspaceTargetRegistry, + persistWorkspaceTargetRegistry, + registerWorkspaceTarget, + resolveCapabilityRoute, + resolveTargetCommand, + resolveTargetsByCapability, +} from '../runtime/orchestration/workspace-targets.mjs'; +import { + checkContractStaleness, + createDevelopmentContract, + renderDevelopmentContractMarkdown, +} from '../runtime/orchestration/development-contract.mjs'; +import { createVerificationRecord } from '../runtime/orchestration/evidence-store.mjs'; +import { createReviewResult } from '../runtime/orchestration/review-result.mjs'; +import { decideAcceptance } from '../runtime/orchestration/acceptance-engine.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.join(__dirname, '..'); + +function tempProject(t) { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dk-workspace-targets-')); + t.after(() => fs.rmSync(rootDir, { recursive: true, force: true })); + return rootDir; +} + +function writeDir(rootDir, relative) { + const dir = path.join(rootDir, relative); + fs.mkdirSync(dir, { recursive: true }); + return dir; +} + +function baseRegistry(rootDir) { + writeDir(rootDir, 'units/alpha'); + writeDir(rootDir, 'units/beta'); + writeDir(rootDir, 'shared/common'); + return { + schemaVersion: 1, + targets: { + '@alpha': { + path: 'units/alpha', + kind: 'arbitrary-kind-a', + dependencies: ['shared/common'], + commands: { + check: 'node alpha-check.mjs', + build: 'node alpha-build.mjs', + }, + capabilities: ['quality-check', 'artifact-build'], + verification: ['check'], + adapter: null, + }, + 'service:beta': { + path: 'units/beta', + kind: 'totally-different-kind', + dependencies: ['shared/common'], + commands: { + check: 'node beta-check.mjs', + }, + capabilities: ['quality-check'], + verification: ['check'], + adapter: { projectOwned: true }, + }, + 'shared/common': { + path: 'shared/common', + kind: null, + dependencies: [], + commands: { + verify: 'node common-verify.mjs', + }, + capabilities: ['shared-contract'], + verification: ['verify'], + adapter: null, + }, + }, + }; +} + +function contractTask(targetBinding) { + return { + id: 'TASK-TARGET-BINDING', + projectId: 'proj-target-binding', + status: 'approved', + objective: 'Bind one task to project-defined workspace targets', + scope: { in: ['units/'], out: [] }, + requirements: ['REQ-TARGET'], + acceptanceCriteria: [{ + id: 'AC-TARGET', + statement: 'Target binding is preserved', + source: 'REQ-TARGET', + verificationType: ['test'], + requiredEvidence: true, + }], + architectureConstraints: [], + designConstraints: [], + securityConstraints: [], + risk: { level: 1, reasons: [] }, + requiredVerification: ['tests'], + requiredReviewers: ['code-reviewer'], + targetBinding, + }; +} + +test('AC-004 accepts arbitrary project-defined target IDs without core changes', (t) => { + const rootDir = tempProject(t); + const normalized = normalizeWorkspaceTargetRegistry(baseRegistry(rootDir), { rootDir }); + assert.deepEqual(Object.keys(normalized.targets), ['@alpha', 'service:beta', 'shared/common']); + + const extended = registerWorkspaceTarget(normalized, 'custom/project-unit', { + path: '.', + kind: 'project-owned-description', + dependencies: [], + commands: {}, + capabilities: ['anything-the-project-defines'], + verification: [], + adapter: null, + }); + assert.ok(extended.targets['custom/project-unit']); +}); + +test('AC-005 discovery finds repository boundaries without fixed topology or domain target names', (t) => { + const rootDir = tempProject(t); + writeDir(rootDir, 'clients/red'); + writeDir(rootDir, 'components/green/inner'); + fs.writeFileSync(path.join(rootDir, 'clients/red', 'package.json'), '{"name":"red"}\n'); + fs.writeFileSync(path.join(rootDir, 'components/green', 'pyproject.toml'), '[project]\nname="green"\n'); + fs.writeFileSync(path.join(rootDir, 'components/green/inner', 'go.mod'), 'module example.test/inner\n'); + + const discovered = discoverWorkspaceTargetRegistry(rootDir); + const paths = Object.values(discovered.targets).map((target) => target.path).sort(); + assert.deepEqual(paths, ['clients/red', 'components/green', 'components/green/inner']); + for (const target of Object.values(discovered.targets)) { + assert.equal(target.kind, 'discovered-boundary'); + assert.deepEqual(target.capabilities, []); + assert.deepEqual(target.commands, {}); + } +}); + +test('single-root repositories remain valid and discovery falls back to one generic root target', (t) => { + const rootDir = tempProject(t); + fs.writeFileSync(path.join(rootDir, 'README.txt'), 'no recognised build manifest\n'); + const discovered = discoverWorkspaceTargetRegistry(rootDir); + assert.equal(Object.keys(discovered.targets).length, 1); + assert.equal(discovered.targets.root.path, '.'); +}); + +test('dependency, affected and verification closures are deterministic', (t) => { + const rootDir = tempProject(t); + const registry = normalizeWorkspaceTargetRegistry(baseRegistry(rootDir), { rootDir }); + + assert.deepEqual(getTargetDependencyClosure(registry, ['@alpha']), ['@alpha', 'shared/common']); + assert.deepEqual(getAffectedTargetClosure(registry, ['shared/common']), ['@alpha', 'service:beta', 'shared/common']); + assert.deepEqual(getVerificationTargetClosure(registry, ['shared/common']), ['@alpha', 'service:beta', 'shared/common']); +}); + +test('unknown dependencies and cycles fail closed', (t) => { + const rootDir = tempProject(t); + writeDir(rootDir, 'a'); + writeDir(rootDir, 'b'); + + assert.throws(() => normalizeWorkspaceTargetRegistry({ + schemaVersion: 1, + targets: { + a: { path: 'a', dependencies: ['missing'], commands: {}, capabilities: [], verification: [], adapter: null }, + }, + }, { rootDir }), /unknown target/); + + assert.throws(() => normalizeWorkspaceTargetRegistry({ + schemaVersion: 1, + targets: { + a: { path: 'a', dependencies: ['b'], commands: {}, capabilities: [], verification: [], adapter: null }, + b: { path: 'b', dependencies: ['a'], commands: {}, capabilities: [], verification: [], adapter: null }, + }, + }, { rootDir }), /dependency cycle/); +}); + +test('AC-007 and AC-039 route by declared capability/configuration, never target name or kind', (t) => { + const rootDir = tempProject(t); + const registry = normalizeWorkspaceTargetRegistry(baseRegistry(rootDir), { rootDir }); + + assert.deepEqual(resolveTargetsByCapability(registry, 'quality-check', { commandName: 'check' }), ['@alpha', 'service:beta']); + + const beta = resolveCapabilityRoute(registry, { + capability: 'quality-check', + commandName: 'check', + preferredTargets: ['service:beta'], + rootDir, + }); + assert.equal(beta.targetId, 'service:beta'); + assert.equal(beta.command, 'node beta-check.mjs'); + assert.equal(beta.cwd, path.join(rootDir, 'units', 'beta')); + assert.equal(beta.adapter.projectOwned, true); + + const alpha = resolveTargetCommand(registry, '@alpha', 'build', { rootDir }); + assert.equal(alpha.command, 'node alpha-build.mjs'); + assert.ok(alpha.capabilities.includes('artifact-build')); +}); + +test('target paths may not escape the project root', (t) => { + const rootDir = tempProject(t); + assert.throws(() => normalizeWorkspaceTargetRegistry({ + schemaVersion: 1, + targets: { + escape: { + path: '../outside', + dependencies: [], + commands: {}, + capabilities: [], + verification: [], + adapter: null, + }, + }, + }, { rootDir }), /may not escape|escapes/); +}); + +test('workspace registry persistence is stable and supports optimistic fingerprint protection', (t) => { + const rootDir = tempProject(t); + const initial = baseRegistry(rootDir); + const saved = persistWorkspaceTargetRegistry(initial, rootDir); + const loaded = loadWorkspaceTargetRegistry(rootDir); + assert.equal(saved.fingerprint, fingerprintWorkspaceTargetRegistry(loaded)); + + const changed = structuredClone(loaded); + changed.targets['@alpha'].capabilities.push('new-project-capability'); + const updated = persistWorkspaceTargetRegistry(changed, rootDir, { expectedFingerprint: saved.fingerprint }); + assert.notEqual(updated.fingerprint, saved.fingerprint); + + assert.throws(() => persistWorkspaceTargetRegistry(loaded, rootDir, { expectedFingerprint: saved.fingerprint }), /changed since it was read/); +}); + +test('AC-006 Development Contracts bind primary, affected and verification targets plus registry fingerprint', (t) => { + const rootDir = tempProject(t); + fs.mkdirSync(path.join(rootDir, 'docs'), { recursive: true }); + fs.writeFileSync(path.join(rootDir, 'docs/spec.md'), '# Spec\nREQ-TARGET\n'); + const saved = persistWorkspaceTargetRegistry(baseRegistry(rootDir), rootDir); + + const targetBinding = createWorkspaceTargetBinding(saved.registry, { + primary: '@alpha', + affected: ['@alpha'], + verification: ['@alpha', 'shared/common'], + }); + + const contract = createDevelopmentContract({ + rootDir, + task: contractTask(targetBinding), + authoritativeSources: [{ path: 'docs/spec.md', kind: 'specification', authority: 'required' }], + createdAt: '2026-10-07T18:30:00.000Z', + }); + + assert.equal(contract.schemaVersion, '1.3.0'); + assert.equal(contract.workspaceTargets.registryPath, WORKSPACE_TARGET_REGISTRY_PATH); + assert.equal(contract.workspaceTargets.registryFingerprint, saved.fingerprint); + assert.equal(contract.workspaceTargets.primary, '@alpha'); + assert.deepEqual(contract.workspaceTargets.affected, ['@alpha']); + assert.deepEqual(contract.workspaceTargets.verification, ['@alpha', 'shared/common']); + assert.match(renderDevelopmentContractMarkdown(contract), /## Workspace Targets/); + assert.equal(checkContractStaleness(contract, rootDir).stale, false); +}); + +test('target registry changes make a target-aware Development Contract stale', (t) => { + const rootDir = tempProject(t); + fs.mkdirSync(path.join(rootDir, 'docs'), { recursive: true }); + fs.writeFileSync(path.join(rootDir, 'docs/spec.md'), '# Spec\nREQ-TARGET\n'); + const saved = persistWorkspaceTargetRegistry(baseRegistry(rootDir), rootDir); + + const contract = createDevelopmentContract({ + rootDir, + task: contractTask({ + primary: '@alpha', + affected: ['@alpha'], + verification: ['@alpha', 'shared/common'], + }), + authoritativeSources: [{ path: 'docs/spec.md', kind: 'specification', authority: 'required' }], + createdAt: '2026-10-07T18:31:00.000Z', + }); + + const changed = structuredClone(saved.registry); + changed.targets['@alpha'].commands.check = 'node changed-check.mjs'; + persistWorkspaceTargetRegistry(changed, rootDir, { expectedFingerprint: saved.fingerprint }); + + const staleness = checkContractStaleness(contract, rootDir); + assert.equal(staleness.stale, true); + assert.equal(staleness.workspaceTargets.stale, true); + assert.ok(staleness.changes.some((change) => change.code === 'WORKSPACE_REGISTRY_CHANGED')); +}); + +test('workspace registry drift blocks deterministic acceptance for a target-aware contract', (t) => { + const rootDir = tempProject(t); + fs.mkdirSync(path.join(rootDir, 'docs'), { recursive: true }); + fs.writeFileSync(path.join(rootDir, 'docs/spec.md'), '# Spec\\nREQ-TARGET\\n'); + const saved = persistWorkspaceTargetRegistry(baseRegistry(rootDir), rootDir); + + const contract = createDevelopmentContract({ + rootDir, + task: contractTask({ + primary: '@alpha', + affected: ['@alpha'], + verification: ['@alpha', 'shared/common'], + }), + authoritativeSources: [{ path: 'docs/spec.md', kind: 'specification', authority: 'required' }], + createdAt: '2026-10-07T18:32:00.000Z', + }); + + const verification = createVerificationRecord({ + contract, + runId: 'run-target-stale', + role: 'spec-verifier', + sourceFingerprint: contract.sourceFingerprint, + createdAt: '2026-10-07T18:33:00.000Z', + criteria: [{ + id: 'AC-TARGET', + status: 'PASS', + evidence: [{ type: 'test', id: 'target-binding-pass' }], + }], + }); + const review = createReviewResult({ + contract, + runId: 'run-target-stale', + role: 'code-reviewer', + sourceFingerprint: contract.sourceFingerprint, + createdAt: '2026-10-07T18:34:00.000Z', + findings: [], + }); + assert.equal(decideAcceptance({ contract, verification, reviews: [review], rootDir }).state, 'ACCEPTED'); + + const changed = structuredClone(saved.registry); + changed.targets['@alpha'].path = 'units/beta'; + persistWorkspaceTargetRegistry(changed, rootDir, { expectedFingerprint: saved.fingerprint }); + + const blocked = decideAcceptance({ contract, verification, reviews: [review], rootDir }); + assert.equal(blocked.state, 'BLOCKED'); + assert.ok(blocked.blockers.some((item) => item.code === 'STALE_CONTRACT')); +}); + +test('workspace target binding rejects unknown target references', (t) => { + const rootDir = tempProject(t); + const registry = normalizeWorkspaceTargetRegistry(baseRegistry(rootDir), { rootDir }); + assert.throws(() => createWorkspaceTargetBinding(registry, { primary: 'not-there' }), /Unknown workspace target/); +}); + +test('repository workspace registry is valid and routes its declared validation capability', () => { + const registry = loadWorkspaceTargetRegistry(REPO_ROOT); + assert.ok(registry.targets['dkf-framework']); + const route = resolveCapabilityRoute(registry, { + capability: 'release-validation', + commandName: 'test', + rootDir: REPO_ROOT, + }); + assert.equal(route.targetId, 'dkf-framework'); + assert.equal(route.command, 'npm run release:validate'); +}); + +test('Workspace Target Registry schema and Development Contract schema expose v0.12 T02 contracts', () => { + const workspaceSchema = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'schemas', 'workspace-target-registry.schema.json'), 'utf8')); + const contractSchema = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'schemas', 'development-contract.schema.json'), 'utf8')); + assert.equal(workspaceSchema.title, 'Development Kit Workspace Target Registry'); + assert.ok(contractSchema.properties.schemaVersion.enum.includes('1.3.0')); + assert.ok(contractSchema.properties.workspaceTargets); +});